scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

plugins and precomputed data

Browse source 
So in some cases, when we do a query, the results of one
part of the query are innately shared-across-nodes; for
instance, a hypothetical Distinct query. More generally,
we allow cross-index queries; calls can have "index=foo"
in them.

This patch lets us handle that without duplicating that
query all over. Before we actually start doing the
separate calls, we run the query once from the coordinating
node, then patch the results in, and send relevant subsets
over to each client, etcetera. Also provides slightly
friendlier (and I hope faster) support for converting
bitmaps to/from sets of rows.

We also add an extension interface, and some fancy stuff
to let us define new calls, which use this. They're sort
of tied together because the first extension I wanted to
implement needed precomputed calls. The extension API
lets us create extensions using `pkg/plugin` (with all its
associated limitations, unfortunately), then query them
at load time for functionality.

This also implies some revamping of the argument
validation for PQL, like verifying that functions exist
and knowing things about their argument types.

So basically this is an overly intrusive patch, and would
be better as separate patches, but they're hard to detangle.

add trivial execution-time profiling

What if you could ?profile=true on a query and get some
numbers back? That'd be really cool.

We already have tracing/spans, but right now, those only generate
any data if you have something set up for them to trace to. Add a
fancy wrapper that lets us generate our own tracing data, and dump
it into the request response, if ?profile=true.

add a sample extension, add missing features to extension interface

Implement a naive probabilistic filter extension as an example of
what an extension looks like. In the process, discover multiple
omissions in the bitmap API. Well, I did *say* it was experimental.
This commit is contained in:
Seebs 2019-05-01 16:52:24 -05:00
parent b25eb8f596
commit 3b696da34a
21 changed files with 4421 additions and 3184 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
api.go
View file

@ -152,6 +152,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
ExcludeRowAttrs: req.ExcludeRowAttrs, // NOTE: Kept for Pilosa 1.x compat.
ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat.
ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat.
EmbeddedData: req.EmbeddedData, // precomputed values that needed to be passed with the request
}
resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
if err != nil {

65
encoding/proto/proto.go
View file

@ -396,14 +396,19 @@ func encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) *internal.Import
}
func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest {
return &internal.QueryRequest{
r := &internal.QueryRequest{
Query: m.Query,
Shards: m.Shards,
ColumnAttrs: m.ColumnAttrs,
Remote: m.Remote,
ExcludeRowAttrs: m.ExcludeRowAttrs,
ExcludeColumns: m.ExcludeColumns,
EmbeddedData: make([]*internal.Row, len(m.EmbeddedData)),
}
for i := range m.EmbeddedData {
r.EmbeddedData[i] = encodeRow(m.EmbeddedData[i])
}
return r
}
func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
@ -416,6 +421,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
pb.Results[i] = &internal.QueryResult{}
switch result := m.Results[i].(type) {
case pilosa.SignedRow:
pb.Results[i].Type = queryResultTypeSignedRow
pb.Results[i].SignedRow = encodeSignedRow(result)
case *pilosa.Row:
pb.Results[i].Type = queryResultTypeRow
pb.Results[i].Row = encodeRow(result)
@ -446,7 +454,7 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
case nil:
pb.Results[i].Type = queryResultTypeNil
default:
panic(fmt.Errorf("unknown type: %d", pb.Results[i].Type))
panic(fmt.Errorf("unknown type: %T", m.Results[i]))
}
}
@ -966,6 +974,10 @@ func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) {
m.Remote = pb.Remote
m.ExcludeRowAttrs = pb.ExcludeRowAttrs
m.ExcludeColumns = pb.ExcludeColumns
m.EmbeddedData = make([]*pilosa.Row, len(pb.EmbeddedData))
for i := range pb.EmbeddedData {
m.EmbeddedData[i] = decodeRow(pb.EmbeddedData[i])
}
}
func decodeImportRequest(pb *internal.ImportRequest, m *pilosa.ImportRequest) {
@ -1068,10 +1080,13 @@ const (
queryResultTypeGroupCounts
queryResultTypeRowIdentifiers
queryResultTypePair
queryResultTypeSignedRow
)
func decodeQueryResult(pb *internal.QueryResult) interface{} {
switch pb.Type {
case queryResultTypeSignedRow:
return decodeSignedRow(pb.SignedRow)
case queryResultTypeRow:
return decodeRow(pb.Row)
case queryResultTypePairs:
@ -1099,14 +1114,31 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} {
// DecodeRow converts r from its internal representation.
func decodeRow(pr *internal.Row) *pilosa.Row {
if pr == nil {
return nil
return pilosa.NewRow()
}
r := pilosa.NewRow()
var r *pilosa.Row
if len(pr.Roaring) > 0 {
r = pilosa.NewRowFromRoaring(pr.Roaring)
} else {
r = pilosa.NewRow()
for _, v := range pr.Columns {
r.SetBit(v)
}
}
r.Attrs = decodeAttrs(pr.Attrs)
r.Keys = pr.Keys
for _, v := range pr.Columns {
r.SetBit(v)
return r
}
func decodeSignedRow(pr *internal.SignedRow) pilosa.SignedRow {
if pr == nil {
return pilosa.SignedRow{}
}
r := pilosa.SignedRow{
Pos: decodeRow(pr.Pos),
Neg: decodeRow(pr.Neg),
}
return r
}
@ -1213,16 +1245,29 @@ func encodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet {
}
}
func encodeSignedRow(r pilosa.SignedRow) *internal.SignedRow {
ir := &internal.SignedRow{
Pos: encodeRow(r.Pos),
Neg: encodeRow(r.Neg),
}
return ir
}
func encodeRow(r *pilosa.Row) *internal.Row {
if r == nil {
return nil
}
return &internal.Row{
Columns: r.Columns(),
Keys: r.Keys,
Attrs: encodeAttrs(r.Attrs),
ir := &internal.Row{
Keys: r.Keys,
Attrs: encodeAttrs(r.Attrs),
}
if false {
ir.Columns = r.Columns()
} else {
ir.Roaring = r.Roaring()
}
return ir
}
func encodeRowIdentifiers(r pilosa.RowIdentifiers) *internal.RowIdentifiers {

475
executor.go
View file

@ -22,7 +22,9 @@ import (
"sync"
"time"
"github.com/pilosa/pilosa/v2/ext"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/shardwidth"
"github.com/pilosa/pilosa/v2/tracing"
"github.com/pkg/errors"
@ -57,6 +59,12 @@ type executor struct {
workersWG sync.WaitGroup
workerPoolSize int
work chan job
// global registry to check for name clashes
additionalOps map[string]*ext.BitmapOp
// typed registries we can use in lookups
additionalBitmapOps map[string]ext.BitmapOpBitmap
additionalCountOps map[string]ext.BitmapOpUnaryCount
additionalFieldOps map[string]ext.BitmapOpBSIBitmap
}
// executorOption is a functional option type for pilosa.Executor
@ -109,6 +117,38 @@ func (e *executor) Close() error {
return nil
}
func (e *executor) registerOps(ops []ext.BitmapOp) error {
if e.additionalOps == nil {
e.additionalOps = make(map[string]*ext.BitmapOp)
e.additionalBitmapOps = make(map[string]ext.BitmapOpBitmap)
e.additionalCountOps = make(map[string]ext.BitmapOpUnaryCount)
e.additionalFieldOps = make(map[string]ext.BitmapOpBSIBitmap)
}
for i, op := range ops {
name := op.Name
if _, exists := e.additionalOps[name]; exists {
return fmt.Errorf("op name '%s' already defined", name)
}
e.additionalOps[name] = &ops[i]
typ := ops[i].Func.BitmapOpType()
switch {
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:
e.additionalCountOps[name] = ops[i].Func.(ext.BitmapOpUnaryCount)
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap:
e.additionalBitmapOps[name] = ops[i].Func.(ext.BitmapOpBitmap)
case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap:
if fn, ok := ops[i].Func.(ext.BitmapOpBSIBitmapPrecall); ok {
e.additionalFieldOps[name] = ext.BitmapOpBSIBitmap(fn)
} else {
e.additionalFieldOps[name] = ops[i].Func.(ext.BitmapOpBSIBitmap)
}
default:
return fmt.Errorf("unsupported types for '%s': input type %d, output type %d", name, typ.Input, typ.Output)
}
}
return nil
}
// Execute executes a PQL query.
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute")
@ -237,6 +277,97 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr
return ax, nil
}
// handlePreCalls traverses the call tree looking for calls that need
// precomputed values. Right now, that's just Distinct.
func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
if c.Name == "Precomputed" {
idx := c.Args["valueidx"].(int64)
if idx >= 0 && idx < int64(len(opt.EmbeddedData)) {
row := opt.EmbeddedData[idx]
c.Precomputed = make(map[uint64]interface{}, len(row.segments))
for _, segment := range row.segments {
c.Precomputed[segment.shard] = &Row{segments: []rowSegment{segment}}
}
} else {
return fmt.Errorf("no precomputed data! index %d, len %d", idx, len(opt.EmbeddedData))
}
return nil
}
newIndex := c.CallIndex()
// A cross-index query is handled by precall. This is inefficient,
// but we have to do it for now because shards might be different and
// we haven't implemented the local precalls that would be enough
// in some cases.
//
// This makes simple cross-index queries noticably inefficient.
//
// If you're here because of that: We should be using PrecallLocal
// in cases where the call isn't already PrecallGlobal, and
// PrecallLocal should wait until we're running on a specific node
// to do the farming-out of just the sub-queries it has to run
// for its local shards.
//
// As is, we have one node querying every node, then sending out
// all the data to every node, including the data that node already
// has. We could reduce the actual copying around dramatically,
// but only in the cases where local is good enough -- not something
// like Distinct, where you can't predict output shard for a result
// from the shard being queried.
if newIndex != "" && newIndex != index {
c.Type = pql.PrecallGlobal
index = newIndex
}
if c.Type == pql.PrecallNone {
// otherwise, handle the children
return e.handlePreCallChildren(ctx, index, c, shards, opt)
}
// We don't try to handle sub-calls from here. I'm not 100%
// sure that's right, but I think the fact that they're happening
// inside a precomputed call may mean they need different
// handling. In any event, the sub-calls will get handled by
// the executeCall when it gets to them...
// We set c to look like a normal call, and actually execute it:
c.Type = pql.PrecallNone
// possibly override call index.
v, err := e.executeCall(ctx, index, c, shards, opt)
if err != nil {
return err
}
var row *Row
switch r := v.(type) {
case *Row:
row = r
case SignedRow:
row = r.Pos
default:
return fmt.Errorf("precomputed call %s returned unexpected non-Row data: %T", c.Name, v)
}
c.Children = []*pql.Call{}
c.Name = "Precomputed"
c.Args = map[string]interface{}{"valueidx": len(opt.EmbeddedData)}
// stash a copy of the full results, which can be forwarded to other
// shards if the query has to go to them
opt.EmbeddedData = append(opt.EmbeddedData, row)
// and stash a copy locally, so local calls can use it
c.Precomputed = make(map[uint64]interface{}, len(row.segments))
for _, segment := range row.segments {
c.Precomputed[segment.shard] = &Row{segments: []rowSegment{segment}}
}
return nil
}
// handlePreCallChildren handles any pre-calls in the children of a given call.
func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
for i := range c.Children {
if err := e.handlePreCalls(ctx, index, c.Children[i], shards, opt); err != nil {
return err
}
}
return nil
}
func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.execute")
defer span.Finish()
@ -270,7 +401,27 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar
return nil, err
}
v, err := e.executeCall(ctx, index, call, shards, opt)
// If you actually make a top-level Distinct call, you
// want a SignedRow back. Otherwise, it's something else
// that will be using it as a row, and we only care
// about the positive values, because only positive values
// are valid column IDs. So we don't actually eat top-level
// pre calls.
err := e.handlePreCallChildren(ctx, index, call, shards, opt)
if err != nil {
return nil, err
}
var v interface{}
// Top-level calls don't need to precompute cross-index things,
// because we can just pick whatever index we want, but we
// still need to handle them. Since everything else was
// already precomputed by handlePreCallChildren, though,
// we don't need this logic in executeCall.
if newIndex := call.CallIndex(); newIndex != "" {
v, err = e.executeCall(ctx, newIndex, call, shards, opt)
} else {
v, err = e.executeCall(ctx, index, call, shards, opt)
}
if err != nil {
return nil, err
}
@ -299,6 +450,14 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
}
// Special handling for mutation and top-n calls.
if op, ok := e.additionalCountOps[c.Name]; ok {
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeGenericCount(ctx, index, c, op, shards, opt)
}
if op, ok := e.additionalFieldOps[c.Name]; ok {
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeGenericField(ctx, index, c, op, shards, opt)
}
switch c.Name {
case "Sum":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
@ -343,6 +502,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
return e.executeOptionsCall(ctx, index, c, shards, opt)
case "IncludesColumn":
return e.executeIncludesColumnCall(ctx, index, c, shards, opt)
case "Precomputed":
return e.executePrecomputedCall(ctx, index, c, shards, opt)
default:
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeBitmapCall(ctx, index, c, shards, opt)
@ -501,6 +662,37 @@ func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, sh
return other, nil
}
// executeGenericField executes a generic call on a field. Note that in this
// implementation, the operation is always a BSI op.
func (e *executor) executeGenericField(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shards []uint64, opt *execOptions) (SignedRow, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericField")
span.LogKV("name", c.Name)
defer span.Finish()
if field := c.Args["field"]; field == "" {
return SignedRow{}, fmt.Errorf("plugin operation %s(): field required", c.Name)
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
return e.executeGenericFieldShard(ctx, index, c, op, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(SignedRow)
return other.union(v.(SignedRow))
}
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return SignedRow{}, err
}
other, _ := result.(SignedRow)
return other, nil
}
// executeMin executes a Min() call.
func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMin")
@ -641,6 +833,41 @@ func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call,
return e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
}
// executePrecomputedCall pretends to execute a call that we have a precomputed value for.
func (e *executor) executePrecomputedCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executePrecomputedCall")
defer span.Finish()
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
if c.Precomputed != nil {
return c.Precomputed[shard], nil
}
// This might not be an error -- if there were no values, we will not have created
// the corresponding row.
return NewRow(), nil
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(*Row)
if other == nil {
other = NewRow()
}
other.Merge(v.(*Row))
return other
}
other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return nil, errors.Wrap(err, "map reduce")
}
row, _ := other.(*Row)
return row, nil
}
// executeBitmapCall executes a call that returns a bitmap.
func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCall")
@ -719,6 +946,13 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard")
defer span.Finish()
if _, ok := e.additionalCountOps[c.Name]; ok {
return nil, fmt.Errorf("count op %s used as bitmap call", c.Name)
}
if op, ok := e.additionalBitmapOps[c.Name]; ok {
return e.executeGenericBitmapShard(ctx, index, c, op, shard)
}
switch c.Name {
case "Row", "Range":
return e.executeRowShard(ctx, index, c, shard)
@ -734,11 +968,63 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *
return e.executeNotShard(ctx, index, c, shard)
case "Shift":
return e.executeShiftShard(ctx, index, c, shard)
case "Precomputed":
return e.executePrecomputedCallShard(ctx, index, c, shard)
default:
return nil, fmt.Errorf("unknown call: %s", c.Name)
}
}
// executeGenericFieldShard executes a generic/extension command on a
// single shard. Note that in this implementation, the op is always
// a BSI op.
func (e *executor) executeGenericFieldShard(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shard uint64) (SignedRow, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericShard")
defer span.Finish()
var filter *Row
var filterBitmap *roaring.Bitmap
if len(c.Children) == 1 {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return SignedRow{}, errors.Wrap(err, "executing bitmap call")
}
filter = row
if filter != nil && len(filter.segments) > 0 {
filterBitmap = filter.segments[0].data
}
}
fieldName, _ := c.Args["field"].(string)
field := e.Holder.Field(index, fieldName)
if field == nil {
return SignedRow{}, nil
}
bsig := field.bsiGroup(fieldName)
if bsig == nil {
return SignedRow{}, nil
}
fragment := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
if fragment == nil {
return SignedRow{}, nil
}
var out ext.SignedBitmap
if filterBitmap != nil {
out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{WrapBitmap(filterBitmap)}, c.Args)
} else {
out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{}, c.Args)
}
return SignedRow{
Neg: NewRowFromBitmap(UnwrapBitmap(out.Neg)),
Pos: NewRowFromBitmap(UnwrapBitmap(out.Pos)),
}, nil
}
// executeSumCountShard calculates the sum and count for bsiGroups on a shard.
func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSumCountShard")
@ -1749,6 +2035,44 @@ func (e *executor) executeIntersectShard(ctx context.Context, index string, c *p
return other, nil
}
// executeGenericBitmapShard executes a generic bitmap call for a local shard.
func (e *executor) executeGenericBitmapShard(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBitmap, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericBitmapShard")
defer span.Finish()
if op.BitmapOpArity() == ext.OpArityUnary {
if len(c.Children) != 1 {
return nil, fmt.Errorf("%s needs exactly one row parameter", c.Name)
}
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return nil, err
}
return row.GenericUnaryOp(op.BitmapOpFunc(), c.Args), nil
}
var err error
rows := make([]*Row, len(c.Children))
for i, input := range c.Children {
rows[i], err = e.executeBitmapCallShard(ctx, index, input, shard)
if err != nil {
return nil, err
}
}
var other *Row
switch op.BitmapOpArity() {
case ext.OpArityBinary:
other = rows[0]
for _, row := range rows[1:] {
other = other.GenericBinaryOp(op.BitmapOpFunc(), row, c.Args)
}
case ext.OpArityNary:
other = rows[0].GenericNaryOp(op.BitmapOpFunc(), rows[1:], c.Args)
}
other.invalidateCount()
return other, nil
}
// executeUnionShard executes a union() call for a local shard.
func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard")
@ -1793,6 +2117,25 @@ func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Cal
return other, nil
}
// executePrecomputedCallShard pretends to execute a precomputed call for a local shard.
func (e *executor) executePrecomputedCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
if c.Precomputed != nil {
v := c.Precomputed[shard]
if v == nil {
return NewRow(), nil
}
if r, ok := v.(*Row); ok {
if r != nil {
return r, nil
} else {
return NewRow(), nil
}
}
return nil, fmt.Errorf("precomputed value is not a row: %T", v)
}
return nil, fmt.Errorf("per-shard: missing precomputed values for shard %d", shard)
}
// executeNotShard executes a not() call for a local shard.
func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeNotShard")
@ -1849,6 +2192,41 @@ func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.C
return row.Shift(n)
}
// executeGeneric executes a provided count-like call.
func (e *executor) executeGenericCount(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpUnaryCount, shards []uint64, opt *execOptions) (uint64, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericCount")
defer span.Finish()
if len(c.Children) == 0 {
return 0, fmt.Errorf("%s() requires an input bitmap", c.Name)
} else if len(c.Children) > 1 {
return 0, fmt.Errorf("%s() only accepts a single bitmap input", c.Name)
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return 0, err
}
return row.GenericCount(op, c.Args), nil
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(uint64)
return other + v.(uint64)
}
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return 0, err
}
n, _ := result.(uint64)
return n, nil
}
// executeCount executes a count() call.
func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount")
@ -1946,7 +2324,7 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq
}
// Forward call to remote node otherwise.
res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
if err != nil {
return false, err
}
@ -2222,7 +2600,7 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.
}
// Forward call to remote node otherwise.
res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
if err != nil {
return false, err
}
@ -2257,7 +2635,7 @@ func (e *executor) executeSetValueField(ctx context.Context, index string, c *pq
}
// Forward call to remote node otherwise.
res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
if err != nil {
return false, err
}
@ -2311,7 +2689,7 @@ func (e *executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
resp <- err
}(node)
}
@ -2406,7 +2784,7 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil)
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil, nil)
resp <- err
}(node)
}
@ -2458,7 +2836,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
resp <- err
}(node)
}
@ -2474,15 +2852,16 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
}
// remoteExec executes a PQL query remotely for a set of shards on a node.
func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64) (results []interface{}, err error) { // nolint: interfacer
func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, embed []*Row) (results []interface{}, err error) { // nolint: interfacer
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeExec")
defer span.Finish()
// Encode request object.
pbreq := &QueryRequest{
Query: q.String(),
Shards: shards,
Remote: true,
Query: q.String(),
Shards: shards,
Remote: true,
EmbeddedData: embed,
}
pb, err := e.client.QueryNode(ctx, &node.URI, index, pbreq)
@ -2577,6 +2956,43 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
}
}
// makeEmbeddedDataForShards produces new rows containing the rowSegments
// that would correspond to a given set of shards.
func makeEmbeddedDataForShards(allRows []*Row, shards []uint64) []*Row {
if len(allRows) == 0 || len(shards) == 0 {
return nil
}
newRows := make([]*Row, len(allRows))
for i, row := range allRows {
if row == nil || len(row.segments) == 0 {
continue
}
segments := row.segments
segmentIndex := 0
newRows[i] = &Row{}
for _, shard := range shards {
for segmentIndex < len(segments) && segments[segmentIndex].shard < shard {
segmentIndex++
}
// no more segments in this row
if segmentIndex >= len(segments) {
break
}
if segments[segmentIndex].shard == shard {
newRows[i].segments = append(newRows[i].segments, segments[segmentIndex])
segmentIndex++
if segmentIndex >= len(segments) {
// no more segments, we're done
break
}
}
// if we got here, segments[segmentIndex].shard exists
// but is greater than the current shard, so we continue.
}
}
return newRows
}
func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapper")
defer span.Finish()
@ -2596,7 +3012,11 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
if n.ID == e.Node.ID {
resp.result, resp.err = e.mapperLocal(ctx, nodeShards, mapFn, reduceFn)
} else if !opt.Remote {
results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeShards)
var embeddedRowsForNode []*Row
if opt.EmbeddedData != nil {
embeddedRowsForNode = makeEmbeddedDataForShards(opt.EmbeddedData, nodeShards)
}
results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeShards, embeddedRowsForNode)
if len(results) > 0 {
resp.result = results[0]
}
@ -3026,6 +3446,7 @@ type execOptions struct {
ExcludeRowAttrs bool
ExcludeColumns bool
ColumnAttrs bool
EmbeddedData []*Row
}
// hasOnlySetRowAttrs returns true if calls only contains SetRowAttrs() calls.
@ -3060,6 +3481,36 @@ func needsShards(calls []*pql.Call) bool {
return false
}
// SignedRow represents a signed *Row with two (neg/pos) *Rows.
type SignedRow struct {
Neg *Row `json:"neg"`
Pos *Row `json:"pos"`
}
func (sr *SignedRow) union(other SignedRow) SignedRow {
ret := SignedRow{&Row{}, &Row{}}
// merge in sr
if sr != nil {
if sr.Neg != nil {
ret.Neg = ret.Neg.Union(sr.Neg)
}
if sr.Pos != nil {
ret.Pos = ret.Pos.Union(sr.Pos)
}
}
// merge in other
if other.Neg != nil {
ret.Neg = ret.Neg.Union(other.Neg)
}
if other.Pos != nil {
ret.Pos = ret.Pos.Union(other.Pos)
}
return ret
}
// ValCount represents a grouping of sum & count for Sum() and Average() calls.
type ValCount struct {
Val int64 `json:"value"`

224
ext/ext.go Normal file
View file

@ -0,0 +1,224 @@
// Package ext provides an EXPERIMENTAL AND TEMPORARY interface to use for
// plugin extensions to Pilosa. DO NOT DEVELOP NEW PLUGINS WITH THIS. The
// replacement design is already in process, but it needs more refinement
// to address issues. This one has those issues, and more.
//
// In the current design, plugins will be loaded at runtime using the
// go `plugin` package, so they should be built as a main package using
// the plugin build mode.
//
// Plugins should not import other packages from Pilosa.
//
// To advertise their functionality, plugins define one or more of a
// handful of symbols which will be checked for at plugin load and used
// to register their functionality.
//
// The plugin interface will check for the following function(s). If the
// functions exist, they must have the given signatures. If they return
// a non-nil error, no ops are registered, and the error message will
// be reported in the Pilosa server's logs.
//
// BitmapOps() ([]BitmapOp, error)
//
// These functions may be absent, and may return nil slices; in either
// case, no ops are registered.
package ext
// The Bitmap type represents a Pilosa bitmap, and is used for bitmap
// operations.
type Bitmap interface {
// AddN and RemoveN can be used to add or remove values from a bitmap.
AddN(a ...uint64) (int, error)
RemoveN(a ...uint64) (int, error)
// Lookups
Max() uint64
Min() (uint64, bool)
Count() uint64
Any() bool
Contains(uint64) bool
Slice() []uint64
SliceRange(uint64, uint64) []uint64
// ContainerBits stores the next 1<<16 bits, starting at the provided
// bit index. It may use a provided []uint64 to store them, or may
// provide its own. Don't write to those bits. Offset must be a multiple
// of 1<<16.
ContainerBits(uint64, []uint64) []uint64
// These operators provide existing implemented binary ops.
Intersect(Bitmap) Bitmap
Union(Bitmap) Bitmap
IntersectionCount(Bitmap) uint64
Difference(Bitmap) Bitmap
Xor(Bitmap) Bitmap
Shift(int) (Bitmap, error)
Flip(uint64, uint64) Bitmap
// New() is an atrocity: it creates a new bitmap, unrelated to the
// existing bitmap. This lets you create a new bitmap without having
// imported any of the packages that have bitmap creation tools, because
// the bitmap wrapper type has to give you one.
New() Bitmap
}
// SignedBitmap represents a bitmap that can contain both positive and negative
// values.
type SignedBitmap struct {
Pos, Neg Bitmap
}
// A BitmapOp represents a new bitmap operation that should be exposed
// in PQL.
type BitmapOpInput byte
type BitmapOpOutput byte
type BitmapOpArity byte
type BitmapOpPrecall byte
type BitmapOpType struct {
Input BitmapOpInput
Arity BitmapOpArity
Output BitmapOpOutput
Precall BitmapOpPrecall
}
const (
OpArityUnary = BitmapOpArity(iota)
OpArityBinary
OpArityNary
)
const (
// Unary: Exactly one bitmap.
OpInputBitmap = BitmapOpInput(iota)
// The really weird special case used for BSI, where we end up
// needing to do BSI computations. Arguments will be a
// single BitmapBSI, and a []Bitmap for other operands if any.
OpInputNaryBSI
)
const (
OpOutputCount = BitmapOpOutput(iota)
OpOutputBitmap
OpOutputSignedBitmap
)
const (
OpPrecallNone = BitmapOpPrecall(iota)
OpPrecallGlobal
OpPrecallLocal // unimplemented
)
// Regardless of arity, non-BSI functions should always take []Bitmap.
type BitmapOpFunc interface {
BitmapOpType() BitmapOpType
}
// BitmapOpBitmap should actually always be func([]Bitmap) Bitmap, but
// might be different kinds.
type BitmapOpBitmap interface {
BitmapOpArity() BitmapOpArity
BitmapOpFunc() GenericBitmapOpBitmap
}
// the common underlying type of the other BitmapOpBitmap functions
type GenericBitmapOpBitmap func([]Bitmap, map[string]interface{}) Bitmap
// BitmapBSI represents the way a single BSI field is passed into a function
// which takes a BSI field.
type BitmapBSI struct {
FieldData Bitmap
ShardWidth uint64
Offset int64
Depth uint
}
type BitmapOpBSIBitmap func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap
func (b BitmapOpBSIBitmap) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Output: OpOutputSignedBitmap}
}
type BitmapOpBSIBitmapPrecall func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap
func (b BitmapOpBSIBitmapPrecall) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Precall: OpPrecallGlobal, Output: OpOutputSignedBitmap}
}
type BitmapOpUnaryCount func([]Bitmap, map[string]interface{}) int64
func (b BitmapOpUnaryCount) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputCount}
}
type BitmapOpUnaryBitmap func([]Bitmap, map[string]interface{}) Bitmap
func (b BitmapOpUnaryBitmap) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputBitmap}
}
func (b BitmapOpUnaryBitmap) BitmapOpArity() BitmapOpArity {
return OpArityUnary
}
func (b BitmapOpUnaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap {
return GenericBitmapOpBitmap(b)
}
type BitmapOpBinaryBitmap func([]Bitmap, map[string]interface{}) Bitmap
func (b BitmapOpBinaryBitmap) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputBitmap, Arity: OpArityBinary, Output: OpOutputBitmap}
}
func (b BitmapOpBinaryBitmap) BitmapOpArity() BitmapOpArity {
return OpArityBinary
}
func (b BitmapOpBinaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap {
return GenericBitmapOpBitmap(b)
}
type BitmapOpNaryBitmap func([]Bitmap, map[string]interface{}) Bitmap
func (b BitmapOpNaryBitmap) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputBitmap, Arity: OpArityNary, Output: OpOutputBitmap}
}
func (b BitmapOpNaryBitmap) BitmapOpArity() BitmapOpArity {
return OpArityNary
}
func (b BitmapOpNaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap {
return GenericBitmapOpBitmap(b)
}
// BitmapOp represents an operation to be supported in PQL. Operations
// on bitmaps should always take []Bitmap. Operations on InputNaryBSI should
// take a []Bitmap, plus a Bitmap/shard-width/offset/depth.
//
// Reserved is a list of words to treat as reserved words in a prototype.
// This is not currently used but might be later, and I want to have the
// concept handy now.
type BitmapOp struct {
Name string
Func BitmapOpFunc
Reserved []string
}
// ExtensionInfo tells us about the extension. The ExtensionAPI string
// should be "v0". The version is a human-readable version, use something
// that seems meaningful. Name and Description are reasonably self-explanatory,
// I hope.
//
// Extensions should define a function:
// func ExtensionInfo(extensionAPI string) (*ExtensionInfo, error)
// which reports their extension info if they think they can coexist with that
// API string.
type ExtensionInfo struct {
Name string // Extension name.
Description string // Short description.
Version string // Human-readable version info for extension.
ExtensionAPI string // Extension API version. Should be v0 for now.
License string // License info.
BitmapOps []BitmapOp // List of provided ops.
}

1
ext/samples/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
*/*.so

111
ext/samples/some/some.go Normal file
View file

@ -0,0 +1,111 @@
package main
import (
"fmt"
"math/bits"
"github.com/molecula/apophenia"
"github.com/pilosa/pilosa/v2/ext"
)
// This could be dynamically generated, but for now it's not.
// nolint:unused,deadcode
var extInfoTemplate = &ext.ExtensionInfo{
Name: "some",
Description: "some of the bits/all of the bits/none of the bits",
Version: "0.01",
ExtensionAPI: "v0",
License: "unreleased",
BitmapOps: []ext.BitmapOp{
{Name: "Some", Func: ext.BitmapOpUnaryBitmap(Some), Reserved: []string{"p", "seed"}},
},
}
// ExtensionInfo is the entry point used by the plugin code.
func ExtensionInfo(api string) (*ext.ExtensionInfo, error) { // nolint:unused,deadcode
return extInfoTemplate, nil
}
const batchSize = 1024
// Some returns some of the bits from its first input bitmap. Takes seed (int)
// and p (float) values. Seed defaults to 0.
func Some(inputs []ext.Bitmap, args map[string]interface{}) ext.Bitmap {
if len(inputs) == 0 || inputs[0] == nil {
return nil
}
input := inputs[0]
min, ok := input.Min()
// no bits found?
if !ok {
return nil
}
// start at multiple of 128 not greater than min.
min &^= 127
max := input.Max()
p, ok := args["p"].(float64)
if !ok {
return nil
}
// no bits or impossible probability range
if p <= 0 || p > 1 {
return nil
}
// every bit
if p == 1 {
return inputs[0]
}
// On failure, we default to 0.
seed, _ := args["seed"].(int64)
densityScale := uint64(256)
density := uint64(p * float64(densityScale))
for density == 0 {
densityScale <<= 1
density = uint64(p * float64(densityScale))
// too small
if densityScale > (1 << 32) {
return nil
}
}
w, err := apophenia.NewWeighted(apophenia.NewSequence(seed))
if err != nil {
return nil
}
someBits := input.New()
toAdd := make([]uint64, batchSize)
toAddN := 0
offset := apophenia.OffsetFor(apophenia.SequenceWeighted, 0, 0, 0)
for i := min; i < max; i += 128 {
offset.Lo = i
randomBits := w.Bits(offset, density, densityScale)
bit := uint64(0)
for randomBits.Lo != 0 {
next := uint64(bits.TrailingZeros64(randomBits.Lo) + 1)
randomBits.Lo >>= next
toAdd[toAddN] = next + bit + i
toAddN++
bit += next
}
bit = 64
for randomBits.Hi != 0 {
next := uint64(bits.TrailingZeros64(randomBits.Hi) + 1)
randomBits.Hi >>= next
toAdd[toAddN] = next + bit + i
toAddN++
bit += next
}
if toAddN > (batchSize - 128) {
// ignore error
_, _ = someBits.AddN(toAdd[:toAddN]...)
toAddN = 0
}
}
if toAddN > 0 {
_, _ = someBits.AddN(toAdd[:toAddN]...)
}
return input.Intersect(someBits)
}
func main() {
fmt.Printf("this is a plugin module only.\n")
}

82
extension.go Normal file
View file

@ -0,0 +1,82 @@
package pilosa
import (
"fmt"
"github.com/pilosa/pilosa/v2/ext"
"github.com/pilosa/pilosa/v2/roaring"
)
// WrapBitmap yields an extension-Bitmap from a roaring Bitmap.
func WrapBitmap(bm *roaring.Bitmap) ext.Bitmap {
return wrappedBitmap{bm}
}
// wrappedBitmap is a very shallow glue shim to convert a roaring Bitmap to
// an extension Bitmap.
type wrappedBitmap struct{ *roaring.Bitmap }
// UnwrapBitmap converts an extension-bitmap to its underlying roaring Bitmap.
func UnwrapBitmap(bm ext.Bitmap) *roaring.Bitmap {
if inner, ok := bm.(wrappedBitmap); ok {
if inner.Bitmap != nil {
return inner.Bitmap
}
return roaring.NewFileBitmap()
}
return roaring.NewFileBitmap()
}
func (b wrappedBitmap) Intersect(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Intersect(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) Union(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Union(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) IntersectionCount(other ext.Bitmap) uint64 {
return b.Bitmap.IntersectionCount(other.(wrappedBitmap).Bitmap)
}
func (b wrappedBitmap) Difference(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Difference(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) Xor(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Xor(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) Shift(n int) (ext.Bitmap, error) {
shifted, err := b.Bitmap.Shift(n)
return wrappedBitmap{shifted}, err
}
func (b wrappedBitmap) Flip(start, last uint64) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Flip(start, last)}
}
func (b wrappedBitmap) New() ext.Bitmap {
return WrapBitmap(roaring.NewFileBitmap())
}
// ContainerBits tries to get one container's worth of bits.
func (b wrappedBitmap) ContainerBits(offset uint64, target []uint64) (out []uint64) {
// it's an error to call this with a non-container-aligned offset
if offset&0xFFFF != 0 {
return nil
}
if b.Bitmap == nil {
fmt.Printf("ContainerBits on bitmap with no contents\n")
return nil
}
if b.Bitmap.Containers == nil {
fmt.Printf("ContainerBits on bitmap with nil Containers\n")
return nil
}
c := b.Bitmap.Containers.Get(offset >> 16)
if c == nil {
return nil
}
return c.AsBitmap(target)
}

3
go.mod
View file

@ -18,6 +18,7 @@ require (
github.com/gorilla/mux v1.7.0
github.com/hashicorp/memberlist v0.1.3
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b
github.com/opentracing/opentracing-go v1.1.0
github.com/pelletier/go-toml v1.2.0
github.com/pkg/errors v0.8.1
@ -44,4 +45,4 @@ require (
modernc.org/strutil v1.0.0
)
go 1.11
go 1.13

3
go.sum
View file

@ -86,6 +86,8 @@ github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y=
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
@ -96,6 +98,7 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU=
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

4
handler.go
View file

@ -48,6 +48,10 @@ type QueryRequest struct {
// Should we profile this query?
Profile bool
// Additional data associated with the query, in cases where there's
// row-style inputs for precomputed values.
EmbeddedData []*Row
}
// QueryResponse represent a response from a processed query.

2850
internal/private.pb.go
View file

File diff suppressed because it is too large Load diff

2962
internal/public.pb.go
View file

File diff suppressed because it is too large Load diff

10
internal/public.proto
View file

@ -6,6 +6,12 @@ message Row {
repeated uint64 Columns = 1;
repeated string Keys = 3;
repeated Attr Attrs = 2;
bytes Roaring = 4;
}
message SignedRow {
Row Pos = 1;
Row Neg = 2;
}
message RowIdentifiers {
@ -61,6 +67,7 @@ message QueryRequest {
bool Remote = 5;
bool ExcludeRowAttrs = 6;
bool ExcludeColumns = 7;
repeated Row EmbeddedData = 8;
}
message QueryResponse {
@ -79,6 +86,7 @@ message QueryResult {
repeated uint64 RowIDs = 7;
repeated GroupCount GroupCounts = 8;
RowIdentifiers RowIdentifiers = 9;
SignedRow SignedRow = 10;
}
message ImportRequest {
@ -120,4 +128,4 @@ message ImportRoaringRequestView {
message ImportRoaringRequest {
bool Clear = 1;
repeated ImportRoaringRequestView views = 2;
}
}

268
pql/ast.go
View file

@ -17,10 +17,13 @@ package pql
import (
"bytes"
"fmt"
"reflect"
"sort"
"strconv"
"strings"
"time"
"github.com/pilosa/pilosa/v2/ext"
)
// Query represents a PQL query.
@ -259,11 +262,250 @@ type callStackElem struct {
inList bool
}
// Call represents a function call in the AST.
// Some call types may require special handling, which needs to occur
// before distributing processing to individual shards.
type CallType byte
const (
// Normal calls can be executed per shard.
PrecallNone = CallType(iota)
// PreCallGlobal indicates a call which must be run globally *before*
// distributing the call to other shards. Example: A Distinct query,
// where every shard could potentially produce results for any shard,
// so you have to produce the results up front.
PrecallGlobal
// PreCallPerNode indicates a call which needs to be run per-shard
// in a way that lets it be done on each shard, but where it should
// be done prior to spawning per-shard goroutines. Example:
// A cross-index query, where each local shard may or may not need
// to get data from a remote node, but batches of shards can
// probably be gotten from the same remote node.
PrecallPerNode
)
// Call represents a function call in the AST. The Precomputed field
// is used by the executor to handle non-standard call types; it does
// these by actually executing them separately, then replacing them
// in the call tree with a new call using the special precomputed
// type, with the Precomputed field set to a map from shards to results.
type Call struct {
Name string
Args map[string]interface{}
Children []*Call
Name string
Args map[string]interface{}
Children []*Call
Type CallType
Precomputed map[uint64]interface{}
}
// callInfo defines the arguments allowed for a particular PQL call, and
// possibly things about its semantics. If allowUnknown is true, unfamiliar
// non-reserved names are allowed on the assumption that they're field names.
// Otherwise, only those names explicitly listed are allowed. Reserved args
// (those with a leading underscore) are never allowed unless explicitly
// present.
//
// The prototypes map maps from argument names to a value. If the value is
// non-nil, the argument will be checked for type-matching. So, for instance,
// `x: 10` would indicate that x must be an int.
type callInfo struct {
allowUnknown bool
prototypes map[string]interface{}
callType CallType
}
// We want to be able to accept either a string or int64 for
// field names. Special-case type:
type stringOrInt64Type struct{}
var stringOrInt64 stringOrInt64Type
var allowUnderField = callInfo{
allowUnknown: true,
prototypes: map[string]interface{}{
"_field": "",
},
}
var allowField = callInfo{
allowUnknown: false,
prototypes: map[string]interface{}{
"field": "",
},
}
var callInfoByFunc = map[string]callInfo{
// the easy cases: things that take arbitrary inputs, because they're
// taking field=value cases
"Bitmap": {allowUnknown: true},
"Count": {allowUnknown: true},
"Row": {allowUnknown: true},
"Range": {allowUnknown: true},
// allow only "field=X" cases with string field names
"Max": allowField,
"Min": allowField,
"Sum": allowField,
// only take other calls, should never have "args"
"Difference": {allowUnknown: false},
"Intersect": {allowUnknown: false},
"Not": {allowUnknown: false},
"ClearRow": {allowUnknown: true},
"Store": {allowUnknown: true},
"MinRow": allowField,
"MaxRow": allowField,
"Rows": {
allowUnknown: false,
prototypes: map[string]interface{}{
"_field": "",
"field": "",
"limit": int64(0),
"column": nil,
"previous": nil,
"from": nil,
"to": nil,
},
},
"Shift": {allowUnknown: false,
prototypes: map[string]interface{}{
"n": int64(0),
},
},
"Union": {allowUnknown: false},
"Xor": {allowUnknown: false},
// things that take _field
"TopN": allowUnderField,
// special cases:
"Clear": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_col": stringOrInt64,
},
},
"GroupBy": {
allowUnknown: false,
prototypes: map[string]interface{}{
"filter": nil,
"limit": int64(0),
"previous": nil,
},
},
"Options": {
allowUnknown: false,
prototypes: map[string]interface{}{
"excludeRowAttrs": true,
"excludeColumns": true,
"columnAttrs": true,
"shards": nil,
},
},
"Set": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_col": stringOrInt64,
"_timestamp": "",
},
},
"Precomputed": {
allowUnknown: true,
},
"SetBit": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_col": stringOrInt64,
},
},
"SetRowAttrs": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_field": "",
"_row": stringOrInt64,
},
},
"SetColumnAttrs": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_field": "",
"_col": stringOrInt64,
},
},
"IncludesColumn": {
allowUnknown: false,
prototypes: map[string]interface{}{
"column": stringOrInt64,
},
},
}
// RegisterPluginFuncs adds arg validation for plugin funcs. Not very good
// arg validation.
func RegisterPluginFuncs(ops []ext.BitmapOp) {
for _, op := range ops {
// ignore overlap for now. This should change.
if _, ok := callInfoByFunc[op.Name]; ok {
continue
}
ci := callInfo{allowUnknown: true}
if len(op.Reserved) > 0 {
// mark these as valid/known reserved words
ci.prototypes = make(map[string]interface{})
for _, res := range op.Reserved {
ci.prototypes[res] = nil
}
}
t := op.Func.BitmapOpType()
if t.Precall == ext.OpPrecallGlobal {
ci.callType = PrecallGlobal
}
callInfoByFunc[op.Name] = ci
}
}
// CheckCallInfo tries to validate that arguments are correct and valid for the
// given call. It does not guarantee checking all possible errors; for instance,
// if an argument is a field name, CheckCallInfo can't validate that the field
// exists. It also updates with information like whether the call is expected
// to require precalling.
func (c *Call) CheckCallInfo() error {
valid, ok := callInfoByFunc[c.Name]
if !ok {
return fmt.Errorf("no arg validation for '%s'", c.Name)
}
c.Type = valid.callType
for k, v := range c.Args {
acceptable, ok := valid.prototypes[k]
if !ok && !valid.allowUnknown {
return fmt.Errorf("'%s': unknown arg '%s'", c.String(), k)
}
if !ok && strings.HasPrefix(k, "_") {
return fmt.Errorf("'%s': unknown reserved arg '%s'", c.String(), k)
}
if acceptable == nil {
continue
}
// if the types are identical, that's fine
if reflect.TypeOf(acceptable) == reflect.TypeOf(v) {
continue
}
if reflect.TypeOf(acceptable) == reflect.TypeOf(stringOrInt64) {
switch v.(type) {
case string, int64:
continue
default:
return fmt.Errorf("'%s': arg '%s' needed a string or integer value, got %T.",
c.String(), k, v)
}
}
return fmt.Errorf("'%s': arg '%s' wrong type (got %T, expected %T)",
c.String(), k, v, acceptable)
}
// call-specific checking
for _, child := range c.Children {
if err := child.CheckCallInfo(); err != nil {
return err
}
}
return nil
}
// FieldArg determines which key-value pair contains the field and rowID,
@ -283,13 +525,29 @@ func IsReservedArg(name string) bool {
return true
}
switch name {
case "from", "to":
case "from", "to", "index":
return true
default:
return false
}
}
// CallIndex handles guessing whether we've been asked to apply this to a
// different index. An empty string means "no".
func (c *Call) CallIndex() string {
if index, ok := c.Args["_index"]; ok {
if index, ok := index.(string); ok {
return index
}
}
if index, ok := c.Args["index"]; ok && index != "" {
if index, ok := index.(string); ok {
return index
}
}
return ""
}
// BoolArg is for reading the value at key from call.Args as a bool. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. The value is assumed to be a bool. An error is

8
pql/parser.go
View file

@ -82,6 +82,14 @@ func (p *parser) Parse() (*Query, error) {
panic(v)
}
}
for _, call := range p.Query.Calls {
if call == nil {
return nil, fmt.Errorf("unexpected nil Call in query's call list")
}
if err := call.CheckCallInfo(); err != nil {
return nil, err
}
}
return &p.Query, nil
}

16
pql/parser_test.go
View file

@ -75,12 +75,12 @@ func TestParser_Parse(t *testing.T) {
// Parse with only arguments.
t.Run("ArgumentsOnly", func(t *testing.T) {
q, err := pql.ParseString(`MyCall( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\" )`)
q, err := pql.ParseString(`Row( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\" )`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "MyCall",
Name: "Row",
Args: map[string]interface{}{
"key": "value",
"foo": "bar",
@ -98,12 +98,12 @@ func TestParser_Parse(t *testing.T) {
// Parse with float arguments.
t.Run("WithFloatArgs", func(t *testing.T) {
q, err := pql.ParseString(`MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9)`)
q, err := pql.ParseString(`Row( key=12.25, foo= 13.167, bar=2., baz=0.9)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "MyCall",
Name: "Row",
Args: map[string]interface{}{
"key": 12.25,
"foo": 13.167,
@ -118,12 +118,12 @@ func TestParser_Parse(t *testing.T) {
// Parse with float arguments.
t.Run("WithNegativeArgs", func(t *testing.T) {
q, err := pql.ParseString(`MyCall( key=-12.25, foo= -13)`)
q, err := pql.ParseString(`Row( key=-12.25, foo= -13)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "MyCall",
Name: "Row",
Args: map[string]interface{}{
"key": -12.25,
"foo": int64(-13),
@ -173,12 +173,12 @@ func TestParser_Parse(t *testing.T) {
// Parse with condition arguments.
t.Run("WithCondition", func(t *testing.T) {
q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "MyCall",
Name: "Row",
Args: map[string]interface{}{
"key": "foo",
"x": &pql.Condition{Op: pql.EQ, Value: 12.25},

20
pql/pqlpeg_test.go
View file

@ -46,7 +46,7 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9
t.Fatalf("Failed, got: %s", q)
}
_, err = ParseString("C(a=falsen0)")
_, err = ParseString("Row(a=falsen0)")
if err != nil {
t.Fatalf("falsen0 should have been parsed as a string")
}
@ -109,15 +109,15 @@ func TestPEGWorking(t *testing.T) {
ncalls: 2},
{
name: "SetWithArbCall",
input: "Set(1, a=4)Blerg(z=ha)",
input: "Set(1, a=4)Row(z=ha)",
ncalls: 2},
{
name: "SetArbSet",
input: "Set(1, a=4)Blerg(z=ha)Set(2, z=99)",
input: "Set(1, a=4)Row(z=ha)Set(2, z=99)",
ncalls: 3},
{
name: "ArbSetArb",
input: "Arb(q=1, a=4)Set(1, z=9)Arb(z=99)",
input: "Row(q=1, a=4)Set(1, z=9)Row(z=99)",
ncalls: 3},
{
name: "SetStringArg",
@ -161,11 +161,11 @@ func TestPEGWorking(t *testing.T) {
ncalls: 1},
{
name: "double quoted args",
input: `B(a="zm''e")`,
input: `Row(a="zm''e")`,
ncalls: 1},
{
name: "single quoted args",
input: `B(a='zm""e')`,
input: `Row(a='zm""e')`,
ncalls: 1},
{
name: "SetRowAttrs",
@ -320,7 +320,7 @@ func TestPEGErrors(t *testing.T) {
input: "Set(, 1, a=4)"},
{
name: "StartinCommaArb",
input: "Zeeb(, a=4)"},
input: "Row(, a=4)"},
{
name: "SetRowAttrs0args",
input: "SetRowAttrs(blah, 9)"},
@ -585,11 +585,11 @@ func TestPQLDeepEquality(t *testing.T) {
}},
{
name: "Weird dash",
call: "Sum(field-=f)",
call: "Count(dashy-=f)",
exp: &Call{
Name: "Sum",
Name: "Count",
Args: map[string]interface{}{
"field-": "f",
"dashy-": "f",
},
}},
{

39
roaring/container_stash.go
View file

@ -416,6 +416,45 @@ func (c *Container) bitmap() []uint64 {
return *(*[]uint64)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(c.pointer)), Len: int(c.len), Cap: int(c.cap)}))
}
// AsBitmap yields a 65k-bit bitmap, storing it in the target if a target
// is provided. The target should be zeroed, or this becomes an implicit
// union.
func (c *Container) AsBitmap(target []uint64) (out []uint64) {
if c.typeID == containerBitmap {
return c.bitmap()
}
// Reminder: len(nil) == 0.
if len(target) < 1024 {
out = make([]uint64, 1024)
} else {
out = target
}
if c.typeID == containerArray {
a := c.array()
for _, v := range a {
out[v/64] |= 1 << (v % 64)
}
return out
}
if c.typeID == containerRun {
runs := c.runs()
for _, r := range runs {
splatRun(out, r)
}
return out
}
// in theory this shouldn't happen?
return out
}
func splatRun(into []uint64, from interval16) {
// TODO this can be ~64x faster for long runs by setting maxBitmap instead of single bits
//note v must be int or will overflow
for v := int(from.start); v <= int(from.last); v++ {
into[v/64] |= (uint64(1) << uint(v%64))
}
}
// setBitmap stores a set of uint64s as data.
func (c *Container) setBitmap(bitmap []uint64) {
if c == nil || c.frozen() {

175
roaring/roaring.go
View file

@ -21,6 +21,7 @@ import (
"hash/fnv"
"io"
"math/bits"
"reflect"
"sort"
"unsafe"
@ -433,6 +434,13 @@ func (b *Bitmap) Min() (uint64, bool) {
return v, !eof
}
// MinAt returns the lowest value in the bitmap at least equal to its argument.
// Second return value is true if containers exist in the bitmap.
func (b *Bitmap) MinAt(start uint64) (uint64, bool) {
v, eof := b.IteratorAt(start).Next()
return v, !eof
}
// Max returns the highest value in the bitmap.
// Returns zero if the bitmap is empty.
func (b *Bitmap) Max() uint64 {
@ -1718,6 +1726,12 @@ func (b *Bitmap) Iterator() *Iterator {
return itr
}
func (b *Bitmap) IteratorAt(start uint64) *Iterator {
itr := &Iterator{bitmap: b}
itr.Seek(start)
return itr
}
// Ops returns the number of write ops the bitmap is aware of in its ops
// log, and their total bit count.
func (b *Bitmap) Ops() (ops int, opN int) {
@ -1730,6 +1744,167 @@ func (b *Bitmap) SetOps(ops int, opN int) {
b.ops, b.opN = ops, opN
}
// RoaringToBitmaps yields a series of bitmaps with specified shard
// keys, based on a single roaring file, with splits at multiples of
// shardWidth, which should be a multiple of container size.
func RoaringToBitmaps(data []byte, shardWidth uint64) ([]*Bitmap, []uint64) {
if data == nil {
return nil, nil
}
var itr roaringIterator
var itrKey uint64
var itrCType byte
var itrN int
var itrLen int
var itrPointer *uint16
var itrErr error
currentShard := ^uint64(0)
var currentBitmap *Bitmap
var bitmaps []*Bitmap
var shards []uint64
keysPerShard := shardWidth >> 16
itr, err := newRoaringIterator(data)
if err != nil || itr == nil {
return nil, nil
}
itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next()
for itrErr == nil {
newC := &Container{
typeID: itrCType,
n: int32(itrN),
len: int32(itrLen),
cap: int32(itrLen),
pointer: itrPointer,
flags: flagMapped,
}
shard := itrKey / keysPerShard
if shard != currentShard {
if currentBitmap != nil {
bitmaps = append(bitmaps, currentBitmap)
shards = append(shards, currentShard)
}
currentBitmap = NewFileBitmap()
currentShard = shard
}
currentBitmap.Containers.Put(itrKey, newC)
itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next()
}
if currentBitmap != nil {
bitmaps = append(bitmaps, currentBitmap)
shards = append(shards, currentShard)
}
// we don't support ops logs for this
return bitmaps, shards
}
// BitmapsToRoaring renders a series of non-overlapping bitmaps as a
// unified roaring file.
func BitmapsToRoaring(bitmaps []*Bitmap) []byte {
count := int64(0)
size := int64(0)
for i, bm := range bitmaps {
c, s := bm.roaringSize()
// skip this bitmap during the next pass, since it's empty
if c == 0 {
bitmaps[i] = nil
continue
}
count += c
size += s
}
if count == 0 {
return nil
}
// we have count headers, which need 12 bytes, plus a magic number,
// plus offsets (4 bytes per container), plus size bytes of data to
// write.
out := make([]byte, headerBaseSize+(12*count)+(4*count)+size)
binary.LittleEndian.PutUint16(out[0:2], uint16(MagicNumber))
out[3] = byte(storageVersion)
binary.LittleEndian.PutUint32(out[4:8], uint32(count))
headerEnd := 8 + (12 * count)
offsetEnd := headerEnd + (4 * count)
headers := out[8:headerEnd]
offsets := out[headerEnd:offsetEnd]
data := out[offsetEnd:]
headerOffset := 0
offsetOffset := 0
dataOffset := 0
prevKey := uint64(0)
for _, bm := range bitmaps {
if bm == nil {
continue
}
citer, _ := bm.Containers.Iterator(0)
for citer.Next() {
k, c := citer.Value()
n := c.N()
if n == 0 {
continue
}
if roaringParanoia {
if k < prevKey {
panic("unsorted keys in multiple-bitmap roaring conversion")
}
}
// place header at header offset, and data at data
// offset
header := headers[headerOffset : headerOffset+12]
offset := offsets[offsetOffset : offsetOffset+4]
headerOffset += 12
offsetOffset += 4
binary.LittleEndian.PutUint64(header[0:8], k)
binary.LittleEndian.PutUint16(header[8:10], uint16(c.typeID))
binary.LittleEndian.PutUint16(header[10:12], uint16(n-1))
binary.LittleEndian.PutUint32(offset[0:4], uint32(dataOffset+int(offsetEnd)))
nextData := data[dataOffset:]
switch c.typeID {
case containerArray:
asUint16 := *(*[]uint16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&nextData[0])), Len: int(c.len), Cap: int(c.len)}))
copy(asUint16, c.array())
dataOffset += 2 * int(c.len)
case containerBitmap:
asUint64 := *(*[]uint64)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&nextData[0])), Len: 1024, Cap: 1024}))
copy(asUint64, c.bitmap())
dataOffset += 8192
case containerRun:
asInterval16 := *(*[]interval16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&nextData[2])), Len: int(c.len), Cap: int(c.len)}))
copy(asInterval16, c.runs())
binary.LittleEndian.PutUint16(nextData[0:2], uint16(c.len))
dataOffset += int(4*c.len) + 2
}
}
}
return out
}
// roaringSize yields the count of non-empty containers, and the size
// of the storage *only* -- not the headers.
func (b *Bitmap) roaringSize() (int64, int64) {
count := int64(0)
size := int64(0)
citer, _ := b.Containers.Iterator(0)
for citer.Next() {
_, c := citer.Value()
if c.N() == 0 {
continue
}
count++
switch c.typeID {
case containerArray:
size += 2 * int64(c.N())
case containerBitmap:
size += 8192
case containerRun:
// 2 bytes for the count of runs, plus 4 bytes per run
size += 2 + (4 * int64(c.len))
}
}
return count, size
}
// Info returns stats for the bitmap.
func (b *Bitmap) Info() bitmapInfo {
info := bitmapInfo{

168
row.go
View file

@ -18,6 +18,7 @@ import (
"encoding/json"
"sort"
"github.com/pilosa/pilosa/v2/ext"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)
@ -43,6 +44,55 @@ func NewRow(columns ...uint64) *Row {
return r
}
// NewRowFromBitmap divides a bitmap into rows, which it now calls shards. This
// transposes; data that was in any shard for Row 0 is now considered shard 0,
// etcetera.
func NewRowFromBitmap(b *roaring.Bitmap) *Row {
r := &Row{}
if b == nil {
return r
}
rowNum := uint64(0)
for col, ok := b.MinAt(rowNum * ShardWidth); ok; col, ok = b.MinAt(rowNum * ShardWidth) {
rowNum = col / ShardWidth
seg := rowSegment{
shard: rowNum,
data: b.OffsetRange(rowNum*ShardWidth, rowNum*ShardWidth, (rowNum+1)*ShardWidth),
writable: true,
}
seg.n = seg.data.Count()
r.segments = append(r.segments, seg)
rowNum++
}
return r
}
// NewRowFromRoaring parses a roaring data file as a row, dividing it into
// bitmaps and rowSegments based on shard width.
func NewRowFromRoaring(data []byte) *Row {
bitmaps, shards := roaring.RoaringToBitmaps(data, ShardWidth)
r := &Row{segments: make([]rowSegment, len(bitmaps))}
for i := range bitmaps {
segment := rowSegment{
shard: shards[i],
data: bitmaps[i],
writable: false,
n: bitmaps[i].Count(),
}
r.segments[i] = segment
}
return r
}
// Roaring returns the row treated as a unified roaring bitmap.
func (r *Row) Roaring() []byte {
bitmaps := make([]*roaring.Bitmap, len(r.segments))
for i := range r.segments {
bitmaps[i] = r.segments[i].data
}
return roaring.BitmapsToRoaring(bitmaps)
}
// IsEmpty returns true if the row doesn't contain any set bits.
func (r *Row) IsEmpty() bool {
if len(r.segments) == 0 {
@ -194,6 +244,65 @@ func (r *Row) Union(others ...*Row) *Row {
return &Row{segments: output}
}
// GenericBinaryOp returns the output of a generic op on r and other.
func (r *Row) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *Row, args map[string]interface{}) *Row {
var segments []rowSegment
itr := newMergeSegmentIterator(r.segments, other.segments)
for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
if s1 == nil {
segments = append(segments, *s0)
continue
} else if s0 == nil {
segments = append(segments, *s1)
continue
}
segments = append(segments, *s0.GenericBinaryOp(op, s1, args))
}
return &Row{segments: segments}
}
// GenericNaryOp returns the output of an nary op on r and others.
func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args map[string]interface{}) *Row {
segments := make([][]rowSegment, 0, len(others)+1)
if len(r.segments) > 0 {
segments = append(segments, r.segments)
}
nextSegs := make([][]rowSegment, 0, len(others)+1)
toProcess := make([]*rowSegment, 0, len(others)+1)
var output []rowSegment
for _, other := range others {
if len(other.segments) > 0 {
segments = append(segments, other.segments)
}
}
for len(segments) > 0 {
shard := segments[0][0].shard
for _, segs := range segments {
if segs[0].shard < shard {
shard = segs[0].shard
}
}
nextSegs = nextSegs[:0]
toProcess := toProcess[:0]
for _, segs := range segments {
if segs[0].shard == shard {
toProcess = append(toProcess, &segs[0])
segs = segs[1:]
}
if len(segs) > 0 {
nextSegs = append(nextSegs, segs)
}
}
// at this point, "toProcess" is a list of all the segments
// sharing the lowest ID, and nextSegs is a list of all the others.
// Swap the segment lists (so we don't have to reallocate it)
segments, nextSegs = nextSegs, segments
output = append(output, *toProcess[0].GenericNaryOp(op, toProcess[1:], args))
}
return &Row{segments: output}
}
// Difference returns the diff of r and other.
func (r *Row) Difference(other *Row) *Row {
var segments []rowSegment
@ -212,6 +321,17 @@ func (r *Row) Difference(other *Row) *Row {
return &Row{segments: segments}
}
// GenericUnary returns the results of a generic op on r.
func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row {
work := r
var segments []rowSegment
for _, segment := range work.segments {
opped := segment.GenericUnaryOp(op, args)
segments = append(segments, *opped)
}
return &Row{segments: segments}
}
// Shift returns the bitwise shift of r by n bits.
// Currently only positive shift values are supported.
func (r *Row) Shift(n int64) (*Row, error) {
@ -299,6 +419,15 @@ func (r *Row) Count() uint64 {
return n
}
// GenericCount applies an op to lots of things.
func (r *Row) GenericCount(op ext.BitmapOpUnaryCount, args map[string]interface{}) uint64 {
var n int64
for i := range r.segments {
n += op([]ext.Bitmap{WrapBitmap(r.segments[i].data)}, args)
}
return uint64(n)
}
// MarshalJSON returns a JSON-encoded byte slice of r.
func (r *Row) MarshalJSON() ([]byte, error) {
var o struct {
@ -419,6 +548,33 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment {
}
}
// GenericOp performs a generic op on s and other
func (s *rowSegment) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *rowSegment, args map[string]interface{}) *rowSegment {
data := op([]ext.Bitmap{WrapBitmap(s.data), WrapBitmap(other.data)}, args)
return &rowSegment{
data: UnwrapBitmap(data),
shard: s.shard,
n: data.Count(),
}
}
// GenericOp performs a generic op on s and others
func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSegment, args map[string]interface{}) *rowSegment {
bitmaps := make([]ext.Bitmap, len(others)+1)
bitmaps[0] = WrapBitmap(s.data)
for i, seg := range others {
bitmaps[i+1] = WrapBitmap(seg.data)
}
data := op(bitmaps, args)
return &rowSegment{
data: UnwrapBitmap(data),
shard: s.shard,
n: data.Count(),
}
}
// Difference returns the diff of s and other.
func (s *rowSegment) Difference(other *rowSegment) *rowSegment {
data := s.data.Difference(other.data)
@ -462,6 +618,18 @@ func (s *rowSegment) Shift() (*rowSegment, error) {
}, nil
}
// GenericUnary returns s subject to op.
func (s *rowSegment) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *rowSegment {
//TODO deal with overflow
data := UnwrapBitmap(op([]ext.Bitmap{WrapBitmap(s.data)}, args))
return &rowSegment{
data: data,
shard: s.shard,
n: data.Count(),
}
}
// SetBit sets the i-th column of the row.
func (s *rowSegment) SetBit(i uint64) (changed bool) {
s.ensureWritable()

120
server.go
View file

@ -17,17 +17,21 @@ package pilosa
import (
"context"
"fmt"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"plugin"
"runtime"
"strconv"
"strings"
"sync"
"time"
"github.com/pilosa/pilosa/v2/ext"
"github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
"github.com/pkg/errors"
@ -57,6 +61,8 @@ type Server struct { // nolint: maligned
hosts []string
clusterDisabled bool
serializer Serializer
extensionPath string
extensions []*ext.ExtensionInfo
// External
systemInfo SystemInfo
@ -335,6 +341,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
if err != nil {
return nil, err
}
s.extensionPath = filepath.Join(path, ".extensions")
s.holder.Path = path
// s.holder.translateFile.Path = filepath.Join(path, ".keys")
@ -376,6 +383,30 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.cluster.broadcaster = s
s.cluster.maxWritesPerRequest = s.maxWritesPerRequest
s.holder.broadcaster = s
err = s.loadPlugins()
if err != nil {
s.logger.Printf("not all plugins loaded successfully")
}
if len(s.extensions) > 0 {
s.logger.Printf("loaded extensions:")
for _, ext := range s.extensions {
if ext == nil {
s.logger.Printf(" inexplicably, a nil extension?!?")
continue
}
s.logger.Printf(" %s %s: %s", ext.Name, ext.Version, ext.Description)
if ext.License != "" {
s.logger.Printf(" License: %s", ext.License)
}
if len(ext.BitmapOps) > 0 {
opList := make([]string, len(ext.BitmapOps))
for i := range ext.BitmapOps {
opList[i] = ext.BitmapOps[i].Name
}
s.logger.Printf(" Ops: %s", strings.Join(opList, ", "))
}
}
}
err = s.cluster.setup()
if err != nil {
@ -389,6 +420,95 @@ func (s *Server) InternalClient() InternalClient {
return s.defaultClient
}
func (s *Server) loadPlugins() error {
var anyError error
dir, err := os.Open(s.extensionPath)
if err != nil {
// don't complain about it not existing, that's fine.
if os.IsNotExist(err) {
s.logger.Printf("extension interface v0: no extensions directory.")
return nil
}
return errors.Wrap(err, "opening extension path:")
}
defer dir.Close()
for files, err := dir.Readdir(64); err != io.EOF; files, err = dir.Readdir(64) {
if err != nil {
return errors.Wrap(err, "searching extension directory:")
}
for _, file := range files {
name := file.Name()
// only .so files are likely plugins.
if !strings.HasSuffix(name, ".so") {
continue
}
// only regular files are candidates for loading.
mode := file.Mode()
if !mode.IsRegular() {
s.logger.Printf("extension file '%s' is not a regular file", name)
continue
}
err = s.loadPlugin(name)
if err != nil {
s.logger.Printf("loading extension %s: %v", name, err)
anyError = err
}
}
}
return anyError
}
func (s *Server) loadPlugin(name string) error {
path := filepath.Join(s.extensionPath, name)
p, err := plugin.Open(path)
if err != nil {
return err
}
pluginExtInfo, err := p.Lookup("ExtensionInfo")
if err != nil {
return fmt.Errorf("%s: no ExtensionInfo found", name)
}
extInfoFunc, ok := pluginExtInfo.(func(string) (*ext.ExtensionInfo, error))
if !ok {
return fmt.Errorf("%s: unexpected %T instead of ExtensionInfo object", name, pluginExtInfo)
}
extInfo, err := extInfoFunc("v0")
if err != nil {
return errors.Wrap(err, name)
}
if extInfo == nil {
return fmt.Errorf("%s: nil ExtensionInfo", name)
}
if extInfo.ExtensionAPI != "v0" {
return fmt.Errorf("%s: unsupported extension API %s", name, extInfo.ExtensionAPI)
}
s.extensions = append(s.extensions, extInfo)
bitmapOps := extInfo.BitmapOps
bmOps, countOps, fieldOps, unknownOps := 0, 0, 0, 0
for i := range bitmapOps {
// title-case the name
bitmapOps[i].Name = strings.Title(bitmapOps[i].Name)
typ := bitmapOps[i].Func.BitmapOpType()
switch {
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:
countOps++
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap:
bmOps++
case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap:
fieldOps++
default:
unknownOps++
}
}
err = s.executor.registerOps(bitmapOps)
if err != nil {
s.logger.Printf("warning: extension registration failed: %v", err)
} else {
pql.RegisterPluginFuncs(bitmapOps)
}
return nil
}
// UpAndDown brings the server up minimally and shuts it down
// again; basically, it exists for testing holder open and close.
func (s *Server) UpAndDown() error {