scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 09:05:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'enterprise' into grpc-logger

Browse source
This commit is contained in:
Travis Turner 2019-11-13 08:31:26 -06:00 • committed by GitHub
commit afd1c004f9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
44 changed files with 6244 additions and 3714 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -141,7 +141,7 @@ docker-test:
# Run golangci-lint
golangci-lint: require-golangci-lint
golangci-lint run
golangci-lint run --skip-files '.*\.peg\.go'
# Run gometalinter with custom flags
gometalinter: require-gometalinter vendor

1
api.go
View file

@ -152,6 +152,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
ExcludeRowAttrs: req.ExcludeRowAttrs, // NOTE: Kept for Pilosa 1.x compat.
ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat.
ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat.
EmbeddedData: req.EmbeddedData, // precomputed values that needed to be passed with the request
}
resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
if err != nil {

1
cluster_internal_test.go
View file

@ -158,6 +158,7 @@ func TestFragSources(t *testing.T) {
c5.addNodeBasicSorted(node3)
idx := newIndexWithTempPath("i")
defer idx.Close()
field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
if err != nil {
t.Fatal(err)

65
encoding/proto/proto.go
View file

@ -396,14 +396,19 @@ func encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) *internal.Import
}
func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest {
return &internal.QueryRequest{
r := &internal.QueryRequest{
Query: m.Query,
Shards: m.Shards,
ColumnAttrs: m.ColumnAttrs,
Remote: m.Remote,
ExcludeRowAttrs: m.ExcludeRowAttrs,
ExcludeColumns: m.ExcludeColumns,
EmbeddedData: make([]*internal.Row, len(m.EmbeddedData)),
}
for i := range m.EmbeddedData {
r.EmbeddedData[i] = encodeRow(m.EmbeddedData[i])
}
return r
}
func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
@ -416,6 +421,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
pb.Results[i] = &internal.QueryResult{}
switch result := m.Results[i].(type) {
case pilosa.SignedRow:
pb.Results[i].Type = queryResultTypeSignedRow
pb.Results[i].SignedRow = encodeSignedRow(result)
case *pilosa.Row:
pb.Results[i].Type = queryResultTypeRow
pb.Results[i].Row = encodeRow(result)
@ -446,7 +454,7 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
case nil:
pb.Results[i].Type = queryResultTypeNil
default:
panic(fmt.Errorf("unknown type: %d", pb.Results[i].Type))
panic(fmt.Errorf("unknown type: %T", m.Results[i]))
}
}
@ -966,6 +974,10 @@ func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) {
m.Remote = pb.Remote
m.ExcludeRowAttrs = pb.ExcludeRowAttrs
m.ExcludeColumns = pb.ExcludeColumns
m.EmbeddedData = make([]*pilosa.Row, len(pb.EmbeddedData))
for i := range pb.EmbeddedData {
m.EmbeddedData[i] = decodeRow(pb.EmbeddedData[i])
}
}
func decodeImportRequest(pb *internal.ImportRequest, m *pilosa.ImportRequest) {
@ -1068,10 +1080,13 @@ const (
queryResultTypeGroupCounts
queryResultTypeRowIdentifiers
queryResultTypePair
queryResultTypeSignedRow
)
func decodeQueryResult(pb *internal.QueryResult) interface{} {
switch pb.Type {
case queryResultTypeSignedRow:
return decodeSignedRow(pb.SignedRow)
case queryResultTypeRow:
return decodeRow(pb.Row)
case queryResultTypePairs:
@ -1099,14 +1114,31 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} {
// DecodeRow converts r from its internal representation.
func decodeRow(pr *internal.Row) *pilosa.Row {
if pr == nil {
return nil
return pilosa.NewRow()
}
r := pilosa.NewRow()
var r *pilosa.Row
if len(pr.Roaring) > 0 {
r = pilosa.NewRowFromRoaring(pr.Roaring)
} else {
r = pilosa.NewRow()
for _, v := range pr.Columns {
r.SetBit(v)
}
}
r.Attrs = decodeAttrs(pr.Attrs)
r.Keys = pr.Keys
for _, v := range pr.Columns {
r.SetBit(v)
return r
}
func decodeSignedRow(pr *internal.SignedRow) pilosa.SignedRow {
if pr == nil {
return pilosa.SignedRow{}
}
r := pilosa.SignedRow{
Pos: decodeRow(pr.Pos),
Neg: decodeRow(pr.Neg),
}
return r
}
@ -1213,16 +1245,29 @@ func encodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet {
}
}
func encodeSignedRow(r pilosa.SignedRow) *internal.SignedRow {
ir := &internal.SignedRow{
Pos: encodeRow(r.Pos),
Neg: encodeRow(r.Neg),
}
return ir
}
func encodeRow(r *pilosa.Row) *internal.Row {
if r == nil {
return nil
}
return &internal.Row{
Columns: r.Columns(),
Keys: r.Keys,
Attrs: encodeAttrs(r.Attrs),
ir := &internal.Row{
Keys: r.Keys,
Attrs: encodeAttrs(r.Attrs),
}
if false {
ir.Columns = r.Columns()
} else {
ir.Roaring = r.Roaring()
}
return ir
}
func encodeRowIdentifiers(r pilosa.RowIdentifiers) *internal.RowIdentifiers {

475
executor.go
View file

@ -22,7 +22,9 @@ import (
"sync"
"time"
"github.com/pilosa/pilosa/v2/ext"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/shardwidth"
"github.com/pilosa/pilosa/v2/tracing"
"github.com/pkg/errors"
@ -57,6 +59,12 @@ type executor struct {
workersWG sync.WaitGroup
workerPoolSize int
work chan job
// global registry to check for name clashes
additionalOps map[string]*ext.BitmapOp
// typed registries we can use in lookups
additionalBitmapOps map[string]ext.BitmapOpBitmap
additionalCountOps map[string]ext.BitmapOpUnaryCount
additionalFieldOps map[string]ext.BitmapOpBSIBitmap
}
// executorOption is a functional option type for pilosa.Executor
@ -109,6 +117,38 @@ func (e *executor) Close() error {
return nil
}
func (e *executor) registerOps(ops []ext.BitmapOp) error {
if e.additionalOps == nil {
e.additionalOps = make(map[string]*ext.BitmapOp)
e.additionalBitmapOps = make(map[string]ext.BitmapOpBitmap)
e.additionalCountOps = make(map[string]ext.BitmapOpUnaryCount)
e.additionalFieldOps = make(map[string]ext.BitmapOpBSIBitmap)
}
for i, op := range ops {
name := op.Name
if _, exists := e.additionalOps[name]; exists {
return fmt.Errorf("op name '%s' already defined", name)
}
e.additionalOps[name] = &ops[i]
typ := ops[i].Func.BitmapOpType()
switch {
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:
e.additionalCountOps[name] = ops[i].Func.(ext.BitmapOpUnaryCount)
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap:
e.additionalBitmapOps[name] = ops[i].Func.(ext.BitmapOpBitmap)
case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap:
if fn, ok := ops[i].Func.(ext.BitmapOpBSIBitmapPrecall); ok {
e.additionalFieldOps[name] = ext.BitmapOpBSIBitmap(fn)
} else {
e.additionalFieldOps[name] = ops[i].Func.(ext.BitmapOpBSIBitmap)
}
default:
return fmt.Errorf("unsupported types for '%s': input type %d, output type %d", name, typ.Input, typ.Output)
}
}
return nil
}
// Execute executes a PQL query.
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute")
@ -237,6 +277,97 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr
return ax, nil
}
// handlePreCalls traverses the call tree looking for calls that need
// precomputed values. Right now, that's just Distinct.
func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
if c.Name == "Precomputed" {
idx := c.Args["valueidx"].(int64)
if idx >= 0 && idx < int64(len(opt.EmbeddedData)) {
row := opt.EmbeddedData[idx]
c.Precomputed = make(map[uint64]interface{}, len(row.segments))
for _, segment := range row.segments {
c.Precomputed[segment.shard] = &Row{segments: []rowSegment{segment}}
}
} else {
return fmt.Errorf("no precomputed data! index %d, len %d", idx, len(opt.EmbeddedData))
}
return nil
}
newIndex := c.CallIndex()
// A cross-index query is handled by precall. This is inefficient,
// but we have to do it for now because shards might be different and
// we haven't implemented the local precalls that would be enough
// in some cases.
//
// This makes simple cross-index queries noticably inefficient.
//
// If you're here because of that: We should be using PrecallLocal
// in cases where the call isn't already PrecallGlobal, and
// PrecallLocal should wait until we're running on a specific node
// to do the farming-out of just the sub-queries it has to run
// for its local shards.
//
// As is, we have one node querying every node, then sending out
// all the data to every node, including the data that node already
// has. We could reduce the actual copying around dramatically,
// but only in the cases where local is good enough -- not something
// like Distinct, where you can't predict output shard for a result
// from the shard being queried.
if newIndex != "" && newIndex != index {
c.Type = pql.PrecallGlobal
index = newIndex
}
if c.Type == pql.PrecallNone {
// otherwise, handle the children
return e.handlePreCallChildren(ctx, index, c, shards, opt)
}
// We don't try to handle sub-calls from here. I'm not 100%
// sure that's right, but I think the fact that they're happening
// inside a precomputed call may mean they need different
// handling. In any event, the sub-calls will get handled by
// the executeCall when it gets to them...
// We set c to look like a normal call, and actually execute it:
c.Type = pql.PrecallNone
// possibly override call index.
v, err := e.executeCall(ctx, index, c, shards, opt)
if err != nil {
return err
}
var row *Row
switch r := v.(type) {
case *Row:
row = r
case SignedRow:
row = r.Pos
default:
return fmt.Errorf("precomputed call %s returned unexpected non-Row data: %T", c.Name, v)
}
c.Children = []*pql.Call{}
c.Name = "Precomputed"
c.Args = map[string]interface{}{"valueidx": len(opt.EmbeddedData)}
// stash a copy of the full results, which can be forwarded to other
// shards if the query has to go to them
opt.EmbeddedData = append(opt.EmbeddedData, row)
// and stash a copy locally, so local calls can use it
c.Precomputed = make(map[uint64]interface{}, len(row.segments))
for _, segment := range row.segments {
c.Precomputed[segment.shard] = &Row{segments: []rowSegment{segment}}
}
return nil
}
// handlePreCallChildren handles any pre-calls in the children of a given call.
func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
for i := range c.Children {
if err := e.handlePreCalls(ctx, index, c.Children[i], shards, opt); err != nil {
return err
}
}
return nil
}
func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.execute")
defer span.Finish()
@ -270,7 +401,27 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar
return nil, err
}
v, err := e.executeCall(ctx, index, call, shards, opt)
// If you actually make a top-level Distinct call, you
// want a SignedRow back. Otherwise, it's something else
// that will be using it as a row, and we only care
// about the positive values, because only positive values
// are valid column IDs. So we don't actually eat top-level
// pre calls.
err := e.handlePreCallChildren(ctx, index, call, shards, opt)
if err != nil {
return nil, err
}
var v interface{}
// Top-level calls don't need to precompute cross-index things,
// because we can just pick whatever index we want, but we
// still need to handle them. Since everything else was
// already precomputed by handlePreCallChildren, though,
// we don't need this logic in executeCall.
if newIndex := call.CallIndex(); newIndex != "" {
v, err = e.executeCall(ctx, newIndex, call, shards, opt)
} else {
v, err = e.executeCall(ctx, index, call, shards, opt)
}
if err != nil {
return nil, err
}
@ -299,6 +450,14 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
}
// Special handling for mutation and top-n calls.
if op, ok := e.additionalCountOps[c.Name]; ok {
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeGenericCount(ctx, index, c, op, shards, opt)
}
if op, ok := e.additionalFieldOps[c.Name]; ok {
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeGenericField(ctx, index, c, op, shards, opt)
}
switch c.Name {
case "Sum":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
@ -343,6 +502,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
return e.executeOptionsCall(ctx, index, c, shards, opt)
case "IncludesColumn":
return e.executeIncludesColumnCall(ctx, index, c, shards, opt)
case "Precomputed":
return e.executePrecomputedCall(ctx, index, c, shards, opt)
default:
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeBitmapCall(ctx, index, c, shards, opt)
@ -501,6 +662,37 @@ func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, sh
return other, nil
}
// executeGenericField executes a generic call on a field. Note that in this
// implementation, the operation is always a BSI op.
func (e *executor) executeGenericField(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shards []uint64, opt *execOptions) (SignedRow, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericField")
span.LogKV("name", c.Name)
defer span.Finish()
if field := c.Args["field"]; field == "" {
return SignedRow{}, fmt.Errorf("plugin operation %s(): field required", c.Name)
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
return e.executeGenericFieldShard(ctx, index, c, op, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(SignedRow)
return other.union(v.(SignedRow))
}
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return SignedRow{}, err
}
other, _ := result.(SignedRow)
return other, nil
}
// executeMin executes a Min() call.
func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMin")
@ -641,6 +833,41 @@ func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call,
return e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
}
// executePrecomputedCall pretends to execute a call that we have a precomputed value for.
func (e *executor) executePrecomputedCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executePrecomputedCall")
defer span.Finish()
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
if c.Precomputed != nil {
return c.Precomputed[shard], nil
}
// This might not be an error -- if there were no values, we will not have created
// the corresponding row.
return NewRow(), nil
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(*Row)
if other == nil {
other = NewRow()
}
other.Merge(v.(*Row))
return other
}
other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return nil, errors.Wrap(err, "map reduce")
}
row, _ := other.(*Row)
return row, nil
}
// executeBitmapCall executes a call that returns a bitmap.
func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCall")
@ -719,6 +946,13 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard")
defer span.Finish()
if _, ok := e.additionalCountOps[c.Name]; ok {
return nil, fmt.Errorf("count op %s used as bitmap call", c.Name)
}
if op, ok := e.additionalBitmapOps[c.Name]; ok {
return e.executeGenericBitmapShard(ctx, index, c, op, shard)
}
switch c.Name {
case "Row", "Range":
return e.executeRowShard(ctx, index, c, shard)
@ -734,11 +968,63 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *
return e.executeNotShard(ctx, index, c, shard)
case "Shift":
return e.executeShiftShard(ctx, index, c, shard)
case "Precomputed":
return e.executePrecomputedCallShard(ctx, index, c, shard)
default:
return nil, fmt.Errorf("unknown call: %s", c.Name)
}
}
// executeGenericFieldShard executes a generic/extension command on a
// single shard. Note that in this implementation, the op is always
// a BSI op.
func (e *executor) executeGenericFieldShard(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shard uint64) (SignedRow, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericShard")
defer span.Finish()
var filter *Row
var filterBitmap *roaring.Bitmap
if len(c.Children) == 1 {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return SignedRow{}, errors.Wrap(err, "executing bitmap call")
}
filter = row
if filter != nil && len(filter.segments) > 0 {
filterBitmap = filter.segments[0].data
}
}
fieldName, _ := c.Args["field"].(string)
field := e.Holder.Field(index, fieldName)
if field == nil {
return SignedRow{}, nil
}
bsig := field.bsiGroup(fieldName)
if bsig == nil {
return SignedRow{}, nil
}
fragment := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
if fragment == nil {
return SignedRow{}, nil
}
var out ext.SignedBitmap
if filterBitmap != nil {
out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{WrapBitmap(filterBitmap)}, c.Args)
} else {
out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{}, c.Args)
}
return SignedRow{
Neg: NewRowFromBitmap(UnwrapBitmap(out.Neg)),
Pos: NewRowFromBitmap(UnwrapBitmap(out.Pos)),
}, nil
}
// executeSumCountShard calculates the sum and count for bsiGroups on a shard.
func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSumCountShard")
@ -1749,6 +2035,44 @@ func (e *executor) executeIntersectShard(ctx context.Context, index string, c *p
return other, nil
}
// executeGenericBitmapShard executes a generic bitmap call for a local shard.
func (e *executor) executeGenericBitmapShard(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBitmap, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericBitmapShard")
defer span.Finish()
if op.BitmapOpArity() == ext.OpArityUnary {
if len(c.Children) != 1 {
return nil, fmt.Errorf("%s needs exactly one row parameter", c.Name)
}
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return nil, err
}
return row.GenericUnaryOp(op.BitmapOpFunc(), c.Args), nil
}
var err error
rows := make([]*Row, len(c.Children))
for i, input := range c.Children {
rows[i], err = e.executeBitmapCallShard(ctx, index, input, shard)
if err != nil {
return nil, err
}
}
var other *Row
switch op.BitmapOpArity() {
case ext.OpArityBinary:
other = rows[0]
for _, row := range rows[1:] {
other = other.GenericBinaryOp(op.BitmapOpFunc(), row, c.Args)
}
case ext.OpArityNary:
other = rows[0].GenericNaryOp(op.BitmapOpFunc(), rows[1:], c.Args)
}
other.invalidateCount()
return other, nil
}
// executeUnionShard executes a union() call for a local shard.
func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard")
@ -1793,6 +2117,25 @@ func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Cal
return other, nil
}
// executePrecomputedCallShard pretends to execute a precomputed call for a local shard.
func (e *executor) executePrecomputedCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
if c.Precomputed != nil {
v := c.Precomputed[shard]
if v == nil {
return NewRow(), nil
}
if r, ok := v.(*Row); ok {
if r != nil {
return r, nil
} else {
return NewRow(), nil
}
}
return nil, fmt.Errorf("precomputed value is not a row: %T", v)
}
return nil, fmt.Errorf("per-shard: missing precomputed values for shard %d", shard)
}
// executeNotShard executes a not() call for a local shard.
func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeNotShard")
@ -1849,6 +2192,41 @@ func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.C
return row.Shift(n)
}
// executeGeneric executes a provided count-like call.
func (e *executor) executeGenericCount(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpUnaryCount, shards []uint64, opt *execOptions) (uint64, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericCount")
defer span.Finish()
if len(c.Children) == 0 {
return 0, fmt.Errorf("%s() requires an input bitmap", c.Name)
} else if len(c.Children) > 1 {
return 0, fmt.Errorf("%s() only accepts a single bitmap input", c.Name)
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return 0, err
}
return row.GenericCount(op, c.Args), nil
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(uint64)
return other + v.(uint64)
}
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return 0, err
}
n, _ := result.(uint64)
return n, nil
}
// executeCount executes a count() call.
func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount")
@ -1946,7 +2324,7 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq
}
// Forward call to remote node otherwise.
res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
if err != nil {
return false, err
}
@ -2222,7 +2600,7 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.
}
// Forward call to remote node otherwise.
res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
if err != nil {
return false, err
}
@ -2257,7 +2635,7 @@ func (e *executor) executeSetValueField(ctx context.Context, index string, c *pq
}
// Forward call to remote node otherwise.
res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
if err != nil {
return false, err
}
@ -2311,7 +2689,7 @@ func (e *executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
resp <- err
}(node)
}
@ -2406,7 +2784,7 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil)
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil, nil)
resp <- err
}(node)
}
@ -2458,7 +2836,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
resp <- err
}(node)
}
@ -2474,15 +2852,16 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
}
// remoteExec executes a PQL query remotely for a set of shards on a node.
func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64) (results []interface{}, err error) { // nolint: interfacer
func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, embed []*Row) (results []interface{}, err error) { // nolint: interfacer
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeExec")
defer span.Finish()
// Encode request object.
pbreq := &QueryRequest{
Query: q.String(),
Shards: shards,
Remote: true,
Query: q.String(),
Shards: shards,
Remote: true,
EmbeddedData: embed,
}
pb, err := e.client.QueryNode(ctx, &node.URI, index, pbreq)
@ -2577,6 +2956,43 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
}
}
// makeEmbeddedDataForShards produces new rows containing the rowSegments
// that would correspond to a given set of shards.
func makeEmbeddedDataForShards(allRows []*Row, shards []uint64) []*Row {
if len(allRows) == 0 || len(shards) == 0 {
return nil
}
newRows := make([]*Row, len(allRows))
for i, row := range allRows {
if row == nil || len(row.segments) == 0 {
continue
}
segments := row.segments
segmentIndex := 0
newRows[i] = &Row{}
for _, shard := range shards {
for segmentIndex < len(segments) && segments[segmentIndex].shard < shard {
segmentIndex++
}
// no more segments in this row
if segmentIndex >= len(segments) {
break
}
if segments[segmentIndex].shard == shard {
newRows[i].segments = append(newRows[i].segments, segments[segmentIndex])
segmentIndex++
if segmentIndex >= len(segments) {
// no more segments, we're done
break
}
}
// if we got here, segments[segmentIndex].shard exists
// but is greater than the current shard, so we continue.
}
}
return newRows
}
func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapper")
defer span.Finish()
@ -2596,7 +3012,11 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
if n.ID == e.Node.ID {
resp.result, resp.err = e.mapperLocal(ctx, nodeShards, mapFn, reduceFn)
} else if !opt.Remote {
results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeShards)
var embeddedRowsForNode []*Row
if opt.EmbeddedData != nil {
embeddedRowsForNode = makeEmbeddedDataForShards(opt.EmbeddedData, nodeShards)
}
results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeShards, embeddedRowsForNode)
if len(results) > 0 {
resp.result = results[0]
}
@ -3026,6 +3446,7 @@ type execOptions struct {
ExcludeRowAttrs bool
ExcludeColumns bool
ColumnAttrs bool
EmbeddedData []*Row
}
// hasOnlySetRowAttrs returns true if calls only contains SetRowAttrs() calls.
@ -3060,6 +3481,36 @@ func needsShards(calls []*pql.Call) bool {
return false
}
// SignedRow represents a signed *Row with two (neg/pos) *Rows.
type SignedRow struct {
Neg *Row `json:"neg"`
Pos *Row `json:"pos"`
}
func (sr *SignedRow) union(other SignedRow) SignedRow {
ret := SignedRow{&Row{}, &Row{}}
// merge in sr
if sr != nil {
if sr.Neg != nil {
ret.Neg = ret.Neg.Union(sr.Neg)
}
if sr.Pos != nil {
ret.Pos = ret.Pos.Union(sr.Pos)
}
}
// merge in other
if other.Neg != nil {
ret.Neg = ret.Neg.Union(other.Neg)
}
if other.Pos != nil {
ret.Pos = ret.Pos.Union(other.Pos)
}
return ret
}
// ValCount represents a grouping of sum & count for Sum() and Average() calls.
type ValCount struct {
Val int64 `json:"value"`

238
ext/ext.go Normal file
View file

@ -0,0 +1,238 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package ext provides an EXPERIMENTAL AND TEMPORARY interface to use for
// plugin extensions to Pilosa. DO NOT DEVELOP NEW PLUGINS WITH THIS. The
// replacement design is already in process, but it needs more refinement
// to address issues. This one has those issues, and more.
//
// In the current design, plugins will be loaded at runtime using the
// go `plugin` package, so they should be built as a main package using
// the plugin build mode.
//
// Plugins should not import other packages from Pilosa.
//
// To advertise their functionality, plugins define one or more of a
// handful of symbols which will be checked for at plugin load and used
// to register their functionality.
//
// The plugin interface will check for the following function(s). If the
// functions exist, they must have the given signatures. If they return
// a non-nil error, no ops are registered, and the error message will
// be reported in the Pilosa server's logs.
//
// BitmapOps() ([]BitmapOp, error)
//
// These functions may be absent, and may return nil slices; in either
// case, no ops are registered.
package ext
// The Bitmap type represents a Pilosa bitmap, and is used for bitmap
// operations.
type Bitmap interface {
// AddN and RemoveN can be used to add or remove values from a bitmap.
AddN(a ...uint64) (int, error)
RemoveN(a ...uint64) (int, error)
// Lookups
Max() uint64
Min() (uint64, bool)
Count() uint64
Any() bool
Contains(uint64) bool
Slice() []uint64
SliceRange(uint64, uint64) []uint64
// ContainerBits stores the next 1<<16 bits, starting at the provided
// bit index. It may use a provided []uint64 to store them, or may
// provide its own. Don't write to those bits. Offset must be a multiple
// of 1<<16.
ContainerBits(uint64, []uint64) []uint64
// These operators provide existing implemented binary ops.
Intersect(Bitmap) Bitmap
Union(Bitmap) Bitmap
IntersectionCount(Bitmap) uint64
Difference(Bitmap) Bitmap
Xor(Bitmap) Bitmap
Shift(int) (Bitmap, error)
Flip(uint64, uint64) Bitmap
// New() is an atrocity: it creates a new bitmap, unrelated to the
// existing bitmap. This lets you create a new bitmap without having
// imported any of the packages that have bitmap creation tools, because
// the bitmap wrapper type has to give you one.
New() Bitmap
}
// SignedBitmap represents a bitmap that can contain both positive and negative
// values.
type SignedBitmap struct {
Pos, Neg Bitmap
}
// A BitmapOp represents a new bitmap operation that should be exposed
// in PQL.
type BitmapOpInput byte
type BitmapOpOutput byte
type BitmapOpArity byte
type BitmapOpPrecall byte
type BitmapOpType struct {
Input BitmapOpInput
Arity BitmapOpArity
Output BitmapOpOutput
Precall BitmapOpPrecall
}
const (
OpArityUnary = BitmapOpArity(iota)
OpArityBinary
OpArityNary
)
const (
// Unary: Exactly one bitmap.
OpInputBitmap = BitmapOpInput(iota)
// The really weird special case used for BSI, where we end up
// needing to do BSI computations. Arguments will be a
// single BitmapBSI, and a []Bitmap for other operands if any.
OpInputNaryBSI
)
const (
OpOutputCount = BitmapOpOutput(iota)
OpOutputBitmap
OpOutputSignedBitmap
)
const (
OpPrecallNone = BitmapOpPrecall(iota)
OpPrecallGlobal
OpPrecallLocal // unimplemented
)
// Regardless of arity, non-BSI functions should always take []Bitmap.
type BitmapOpFunc interface {
BitmapOpType() BitmapOpType
}
// BitmapOpBitmap should actually always be func([]Bitmap) Bitmap, but
// might be different kinds.
type BitmapOpBitmap interface {
BitmapOpArity() BitmapOpArity
BitmapOpFunc() GenericBitmapOpBitmap
}
// the common underlying type of the other BitmapOpBitmap functions
type GenericBitmapOpBitmap func([]Bitmap, map[string]interface{}) Bitmap
// BitmapBSI represents the way a single BSI field is passed into a function
// which takes a BSI field.
type BitmapBSI struct {
FieldData Bitmap
ShardWidth uint64
Offset int64
Depth uint
}
type BitmapOpBSIBitmap func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap
func (b BitmapOpBSIBitmap) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Output: OpOutputSignedBitmap}
}
type BitmapOpBSIBitmapPrecall func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap
func (b BitmapOpBSIBitmapPrecall) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Precall: OpPrecallGlobal, Output: OpOutputSignedBitmap}
}
type BitmapOpUnaryCount func([]Bitmap, map[string]interface{}) int64
func (b BitmapOpUnaryCount) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputCount}
}
type BitmapOpUnaryBitmap func([]Bitmap, map[string]interface{}) Bitmap
func (b BitmapOpUnaryBitmap) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputBitmap}
}
func (b BitmapOpUnaryBitmap) BitmapOpArity() BitmapOpArity {
return OpArityUnary
}
func (b BitmapOpUnaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap {
return GenericBitmapOpBitmap(b)
}
type BitmapOpBinaryBitmap func([]Bitmap, map[string]interface{}) Bitmap
func (b BitmapOpBinaryBitmap) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputBitmap, Arity: OpArityBinary, Output: OpOutputBitmap}
}
func (b BitmapOpBinaryBitmap) BitmapOpArity() BitmapOpArity {
return OpArityBinary
}
func (b BitmapOpBinaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap {
return GenericBitmapOpBitmap(b)
}
type BitmapOpNaryBitmap func([]Bitmap, map[string]interface{}) Bitmap
func (b BitmapOpNaryBitmap) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputBitmap, Arity: OpArityNary, Output: OpOutputBitmap}
}
func (b BitmapOpNaryBitmap) BitmapOpArity() BitmapOpArity {
return OpArityNary
}
func (b BitmapOpNaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap {
return GenericBitmapOpBitmap(b)
}
// BitmapOp represents an operation to be supported in PQL. Operations
// on bitmaps should always take []Bitmap. Operations on InputNaryBSI should
// take a []Bitmap, plus a Bitmap/shard-width/offset/depth.
//
// Reserved is a list of words to treat as reserved words in a prototype.
// This is not currently used but might be later, and I want to have the
// concept handy now.
type BitmapOp struct {
Name string
Func BitmapOpFunc
Reserved []string
}
// ExtensionInfo tells us about the extension. The ExtensionAPI string
// should be "v0". The version is a human-readable version, use something
// that seems meaningful. Name and Description are reasonably self-explanatory,
// I hope.
//
// Extensions should define a function:
// func ExtensionInfo(extensionAPI string) (*ExtensionInfo, error)
// which reports their extension info if they think they can coexist with that
// API string.
type ExtensionInfo struct {
Name string // Extension name.
Description string // Short description.
Version string // Human-readable version info for extension.
ExtensionAPI string // Extension API version. Should be v0 for now.
License string // License info.
BitmapOps []BitmapOp // List of provided ops.
}

1
ext/samples/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
*/*.so

125
ext/samples/some/some.go Normal file
View file

@ -0,0 +1,125 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"math/bits"
"github.com/molecula/apophenia"
"github.com/pilosa/pilosa/v2/ext"
)
// This could be dynamically generated, but for now it's not.
// nolint:unused,deadcode
var extInfoTemplate = &ext.ExtensionInfo{
Name: "some",
Description: "some of the bits/all of the bits/none of the bits",
Version: "0.01",
ExtensionAPI: "v0",
License: "unreleased",
BitmapOps: []ext.BitmapOp{
{Name: "Some", Func: ext.BitmapOpUnaryBitmap(Some), Reserved: []string{"p", "seed"}},
},
}
// ExtensionInfo is the entry point used by the plugin code.
func ExtensionInfo(api string) (*ext.ExtensionInfo, error) { // nolint:unused,deadcode
return extInfoTemplate, nil
}
const batchSize = 1024
// Some returns some of the bits from its first input bitmap. Takes seed (int)
// and p (float) values. Seed defaults to 0.
func Some(inputs []ext.Bitmap, args map[string]interface{}) ext.Bitmap {
if len(inputs) == 0 || inputs[0] == nil {
return nil
}
input := inputs[0]
min, ok := input.Min()
// no bits found?
if !ok {
return nil
}
// start at multiple of 128 not greater than min.
min &^= 127
max := input.Max()
p, ok := args["p"].(float64)
if !ok {
return nil
}
// no bits or impossible probability range
if p <= 0 || p > 1 {
return nil
}
// every bit
if p == 1 {
return inputs[0]
}
// On failure, we default to 0.
seed, _ := args["seed"].(int64)
densityScale := uint64(256)
density := uint64(p * float64(densityScale))
for density == 0 {
densityScale <<= 1
density = uint64(p * float64(densityScale))
// too small
if densityScale > (1 << 32) {
return nil
}
}
w, err := apophenia.NewWeighted(apophenia.NewSequence(seed))
if err != nil {
return nil
}
someBits := input.New()
toAdd := make([]uint64, batchSize)
toAddN := 0
offset := apophenia.OffsetFor(apophenia.SequenceWeighted, 0, 0, 0)
for i := min; i < max; i += 128 {
offset.Lo = i
randomBits := w.Bits(offset, density, densityScale)
bit := uint64(0)
for randomBits.Lo != 0 {
next := uint64(bits.TrailingZeros64(randomBits.Lo) + 1)
randomBits.Lo >>= next
toAdd[toAddN] = next + bit + i
toAddN++
bit += next
}
bit = 64
for randomBits.Hi != 0 {
next := uint64(bits.TrailingZeros64(randomBits.Hi) + 1)
randomBits.Hi >>= next
toAdd[toAddN] = next + bit + i
toAddN++
bit += next
}
if toAddN > (batchSize - 128) {
// ignore error
_, _ = someBits.AddN(toAdd[:toAddN]...)
toAddN = 0
}
}
if toAddN > 0 {
_, _ = someBits.AddN(toAdd[:toAddN]...)
}
return input.Intersect(someBits)
}
func main() {
fmt.Printf("this is a plugin module only.\n")
}

96
extension.go Normal file
View file

@ -0,0 +1,96 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"github.com/pilosa/pilosa/v2/ext"
"github.com/pilosa/pilosa/v2/roaring"
)
// WrapBitmap yields an extension-Bitmap from a roaring Bitmap.
func WrapBitmap(bm *roaring.Bitmap) ext.Bitmap {
return wrappedBitmap{bm}
}
// wrappedBitmap is a very shallow glue shim to convert a roaring Bitmap to
// an extension Bitmap.
type wrappedBitmap struct{ *roaring.Bitmap }
// UnwrapBitmap converts an extension-bitmap to its underlying roaring Bitmap.
func UnwrapBitmap(bm ext.Bitmap) *roaring.Bitmap {
if inner, ok := bm.(wrappedBitmap); ok {
if inner.Bitmap != nil {
return inner.Bitmap
}
return roaring.NewFileBitmap()
}
return roaring.NewFileBitmap()
}
func (b wrappedBitmap) Intersect(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Intersect(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) Union(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Union(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) IntersectionCount(other ext.Bitmap) uint64 {
return b.Bitmap.IntersectionCount(other.(wrappedBitmap).Bitmap)
}
func (b wrappedBitmap) Difference(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Difference(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) Xor(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Xor(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) Shift(n int) (ext.Bitmap, error) {
shifted, err := b.Bitmap.Shift(n)
return wrappedBitmap{shifted}, err
}
func (b wrappedBitmap) Flip(start, last uint64) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Flip(start, last)}
}
func (b wrappedBitmap) New() ext.Bitmap {
return WrapBitmap(roaring.NewFileBitmap())
}
// ContainerBits tries to get one container's worth of bits.
func (b wrappedBitmap) ContainerBits(offset uint64, target []uint64) (out []uint64) {
// it's an error to call this with a non-container-aligned offset
if offset&0xFFFF != 0 {
return nil
}
if b.Bitmap == nil {
fmt.Printf("ContainerBits on bitmap with no contents\n")
return nil
}
if b.Bitmap.Containers == nil {
fmt.Printf("ContainerBits on bitmap with nil Containers\n")
return nil
}
c := b.Bitmap.Containers.Get(offset >> 16)
if c == nil {
return nil
}
return c.AsBitmap(target)
}

31
field.go
View file

@ -91,8 +91,7 @@ type Field struct {
logger logger.Logger
snapshotQueue chan *fragment
snapshotQueue snapshotQueue
// Instantiates new translation store on open.
OpenTranslateStore OpenTranslateStoreFunc
}
@ -316,18 +315,30 @@ func (f *Field) mergeRemoteAvailableShards(b *roaring.Bitmap) {
// loadAvailableShards reads remoteAvailableShards data for the field, if any.
func (f *Field) loadAvailableShards() error {
bm := roaring.NewBitmap()
// Read data from meta file.
path := filepath.Join(f.path, ".available.shards")
buf, err := ioutil.ReadFile(path)
// doesn't exist: this is fine
if os.IsNotExist(err) {
return nil
} else if err != nil {
return errors.Wrap(err, "reading available shards")
} else {
if err := bm.UnmarshalBinary(buf); err != nil {
return errors.Wrap(err, "unmarshaling")
}
// some other problem:
if err != nil {
f.logger.Printf("available shards file present but unreadable, discarding: %v", err)
err = os.Remove(path)
if err != nil {
return errors.Wrap(err, "deleting corrupt available shards list")
}
return nil
}
bm := roaring.NewBitmap()
if err = bm.UnmarshalBinary(buf); err != nil {
f.logger.Printf("available shards file corrupt, discarding: %v", err)
err = os.Remove(path)
if err != nil {
return errors.Wrap(err, "deleting corrupt available shards list")
}
return nil
}
// Merge bitmap from file into field.
f.mergeRemoteAvailableShards(bm)
@ -912,7 +923,9 @@ func (f *Field) newView(path, name string) *view {
view.rowAttrStore = f.rowAttrStore
view.stats = f.Stats
view.broadcaster = f.broadcaster
view.snapshotQueue = f.snapshotQueue
if f.snapshotQueue != nil {
view.snapshotQueue = f.snapshotQueue
}
return view
}

57
field_internal_test.go
View file

@ -19,6 +19,7 @@ import (
"io/ioutil"
"math"
"os"
"path/filepath"
"reflect"
"testing"
"time"
@ -366,6 +367,62 @@ func TestField_PersistAvailableShards(t *testing.T) {
}
func TestField_CorruptAvailableShards(t *testing.T) {
f := MustOpenField(OptFieldTypeDefault())
// bm represents remote available shards.
bm := roaring.NewBitmap(1, 2, 3)
if err := f.AddRemoteAvailableShards(bm); err != nil {
t.Fatal(err)
}
path := filepath.Join(f.path, ".available.shards")
avail, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
t.Fatal(err)
}
n, err := avail.Write([]byte{23})
if err != nil || n != 1 {
t.Fatal(err)
}
avail.Close()
// Reload field and verify that shard data is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
}
}
func TestField_TruncatedAvailableShards(t *testing.T) {
f := MustOpenField(OptFieldTypeDefault())
// bm represents remote available shards.
bm := roaring.NewBitmap(1, 2, 3)
if err := f.AddRemoteAvailableShards(bm); err != nil {
t.Fatal(err)
}
path := filepath.Join(f.path, ".available.shards")
avail, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
t.Fatal(err)
}
avail.Close()
// Reload field and verify that shard data is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
}
}
// Ensure that persisting available shards having a smaller footprint (for example,
// when going from a bitmap to a smaller, RLE representation) succeeds.
func TestField_PersistAvailableShardsFootprint(t *testing.T) {

741
fragment.go
View file

@ -43,7 +43,6 @@ import (
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/shardwidth"
"github.com/pilosa/pilosa/v2/stats"
"github.com/pilosa/pilosa/v2/syswrap"
"github.com/pilosa/pilosa/v2/tracing"
"github.com/pkg/errors"
)
@ -108,21 +107,16 @@ type fragment struct {
shard uint64
// File-backed storage
path string
flags byte // user-defined flags passed to roaring
file *os.File
storage *roaring.Bitmap
storageData []byte
totalOpN int64 // total opN values
totalOps int64 // total ops (across all snapshots)
opN int // number of ops since snapshot (may be approximate for imports)
ops int // number of higher-level operations, as opposed to bit changes
snapshotsRequested int // number of times we've requested a snapshot
snapshotsTaken int // number of actual snapshot operations
snapshotting bool // set to true when requesting a snapshot, set to false after snapshot completes
snapshotCond sync.Cond
snapshotDelays int
snapshotDelayTime time.Duration
path string
flags byte // user-defined flags passed to roaring
gen generation
storage *roaring.Bitmap
opN int // number of ops since snapshot (may be approximate for imports)
ops int // number of higher-level operations, as opposed to bit changes
snapshotPending bool // set to true when requesting a snapshot, set to false after snapshot completes
snapshotCond sync.Cond
snapshotErr error // error yielded by the last snapshot operation
snapshotStamp time.Time // timestamp of last snapshot
// Cache for row counts.
CacheType string // passed in by field
@ -156,7 +150,7 @@ type fragment struct {
stats stats.StatsClient
snapshotQueue chan *fragment
snapshotQueue snapshotQueue
}
// newFragment returns a new instance of Fragment.
@ -174,7 +168,8 @@ func newFragment(path, index, field, view string, shard uint64, flags byte) *fra
Logger: logger.NopLogger,
MaxOpN: defaultFragmentMaxOpN,
stats: stats.NopStatsClient,
stats: stats.NopStatsClient,
snapshotQueue: defaultSnapshotQueue,
}
f.snapshotCond = sync.Cond{L: &f.mu}
return f
@ -183,62 +178,6 @@ func newFragment(path, index, field, view string, shard uint64, flags byte) *fra
// cachePath returns the path to the fragment's cache data.
func (f *fragment) cachePath() string { return f.path + cacheExt }
// newSnapshotQueue makes a new snapshot queue, of depth N, and spawns a
// goroutine for it.
func newSnapshotQueue(n int, w int, l logger.Logger) chan *fragment {
ch := make(chan *fragment, n)
for i := 0; i < w; i++ {
go snapshotQueueWorker(ch, l)
}
return ch
}
func snapshotQueueWorker(snapshotQueue chan *fragment, l logger.Logger) {
for f := range snapshotQueue {
err := f.protectedSnapshot(true)
if err != nil {
l.Printf("snapshot error: %v", err)
}
f.snapshotCond.Broadcast()
}
}
// enqueueSnapshot requests that the fragment be snapshotted at some point
// in the future, if this has not already been requested. Call this only when
// the mutex is held.
func (f *fragment) enqueueSnapshot() {
f.snapshotsRequested++
if f.snapshotting {
return
}
f.snapshotting = true
if f.snapshotQueue != nil {
select {
case f.snapshotQueue <- f:
default:
before := time.Now()
// wait forever, but notice that we're waiting
f.snapshotQueue <- f
f.snapshotDelays++
f.snapshotDelayTime += time.Since(before)
if f.snapshotDelays >= 10 {
f.Logger.Printf("snapshotting %s: last ten enqueue delays took %v", f.path, f.snapshotDelayTime)
f.snapshotDelays = 0
f.snapshotDelayTime = 0
}
}
} else {
// in testing, for instance, there may be no holder, thus no one
// to handle these snapshots.
err := f.snapshot()
if err != nil {
f.Logger.Printf("snapshot failed: %v", err)
}
f.snapshotting = false
f.snapshotCond.Broadcast()
}
}
// Open opens the underlying storage.
func (f *fragment) Open() error {
f.mu.Lock()
@ -254,6 +193,10 @@ func (f *fragment) Open() error {
// Fill cache with rows persisted to disk.
f.Logger.Debugf("open cache for index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
if err := f.openCache(); err != nil {
e2 := f.closeStorage()
if e2 != nil {
return errors.Wrapf(err, "closing storage: %v, after opening cache", e2)
}
return errors.Wrap(err, "opening cache")
}
@ -273,48 +216,103 @@ func (f *fragment) Open() error {
return nil
}
func (f *fragment) reopen() (mustClose bool, err error) {
if f.file == nil {
// Open the data file to be mmap'd and used as an ops log.
f.file, mustClose, err = syswrap.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
return mustClose, fmt.Errorf("open file: %s", err)
}
f.storage.OpWriter = f.file
// emptyStorage is the common case for importStorage/applyStorage where they
// get no data. It tries to write the current storage to the provided file,
// which is assumed to be the file they didn't get any data from.
func (f *fragment) emptyStorage(file *os.File) (bool, error) {
// No data. We'll mark this for no mapping, clear any existing
// mapped containers, and set the Source to nil. We also have no
// ops.
f.opN = 0
f.ops = 0
f.storage.SetOps(0, 0)
f.storage.PreferMapping(false)
_, err := f.storage.RemapRoaringStorage(nil)
f.storage.SetSource(nil)
if err != nil {
return false, fmt.Errorf("applying/importing storage: no data, and clearing old mapping also failed: %v", err)
}
return mustClose, nil
// Write the existing storage out to the file so it's
// a valid Roaring file thereafter. nothing to unmarshal.
// In the unlikely event that this happened even though we
// had significant data, we're not mapping it, but that's
// harmless even if it's not maximally efficient.
bi := bufio.NewWriter(file)
if _, err = f.storage.WriteTo(bi); err != nil {
return false, fmt.Errorf("init storage file: %s", err)
}
bi.Flush()
return false, nil
}
// openStorage opens the storage bitmap. Usually you also want to read in
// the storage, but in the case where we just wrote that file, such as
// unprotectedWriteToFragment, we could also just... not. If we didn't
// have existing storage, we probably need to unmarshal the data. If the
// file we're asked to open is empty, we probably don't.
//
// If we already had mapped storage previously, we want to unmap that, and
// possibly remap it from the file, but we don't need a full unmarshal, just
// an update of mapped pointers.
//
// unmarshalData is somewhat overloaded. it tells us whether or not we
// need to actually create a bitmap from the data (if the data exists to
// do this from).
//
// usually unmarshalData is only set to false when we're in the middle of
// a snapshot, and unprotectedWriteToFragment just wrote the in-memory data
// out.
//
// If we have existing storage data, and we successfully get new data,
// we will unmap the existing storage data.
//
// This function's design is probably a problem -- it is trying to handle
// both cases where there was existing data before, and cases where we
// just wrote the data.
func (f *fragment) openStorage(unmarshalData bool) error {
oldStorageData := f.storageData
// there's a few places where we might encounter an error, but need
// to continue past it through other error checks, before returning it.
var lastError error
// importStorage attempts to import data from storage -- for instance,
// reading in a roaring bitmap from media.
func (f *fragment) importStorage(data []byte, file *os.File, newGen generation, mapped bool) (bool, error) {
f.storage.PreferMapping(mapped)
if len(data) == 0 {
return f.emptyStorage(file)
}
// UnmarshalBinary will have remapped the storage to newGen if it
// succeeded, or if it fails but the error is advisory-only. So we
// optimistically set the source here, but if there's a non-advisory
// error, we'll unmap it and then set the source to nil.
f.storage.SetSource(newGen)
if err := f.storage.UnmarshalBinary(data); err != nil {
// roaring can report advisory-only errors...
cause := errors.Cause(err)
_, ok := cause.(roaring.AdvisoryError)
if !ok {
_, e2 := f.storage.RemapRoaringStorage(nil)
f.storage.SetSource(nil)
if e2 != nil {
return false, fmt.Errorf("unmarshal storage: file=%s, err=%s, clearing old mapping also failed: %v", file.Name(), err, e2)
}
return false, fmt.Errorf("unmarshal storage: file=%s, err=%s", file.Name(), err)
}
f.Logger.Printf("warning: unmarshal storage, file=%s, err=%v", file.Name(), err)
trunc, ok := cause.(roaring.FileShouldBeTruncatedError)
if ok {
// generation code looks for a FileShouldBeTruncatedError
return false, trunc
}
}
f.ops, f.opN = f.storage.Ops()
// For now, we assume that UnmarshalBinary will have mapped at least
// one container if we told it the storage was mapped and it didn't
// error out. This might be wrong in occasional trivial cases, but
// it should be harmless.
return mapped, nil
}
// applyStorage applies storage to a fragment that may already have
// usable data. For instance, this would try to remap existing containers
// to use a new storage as backing store.
func (f *fragment) applyStorage(data []byte, file *os.File, newGen generation, mapped bool) (bool, error) {
if len(data) == 0 {
return f.emptyStorage(file)
}
// Tell storage to prefer mapping if and only if we think the data
// is mmapped and valid.
f.storage.PreferMapping(mapped)
f.storage.SetSource(newGen)
// RemapRoaringStorage will fix any mapped containers to point either
// to the provided data (if PreferMapping was called with true and
// data is provided and there's a corresponding container) or to
// allocated storage, so when it's done, there's nothing in it that
// is mapped to anything *other than* the provided data.
return f.storage.RemapRoaringStorage(data)
}
// openStorage opens the storage bitmap.
//
// This has been massively reworked recently, and now hands a lot of
// file management off to the generation object and the Done method
// of that object. Similarly, the bitmap mapping/remapping
// logic is now mostly in importStorage (reading in a bitmap) and applyStorage
// (remapping an existing bitmap to match a new backing store).
func (f *fragment) openStorage(unmarshalData bool) error {
// Create a roaring bitmap to serve as storage for the shard.
if f.storage == nil {
f.storage = roaring.NewFileBitmap()
@ -323,139 +321,23 @@ func (f *fragment) openStorage(unmarshalData bool) error {
// unmarshal this data in order to have any.
unmarshalData = true
}
// Open the data file to be mmap'd and used as an ops log.
file, mustClose, err := syswrap.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
return fmt.Errorf("open file: %s", err)
}
f.file = file
if mustClose {
defer f.safeClose()
}
// Lock the underlying file.
if err := syscall.Flock(int(f.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
return fmt.Errorf("flock: %s", err)
}
// data is the data we would unmarshal from, if we're unmarshalling; it might
// be obtained by calling ReadAll on a file.
//
// newStorageData is the data we should map things to. it is set only if
// mmapped; if we didn't mmap (say, we couldn't), we won't want to unmap
// the ioutil byte slice. (Theoretically, we shouldn't be using the mapped
// flag in that case...)
var data []byte
var newStorageData []byte
// If the file is empty then initialize it with an empty bitmap.
fi, err := f.file.Stat()
if err != nil {
return errors.Wrap(err, "statting file before")
} else if fi.Size() == 0 {
bi := bufio.NewWriter(f.file)
var err error
if _, err = f.storage.WriteTo(bi); err != nil {
return fmt.Errorf("init storage file: %s", err)
}
bi.Flush()
_, err = f.file.Stat()
if err != nil {
return errors.Wrap(err, "statting file after")
}
// there's nothing here, we're not going to try to unmarshal it.
unmarshalData = false
f.rowCache = &simpleCache{make(map[uint64]*Row)}
} else {
// Mmap the underlying file so it can be zero copied.
data, err = syswrap.Mmap(int(f.file.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
if err == syswrap.ErrMaxMapCountReached {
f.Logger.Debugf("maximum number of maps reached, reading file instead")
if unmarshalData {
data, err = ioutil.ReadAll(file)
if err != nil {
return errors.Wrap(err, "failure file readall")
}
}
} else if err != nil {
return errors.Wrap(err, "mmap failed")
} else {
newStorageData = data
}
}
f.rowCache = &simpleCache{make(map[uint64]*Row)}
var storageOp func([]byte, *os.File, generation, bool) (bool, error)
if unmarshalData {
f.storageData = newStorageData
// We're about to either re-read the bitmap, or fail to do so
// and unconditionally unmap the existing stuff. Either way, we
// want to unmap the old storage data after we're done here, but
// we can't unmap it yet because it's still live until sometime
// later, but we can't unmap it later, because we could return
// early... this is what defer is for.
if oldStorageData != nil {
defer func() {
unmapErr := syswrap.Munmap(oldStorageData)
if unmapErr != nil {
f.Logger.Printf("unmap of old storage failed: %s", err)
}
}()
}
// set the preference for mapping based on whether the data's mmapped
f.storage.PreferMapping(newStorageData != nil)
// so we have a problem here: if this fails, it's unclear whether
// *either* or *both* of old and new storage data might be in use.
// So we call the thing that should unconditionally unmap both of them...
if err := f.storage.UnmarshalBinary(data); err != nil {
_, e2 := f.storage.RemapRoaringStorage(nil)
if e2 != nil {
return fmt.Errorf("unmarshal storage: file=%s, err=%s, clearing old mapping also failed: %v", f.file.Name(), err, e2)
}
return fmt.Errorf("unmarshal storage: file=%s, err=%s", f.file.Name(), err)
}
f.rowCache = &simpleCache{make(map[uint64]*Row)}
f.ops, f.opN = f.storage.Ops()
storageOp = f.importStorage
} else {
// we're moving to new storage, so instead of using the OpN
// derived from reading that storage, we notify the bitmap that
// OpN is now effectively zero.
f.opN = 0
f.ops = 0
f.storage.SetOps(0, 0)
// if oldStorageData is nil, this just tries to unmap any bits that
// are currently mapped. otherwise, it will point them at this
// storage (if the containers match).
var mappedAny bool
mappedAny, lastError = f.storage.RemapRoaringStorage(newStorageData)
if oldStorageData != nil {
unmapErr := syswrap.Munmap(oldStorageData)
if unmapErr != nil {
f.Logger.Printf("unmap of old storage failed: %s", err)
}
}
if mappedAny {
// Advise the kernel that the mmap is accessed randomly.
if err := madvise(newStorageData, syscall.MADV_RANDOM); err != nil {
lastError = fmt.Errorf("madvise: %s", err)
}
} else {
// if we did map data, but for some reason none of it got used
// as backing store, we can unmap it, and set the slice to nil,
// so we don't keep the now-invalid slice in f.storageData.
if newStorageData != nil {
unmapErr := syswrap.Munmap(newStorageData)
if unmapErr != nil {
lastError = fmt.Errorf("unmapping unused storage data: %s", err)
}
newStorageData = nil
}
}
f.storageData = newStorageData
storageOp = f.applyStorage
}
// Attach the file to the bitmap to act as a write-ahead log.
f.storage.OpWriter = f.file
return lastError
var err error
f.gen, err = newGeneration(f.gen, f.path, unmarshalData, storageOp, f.Logger)
if generationDebug {
// We might have already done this anyway, if we think we
// mapped stuff, but when debugging we want to do it
// unconditionally, because the test cases otherwise won't
// exercise this code well.
f.storage.SetSource(f.gen)
}
return err
}
// openCache initializes the cache from row ids persisted to disk.
@ -504,31 +386,12 @@ func (f *fragment) openCache() error {
func (f *fragment) Close() error {
f.mu.Lock()
defer f.mu.Unlock()
for f.snapshotting {
for f.snapshotPending {
f.snapshotCond.Wait()
}
return f.close()
}
// awaitSnapshot lets us delay until the snapshot gets written, preventing tests
// from misleadingly showing amazingly fast performance because the snapshots they
// trigger haven't happened yet.
func (f *fragment) awaitSnapshot() {
f.mu.Lock()
defer f.mu.Unlock()
for f.snapshotting {
f.snapshotCond.Wait()
}
}
// unprotectedAwaitSnapshot assumes you already hold the lock, and waits for
// the snapshot fairy to come along.
func (f *fragment) unprotectedAwaitSnapshot() {
for f.snapshotting {
f.snapshotCond.Wait()
}
}
func (f *fragment) close() error {
// Flush cache if closing gracefully.
if err := f.flushCache(); err != nil {
@ -537,7 +400,7 @@ func (f *fragment) close() error {
}
// Close underlying storage.
if err := f.closeStorage(true); err != nil {
if err := f.closeStorage(); err != nil {
f.Logger.Printf("fragment: error closing storage: err=%s, path=%s", err, f.path)
return errors.Wrap(err, "closing storage")
}
@ -548,54 +411,17 @@ func (f *fragment) close() error {
return nil
}
// safeClose is unprotected.
func (f *fragment) safeClose() error {
// Flush file, unlock & close.
if f.file != nil {
if err := f.file.Sync(); err != nil {
return fmt.Errorf("sync: %s", err)
}
if err := syscall.Flock(int(f.file.Fd()), syscall.LOCK_UN); err != nil {
return fmt.Errorf("unlock: %s", err)
}
if err := syswrap.CloseFile(f.file); err != nil {
return fmt.Errorf("close file: %s", err)
}
}
f.file = nil
f.storage.OpWriter = nil
return nil
}
// closeStorage attempts to close storage, including unmapping the old
// storage if includeMap is true. This would normally make sense if you're
// expecting to be done using the fragment, or to reload it. But it's also
// okay to just leave stuff mmapped; you don't have to keep the file
// descriptor open. So in some cases, we'll just leave the old mmapping
// in place, rather than regenerating everything from the new file.
func (f *fragment) closeStorage(includeMap bool) error {
// Clear the storage bitmap so it doesn't access the closed mmap.
//f.storage = roaring.NewBitmap()
// Unmap the file.
if includeMap && f.storageData != nil {
if err := syswrap.Munmap(f.storageData); err != nil {
return fmt.Errorf("munmap: %s", err)
}
f.storageData = nil
}
if err := f.safeClose(); err != nil {
return err
}
// closeStorage marks the current generation as done. It is not necessary
// to call this before openStorage.
func (f *fragment) closeStorage() error {
// opN is determined by how many bit set/clear operations are in the storage
// write log, so once the storage is closed it should be 0. Opening new
// storage will set opN appropriately.
f.opN = 0
if f.gen != nil {
f.gen.Done()
}
return nil
}
@ -648,22 +474,17 @@ func (f *fragment) rowFromStorage(rowID uint64) *Row {
func (f *fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
mustClose, err := f.reopen()
if err != nil {
return false, errors.Wrap(err, "reopening")
}
if mustClose {
defer f.safeClose()
}
// handle mutux field type
if f.mutexVector != nil {
if err := f.handleMutex(rowID, columnID); err != nil {
return changed, errors.Wrap(err, "handling mutex")
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
// handle mutux field type
if f.mutexVector != nil {
if err := f.handleMutex(rowID, columnID); err != nil {
return errors.Wrap(err, "handling mutex")
}
}
}
return f.unprotectedSetBit(rowID, columnID)
changed, err = f.unprotectedSetBit(rowID, columnID)
return err
})
return changed, err
}
// handleMutex will clear an existing row and store the new row
@ -727,17 +548,14 @@ func (f *fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err
// clearBit clears a bit for a given column & row within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *fragment) clearBit(rowID, columnID uint64) (bool, error) {
func (f *fragment) clearBit(rowID, columnID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
mustClose, err := f.reopen()
if err != nil {
return false, errors.Wrap(err, "reopening")
}
if mustClose {
defer f.safeClose()
}
return f.unprotectedClearBit(rowID, columnID)
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
changed, err = f.unprotectedClearBit(rowID, columnID)
return err
})
return changed, err
}
// unprotectedClearBit TODO should be replaced by an invocation of
@ -783,17 +601,14 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er
// setRow replaces an existing row (specified by rowID) with the given
// Row. This updates both the on-disk storage and the in-cache bitmap.
func (f *fragment) setRow(row *Row, rowID uint64) (bool, error) {
func (f *fragment) setRow(row *Row, rowID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
mustClose, err := f.reopen()
if err != nil {
return false, errors.Wrap(err, "reopening")
}
if mustClose {
defer f.safeClose()
}
return f.unprotectedSetRow(row, rowID)
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
changed, err = f.unprotectedSetRow(row, rowID)
return err
})
return changed, err
}
func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err error) {
@ -834,7 +649,7 @@ func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err
f.rowCache.Add(rowID, nil)
// Snapshot storage.
f.enqueueSnapshot()
f.snapshotQueue.Enqueue(f)
f.stats.Count("setRow", 1, 1.0)
return changed, nil
@ -842,17 +657,14 @@ func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err
// ClearRow clears a row for a given rowID within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *fragment) clearRow(rowID uint64) (bool, error) {
func (f *fragment) clearRow(rowID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
mustClose, err := f.reopen()
if err != nil {
return false, errors.Wrap(err, "reopening")
}
if mustClose {
defer f.safeClose()
}
return f.unprotectedClearRow(rowID)
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
changed, err = f.unprotectedClearRow(rowID)
return err
})
return changed, err
}
func (f *fragment) unprotectedClearRow(rowID uint64) (changed bool, err error) {
@ -878,7 +690,7 @@ func (f *fragment) unprotectedClearRow(rowID uint64) (changed bool, err error) {
f.rowCache.Add(rowID, nil)
// Snapshot storage.
f.enqueueSnapshot()
f.snapshotQueue.Enqueue(f)
f.stats.Count("clearRow", 1, 1.0)
@ -978,67 +790,62 @@ func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64
func (f *fragment) setValueBase(columnID uint64, bitDepth uint, value int64, clear bool) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
mustClose, err := f.reopen()
if err != nil {
return false, errors.Wrap(err, "reopening")
}
if mustClose {
defer f.safeClose()
}
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
// Convert value to an unsigned representation.
uvalue := uint64(value)
if value < 0 {
uvalue = uint64(-value)
}
// Convert value to an unsigned representation.
uvalue := uint64(value)
if value < 0 {
uvalue = uint64(-value)
}
for i := uint(0); i < bitDepth; i++ {
if uvalue&(1<<i) != 0 {
if c, err := f.unprotectedSetBit(uint64(bsiOffsetBit+i), columnID); err != nil {
return err
} else if c {
changed = true
}
} else {
if c, err := f.unprotectedClearBit(uint64(bsiOffsetBit+i), columnID); err != nil {
return err
} else if c {
changed = true
}
}
}
for i := uint(0); i < bitDepth; i++ {
if uvalue&(1<<i) != 0 {
if c, err := f.unprotectedSetBit(uint64(bsiOffsetBit+i), columnID); err != nil {
return changed, err
// Mark value as set (or cleared).
if clear {
if c, err := f.unprotectedClearBit(uint64(bsiExistsBit), columnID); err != nil {
return errors.Wrap(err, "clearing not-null")
} else if c {
changed = true
}
} else {
if c, err := f.unprotectedClearBit(uint64(bsiOffsetBit+i), columnID); err != nil {
return changed, err
if c, err := f.unprotectedSetBit(uint64(bsiExistsBit), columnID); err != nil {
return errors.Wrap(err, "marking not-null")
} else if c {
changed = true
}
}
}
// Mark value as set (or cleared).
if clear {
if c, err := f.unprotectedClearBit(uint64(bsiExistsBit), columnID); err != nil {
return changed, errors.Wrap(err, "clearing not-null")
} else if c {
changed = true
// Mark sign bit (or clear).
if value >= 0 || clear {
if c, err := f.unprotectedClearBit(uint64(bsiSignBit), columnID); err != nil {
return errors.Wrap(err, "clearing sign")
} else if c {
changed = true
}
} else {
if c, err := f.unprotectedSetBit(uint64(bsiSignBit), columnID); err != nil {
return errors.Wrap(err, "marking sign")
} else if c {
changed = true
}
}
} else {
if c, err := f.unprotectedSetBit(uint64(bsiExistsBit), columnID); err != nil {
return changed, errors.Wrap(err, "marking not-null")
} else if c {
changed = true
}
}
// Mark sign bit (or clear).
if value >= 0 || clear {
if c, err := f.unprotectedClearBit(uint64(bsiSignBit), columnID); err != nil {
return changed, errors.Wrap(err, "clearing sign")
} else if c {
changed = true
}
} else {
if c, err := f.unprotectedSetBit(uint64(bsiSignBit), columnID); err != nil {
return changed, errors.Wrap(err, "marking sign")
} else if c {
changed = true
}
}
return changed, nil
return nil
})
return changed, err
}
// importSetValue is a more efficient SetValue just for imports.
@ -2066,52 +1873,46 @@ func (f *fragment) bulkImportStandard(rowIDs, columnIDs []uint64, options *Impor
// snapshot of the fragment or just do in-memory updates while appending
// operations to the op log.
func (f *fragment) importPositions(set, clear []uint64, rowSet map[uint64]struct{}) error {
mustClose, err := f.reopen()
if err != nil {
return errors.Wrap(err, "reopening")
}
if mustClose {
defer f.safeClose()
}
if len(set) > 0 {
f.stats.Count("ImportingN", int64(len(set)), 1)
changedN, err := f.storage.AddN(set...) // TODO benchmark Add/RemoveN behavior with sorted/unsorted positions
if err != nil {
return errors.Wrap(err, "adding positions")
err := f.gen.Transaction(&f.storage.OpWriter, func() error {
if len(set) > 0 {
f.stats.Count("ImportingN", int64(len(set)), 1)
changedN, err := f.storage.AddN(set...) // TODO benchmark Add/RemoveN behavior with sorted/unsorted positions
if err != nil {
return errors.Wrap(err, "adding positions")
}
f.stats.Count("ImportedN", int64(changedN), 1)
f.incrementOpN(changedN)
}
f.stats.Count("ImportedN", int64(changedN), 1)
f.incrementOpN(changedN)
}
if len(clear) > 0 {
f.stats.Count("ClearingN", int64(len(clear)), 1)
changedN, err := f.storage.RemoveN(clear...)
if err != nil {
return errors.Wrap(err, "clearing positions")
if len(clear) > 0 {
f.stats.Count("ClearingN", int64(len(clear)), 1)
changedN, err := f.storage.RemoveN(clear...)
if err != nil {
return errors.Wrap(err, "clearing positions")
}
f.stats.Count("ClearedN", int64(changedN), 1)
f.incrementOpN(changedN)
}
f.stats.Count("ClearedN", int64(changedN), 1)
f.incrementOpN(changedN)
}
// Update cache counts for all affected rows.
for rowID := range rowSet {
// Invalidate block checksum.
delete(f.checksums, int(rowID/HashBlockSize))
// Update cache counts for all affected rows.
for rowID := range rowSet {
// Invalidate block checksum.
delete(f.checksums, int(rowID/HashBlockSize))
if f.CacheType != CacheTypeNone {
n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth)
f.cache.BulkAdd(rowID, n)
}
f.rowCache.Add(rowID, nil)
}
if f.CacheType != CacheTypeNone {
n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth)
f.cache.BulkAdd(rowID, n)
f.cache.Recalculate()
}
f.rowCache.Add(rowID, nil)
}
if f.CacheType != CacheTypeNone {
f.cache.Recalculate()
}
return nil
return nil
})
return err
}
// bulkImportMutex performs a bulk import on a fragment while ensuring
@ -2197,7 +1998,6 @@ func (f *fragment) importValueSmallWrite(columnIDs []uint64, values []int64, bit
}
return nil
}(); err != nil {
_ = f.closeStorage(true)
_ = f.openStorage(true)
return err
}
@ -2245,23 +2045,23 @@ func (f *fragment) importValue(columnIDs []uint64, values []int64, bitDepth uint
}
return nil
}(); err != nil {
_ = f.closeStorage(true)
_ = f.openStorage(true)
return err
}
// We don't actually care, except we want our stats to be accurate.
f.incrementOpN(totalChanges)
// Keep stats accurate. We don't call incrementOpN here because it may
// or may not enqueue a request, which would then be in the queue
// taking up space and otherwise being a possible nuisance, when we're
// about to force a snapshot anyway.
f.opN += totalChanges
f.ops++
// Reset the rowCache.
f.rowCache = &simpleCache{make(map[uint64]*Row)}
// in theory, this should probably have happened anyway, but if enough
// in theory, this should probably have been queued anyway, but if enough
// of the bits matched existing bits, we'll be under our opN estimate, and
// we want to ensure that the snapshot happens.
f.enqueueSnapshot()
f.unprotectedAwaitSnapshot()
return nil
return f.snapshotQueue.Immediate(f)
}
// importRoaring imports from the official roaring data format defined at
@ -2276,7 +2076,13 @@ func (f *fragment) importRoaring(ctx context.Context, data []byte, clear bool) e
defer f.mu.Unlock()
span.Finish()
span, ctx = tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits")
changed, rowSet, err := f.storage.ImportRoaringBits(data, clear, true, rowSize)
var changed int
var rowSet map[uint64]int
err := f.gen.Transaction(&f.storage.OpWriter, func() (err error) {
changed, rowSet, err = f.storage.ImportRoaringBits(data, clear, true, rowSize)
return err
})
span.Finish()
if err != nil {
return err
@ -2315,14 +2121,14 @@ func (f *fragment) incrementOpN(changed int) {
f.opN += changed
f.ops++
if f.opN > f.MaxOpN {
f.enqueueSnapshot()
f.snapshotQueue.Enqueue(f)
}
}
// Snapshot writes the storage bitmap to disk and reopens it. This may
// coexist with existing background-queue snapshotting; it does not remove
// things from the queue. You probably don't want to do this; use
// enqueueSnapshot/awaitSnapshot.
// the snapshotQueue's Enqueue/Await.
func (f *fragment) Snapshot() error {
f.mu.Lock()
defer f.mu.Unlock()
@ -2335,25 +2141,13 @@ func track(start time.Time, message string, stats stats.StatsClient, logger logg
stats.Histogram("snapshot", elapsed.Seconds(), 1.0)
}
// protectedSnapshot grabs the lock and unconditionally calls snapshot(). If
// fromQueue is true, the snapshotting state is also cleared.
func (f *fragment) protectedSnapshot(fromQueue bool) error {
f.mu.Lock()
defer f.mu.Unlock()
err := f.snapshot()
if fromQueue {
f.snapshotting = false
}
return err
}
// snapshot does the actual snapshot operation. it does not check or care
// about f.snapshotting.
// about f.snapshotPending.
func (f *fragment) snapshot() error {
f.totalOpN += int64(f.opN)
f.totalOps += int64(f.ops)
f.snapshotsTaken++
_, err := unprotectedWriteToFragment(f, f.storage)
if err == nil {
f.snapshotStamp = time.Now()
}
return err
}
@ -2370,22 +2164,24 @@ func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) (n int64, err e
if err != nil {
return n, fmt.Errorf("create snapshot file: %s", err)
}
defer file.Close()
// No deferred close, because we want to close it sooner than the
// end of this function.
// Write storage to snapshot.
bw := bufio.NewWriter(file)
if n, err = bm.WriteTo(bw); err != nil {
file.Close()
return n, fmt.Errorf("snapshot write to: %s", err)
}
if err := bw.Flush(); err != nil {
file.Close()
return n, fmt.Errorf("flush: %s", err)
}
// Close current storage.
if err := f.closeStorage(false); err != nil {
return n, fmt.Errorf("close storage: %s", err)
}
// we close the file here so we don't still have it open when trying
// to open it in a moment.
file.Close()
// Move snapshot to data file location.
if err := os.Rename(snapshotPath, f.path); err != nil {
@ -2585,11 +2381,6 @@ func (f *fragment) readStorageFromArchive(r io.Reader) error {
return errors.Wrap(err, "copying")
}
// Close current storage.
if err := f.closeStorage(true); err != nil {
return errors.Wrap(err, "closing")
}
// Move snapshot to data file location.
if err := os.Rename(path, f.path); err != nil {
return errors.Wrap(err, "renaming")

89
fragment_internal_test.go
View file

@ -1397,6 +1397,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
// Read into another fragment.
f1 := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f1.Clean(t)
if rn, err := f1.ReadFrom(&buf); err != nil {
t.Fatal(err)
} else if wn != rn {
@ -2069,7 +2070,11 @@ func BenchmarkImportRoaring(b *testing.B) {
b.StartTimer()
err := f.importRoaringT(data, false)
if err != nil {
f.awaitSnapshot()
// we don't actually particularly
// care whether this succeeds,
// but if it's happening we want
// it to be done.
_ = f.snapshotQueue.Await(f)
f.Clean(b)
b.Fatalf("import error: %v", err)
}
@ -2108,7 +2113,9 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) {
j := j
eg.Go(func() error {
err := frags[j].importRoaringT(data[j], false)
frags[j].awaitSnapshot()
// error unimportant if it happened, but we want
// any snapshots to have finished.
_ = frags[j].snapshotQueue.Await(frags[j])
return err
})
}
@ -2146,11 +2153,13 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
// is excessive. force storage into snapshotted state, then use import
// to generate an op log and/or snapshot.
_, _, err := frags[j].storage.ImportRoaringBits(data, false, false, 0)
frags[j].enqueueSnapshot()
frags[j].awaitSnapshot()
if err != nil {
b.Fatalf("importing roaring: %v", err)
}
err = frags[j].snapshotQueue.Immediate(frags[j])
if err != nil {
b.Fatalf("snapshot after import: %v", err)
}
}
eg := errgroup.Group{}
b.StartTimer()
@ -2158,7 +2167,10 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
j := j
eg.Go(func() error {
err := frags[j].importRoaringT(updata, false)
frags[j].awaitSnapshot()
err2 := frags[j].snapshotQueue.Await(frags[j])
if err == nil {
err = err2
}
return err
})
}
@ -2220,20 +2232,38 @@ func BenchmarkImportRoaringUpdate(b *testing.B) {
// is excessive. force storage into snapshotted state, then use import
// to generate an op log and/or snapshot.
_, _, err := f.storage.ImportRoaringBits(data, false, false, 0)
f.enqueueSnapshot()
f.awaitSnapshot()
if err != nil {
b.Errorf("import error: %v", err)
}
err = f.snapshotQueue.Immediate(f)
if err != nil {
b.Errorf("snapshot after import error: %v", err)
}
b.StartTimer()
err = f.importRoaringT(updata, false)
f.awaitSnapshot()
if err != nil {
f.Clean(b)
b.Errorf("import error: %v", err)
}
err = f.snapshotQueue.Await(f)
if err != nil {
b.Errorf("snapshot after import error: %v", err)
}
b.StopTimer()
stat, _ := f.file.Stat()
var stat os.FileInfo
var statTarget io.Writer
err = f.gen.Transaction(&statTarget, func() error {
targetFile, ok := statTarget.(*os.File)
if ok {
stat, _ = targetFile.Stat()
} else {
b.Errorf("couldn't stat file")
}
return nil
})
if err != nil {
b.Errorf("transaction error: %v", err)
}
fileSize[name] = stat.Size()
f.Clean(b)
}
@ -2382,6 +2412,7 @@ func TestGetZipfRowsSliceRoaring(t *testing.T) {
t.Fatalf("suspect distribution from getZipfRowsSliceRoaring")
}
}
f.Clean(t)
}
// getZipfRowsSliceRoaring generates a random fragment with the given number of
@ -2527,16 +2558,28 @@ func (f *fragment) sanityCheck(t testing.TB) {
}
func (f *fragment) Clean(t testing.TB) {
f.awaitSnapshot()
f.mu.Lock()
err := f.snapshotQueue.Await(f)
f.mu.Unlock()
if err != nil {
t.Fatalf("snapshot failed before sanity check: %v", err)
}
f.sanityCheck(t)
if f.storage != nil && f.storage.Source != nil {
if f.storage.Source.Dead() {
t.Fatalf("cleaning up fragment %s, source %s, source already dead", f.path, f.storage.Source.ID())
}
}
errc := f.Close()
// prevent double-closes of generation during testing.
f.gen = nil
errf := os.Remove(f.path)
errp := os.Remove(f.cachePath())
if errc != nil || errf != nil {
t.Fatal("cleaning up fragment: ", errc, errf, errp)
}
if f.snapshotQueue != nil {
close(f.snapshotQueue)
f.snapshotQueue.Stop()
f.snapshotQueue = nil
}
// not all fragments have cache files
@ -2559,7 +2602,7 @@ func (f *fragment) CleanKeep(t testing.TB) {
t.Fatal("closing fragment: ", errc, errp)
}
if f.snapshotQueue != nil {
close(f.snapshotQueue)
f.snapshotQueue.Stop()
f.snapshotQueue = nil
}
// not all fragments have cache files
@ -3047,10 +3090,10 @@ func TestFragmentRowIterator(t *testing.T) {
func TestUnionInPlaceMapped(t *testing.T) {
f := mustOpenFragment("i", "f", "v", 0, CacheTypeNone)
// note: clean has to be deferred first, because it has to run with
// the lock *not* held, because it is sometimes so it has to grab the
// lock...
defer f.Clean(t)
// I know this doesn't actually matter in our current context, but
// strictly speaking, we do say you have to hold the lock while calling
// unprotectedWriteToFragment...
f.mu.Lock()
defer f.mu.Unlock()
r0 := rand.New(rand.NewSource(2))
@ -3083,8 +3126,15 @@ func TestUnionInPlaceMapped(t *testing.T) {
f.storage.UnionInPlace(setBM1)
countUnion := f.storage.Count()
// UnionInPlace produces no ops log, we have to make it snapshot, to
// ensure that the on-disk representation is correct.
f.enqueueSnapshot()
// ensure that the on-disk representation is correct. Note, UIP is
// not used for things that are modifying real fragments, usually;
// it's used only in computation of things that usually don't go to
// disk, which is why we handle this specially in testing and not
// generically.
err = f.snapshotQueue.Immediate(f)
if err != nil {
t.Fatalf("snapshot after union-in-place: %v", err)
}
if count0 != countF {
t.Fatalf("writing bitmap to storage changed count: %d => %d", count0, countF)
@ -3272,7 +3322,7 @@ func TestImportClearRestart(t *testing.T) {
f2.MaxOpN = maxOpN
f2.CacheType = f.CacheType
err = f.closeStorage(true)
err = f.closeStorage()
if err != nil {
t.Fatalf("closing storage: %v", err)
}
@ -3306,7 +3356,7 @@ func TestImportClearRestart(t *testing.T) {
f3.MaxOpN = maxOpN
f3.CacheType = f.CacheType
err = f2.closeStorage(true)
err = f2.closeStorage()
if err != nil {
t.Fatalf("f2 closing storage: %v", err)
}
@ -3354,6 +3404,7 @@ func check(t *testing.T, f *fragment, exp map[uint64]map[uint64]struct{}) {
func TestImportValueConcurrent(t *testing.T) {
f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0)
defer f.Clean(t)
eg := &errgroup.Group{}
for i := 0; i < 4; i++ {
i := i

407
generation.go Normal file
View file

@ -0,0 +1,407 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"io"
"io/ioutil"
"os"
"runtime"
"sync"
"syscall"
"time"
"github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/syswrap"
"github.com/pkg/errors"
)
// generation represents one "generation" of opening a data file.
// This is what determines when it's safe to unmap a data file, if it
// got mapped, and handles closing/reopening files if we need to
// manage file handle availability. It's an interface because this
// lets us write simpler code for specific cases, rather than handling
// the whole matrix of mapped/unmapped, staying open/being reopened,
// etcetera.
//
// You create a generation by calling newGeneration with a file
// path. If it succeeds in opening that path, it calls a provided
// setup function with the data from the generation, and a flag
// indicating whether the data is mmapped. If the setup function
// fails, newGeneration cleans things up and closes. Otherwise,
// it returns a generation.
//
// The generation itself uses runtime.SetFinalizer to clean up when
// the last reference to it goes away. You should store a pointer
// to the generation in any object which is reliant on the generation.
//
// When you anticipate a generation should be done (for instance,
// opening a new generation), the old one gets marked done, which
// stashes a timestamp in it. Later operations can check whether
// the timestamp is a while back, and if so, complain that something
// might be wrong.
//
// In some cases, we don't have enough open file limit to keep every
// file actually open. To address this, use the `Transaction` function,
// which ensures that the file is open, stores a reference to it in
// a provided `*io.Writer`, and then restores the previous value of
// the io.Writer when it's done. For instance, for a bitmap, this might
// be used with `&b.OpWriter`.
//
// newGeneration takes an optional previous generation; it calls
// that generation's Done function after running the provided setup,
// and bumps the generation count.
type generation interface {
// Transaction runs the given transaction with the generation's
// file open. If the **os.File parameter is
// non-nil, the generation's file will be open, and stored
// into that pointer, during the execution of func, after
// which the previous contents are restored. Otherwise
// the file may or may not be open during the operation.
Transaction(*io.Writer, func() error) error
// Done() should be called exactly once, to indicate that a
// generation is expected not to be in use for long -- for instance,
// when a new generation replaces it.
Done()
// Generation count.
Generation() int64
// ID indicates the source -- path and generation number -- that
// this generation represents.
ID() string
Dead() bool
}
type mmapGeneration struct {
mu sync.Mutex // mutex guards modifiers of generation, not of data
transMu sync.Mutex // guards transactions, specifically
path string
id string
file *os.File
data []byte
generation int64 // generation counter
dead bool // we think this generation is dead
deadSince time.Time // when this generation was marked dead
retries int // for cases where we're retrying
logger logger.Logger
}
func (m *mmapGeneration) Dead() bool {
m.mu.Lock()
defer m.mu.Unlock()
return m.dead
}
func (m *mmapGeneration) ID() string {
return m.id
}
func (m *mmapGeneration) Generation() int64 {
return m.generation
}
// Transaction runs an exclusive call, ensuring that the file is open if
// the *io.Writer parameter is present.
func (m *mmapGeneration) Transaction(fileP *io.Writer, fn func() error) (transactionErr error) {
m.transMu.Lock()
defer m.transMu.Unlock()
// HEY LOOK CAREFULLY AT THIS BIT:
// We can't just defer this unlock. We specifically want to be
// sure to unlock the regular mutex *before* this function is over,
// and if we error out trying to open the file, we want to do it
// even sooner. If we deferred this, the transaction would block
// *everything*, including things like sanity checks against the
// generation being Dead(), but also including the deferred
// re-close-the-file.
m.mu.Lock()
// if we've been asked for a file pointer, we need to ensure that
// our file is open, and that the file pointer to it is stored in
// the requested location, then revert that when we're done.
// if we aren't asked for a file pointer, nothing needs the file
// open.
if m.dead {
elapsed := time.Since(m.deadSince)
m.logger.Printf("WARNING: transaction against %s, which has been dead for %v\n", m.id, elapsed)
}
if fileP != nil {
if m.file == nil {
// we ignore the shouldClose response here; if this
// fragment was previously not being kept open, we're
// going to stick with that.
_, err := m.openFile()
if err != nil {
m.mu.Unlock()
return err
}
defer func() {
// report a close error if we have no other error to report
m.mu.Lock()
defer m.mu.Unlock()
err := m.closeFile()
if transactionErr == nil {
transactionErr = err
}
}()
}
var fileStash io.Writer
fileStash, *fileP = *fileP, m.file
defer func() {
*fileP = fileStash
}()
}
// We are done locking the generation itself for now.
m.mu.Unlock()
return fn()
}
// Done marks the generation done, and closes its file, but may not unmap it.
// It's still conceptually possible to end up doing a Transaction against a
// done generation, but it's a red flag.
func (m *mmapGeneration) Done() {
if m == nil {
return
}
m.mu.Lock()
defer m.mu.Unlock()
if m.dead {
oops := fmt.Sprintf("generation %s, marked done again at %v, previously marked dead at %v",
m.id, time.Now(), m.deadSince)
panic(oops)
}
m.dead = true
m.deadSince = time.Now()
err := m.closeFile()
if err != nil {
m.logger.Printf("error closing generation %s: %v", m.id, err)
}
// If we're not debugging, the finalizer won't have been enabled
// previously. Finalizers have non-zero cost, so having them not be
// created until they're needed seems rewarding?
if !generationDebug {
runtime.SetFinalizer(m, generationFinalizer)
}
endGeneration(m.id)
// note, Done() doesn't close the file; only the finalizer actually
// does the shutdown.
}
// Try to close the file if it's currently open.
func (m *mmapGeneration) closeFile() error {
var lastErr error
// report the most serious error encountered, but still close
// file even if something else failed.
if m.file != nil {
if err := m.file.Sync(); err != nil {
lastErr = fmt.Errorf("sync: %s", err)
}
if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_UN); err != nil {
lastErr = fmt.Errorf("unlock: %s", err)
}
if err := syswrap.CloseFile(m.file); err != nil {
lastErr = fmt.Errorf("close file: %s", err)
}
m.file = nil
}
return lastErr
}
// openFile ensures the file is open and locked, or fails. If it does
// open the file, it will also report the "you need to close this file
// when you're done" flag from syswrap.
func (m *mmapGeneration) openFile() (shouldClose bool, err error) {
if m.file != nil {
return false, nil
}
m.file, shouldClose, err = syswrap.OpenFile(m.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
return false, err
}
// do we actually want this in every openFile? I don't know.
if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
m.file.Close()
m.file = nil
return false, fmt.Errorf("flock: %s", err)
}
return shouldClose, nil
}
func generationFinalizer(m *mmapGeneration) {
m.mu.Lock()
if !m.dead {
m.logger.Printf("finalizing generation %s which isn't dead yet\n",
m.id)
}
m.mu.Unlock()
err := m.closeFile()
if err != nil {
m.logger.Printf("finalizing generation, closing file: %v\n", err)
}
if m.data != nil {
err := syswrap.Munmap(m.data)
if err != nil {
m.logger.Printf("finalizing generation, munmap: %v\n", err)
}
m.data = nil
}
finalizeGeneration(m.id)
}
// Cancel closes a generation out entirely. It cancels any finalizer,
// unmaps any data, ends generation tracking, and closes any files.
// It does each of these separately whether or not the others need to be done,
// or succeed. It's used to handle failures from newGeneration; it makes sure
// the generation isn't holding any resources and doesn't need to be cleaned
// up otherwise.
//
// Mostly a helper function because there's several cases where newGeneration
// might fail.
func (m *mmapGeneration) Cancel() {
if m.data != nil {
_ = syswrap.Munmap(m.data)
m.data = nil
}
err := m.closeFile()
if err != nil {
m.logger.Printf("error cancelling generation %s: %v", m.id, err)
}
runtime.SetFinalizer(m, nil)
m.dead = true
m.deadSince = time.Now()
cancelGeneration(m.id)
}
// newGeneration creates a new generation using the given file path. It
// then calls the provided setup function with the allocated storage, a
// file handle, the new generation, and a flag indicatting whether the storage
// is memory-mapped. If the setup function returns a non-nil error, the
// generation is cleaned up, and newGeneration fails. The setup function
// also returns a boolean indicating whether it used the mapping; if it
// didn't, newGeneration discards the mapping and returns a nil generation.
//
// If generationDebug is enabled, we track the generation even if no mapping
// is actually in use, so we can verify that the tracking is working.
//
// On failure, newGeneration returns nil values for generation and func,
// and an error. On success, the func returned is the close func to use
// when the generation is no longer needed by the caller.
func newGeneration(existing generation, path string, readData bool, setup func([]byte, *os.File, generation, bool) (bool, error), logger logger.Logger) (generation, error) {
m := mmapGeneration{path: path, logger: logger}
if existing != nil {
m.generation = existing.Generation() + 1
// we might keep a previous generation around just for its generation count.
if !existing.Dead() {
defer existing.Done()
}
}
shouldClose, err := m.openFile()
if err != nil {
return nil, err
}
m.id = fmt.Sprintf("%s:%d", m.path, m.generation)
// possibly assign new generation ID if this one's been used, which can
// happen with reopens, especially during testing.
m.id = registerGeneration(m.id)
// if debugging, we always want the finalizer on so we notice if a
// generation is finalized without being closed. for non-debugging
// use, we only need it when the generation is closed.
if generationDebug {
runtime.SetFinalizer(&m, generationFinalizer)
}
// Mmap the underlying file so it can be zero copied.
var mapped bool
var data []byte
fi, err := m.file.Stat()
if err == nil && fi.Size() > 0 {
data, err = syswrap.Mmap(int(m.file.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
if err == syswrap.ErrMaxMapCountReached {
// I have no idea where/how to display this message.
m.logger.Printf("maximum number of maps reached, reading file '%s' instead", m.path)
} else if err != nil {
m.Cancel()
return nil, errors.Wrap(err, "mmap failed")
} else {
mapped = true
}
}
if data == nil && readData {
data, err = ioutil.ReadAll(m.file)
if err != nil {
m.Cancel()
return nil, errors.Wrap(err, "failure file readall")
}
}
// if we got here, data's the expected data, so let's try to use it
mappedAny, err := setup(data, m.file, &m, mapped)
// if the setup failed, we unmap data if we previously mapped it,
// and exit. Note that having no data, or having only trivial
// data (like a zero-container Roaring file) isn't "failed".
if err != nil {
m.Cancel()
// Unless, that is, we think the file probably ought to
// be truncated: For instance, if a bitmap has a corrupted
// ops log, we could truncate that part of it and retry.
if err, ok := err.(roaring.FileShouldBeTruncatedError); ok && m.retries < 1 {
m.logger.Printf("file %s read partially, but should-be-truncated at %d bytes\n", m.path, err.SuggestedLength())
// close this generation, then try again. once.
m.retries++
err := os.Truncate(m.path, err.SuggestedLength())
if err != nil {
m.logger.Printf("truncating file failed [but retrying anyway]: %v\n", err)
}
return newGeneration(&m, path, readData, setup, logger)
}
return nil, err
}
if mapped {
// when generationDebug is on, we want to track this even
// if it's not being used.
if generationDebug || mappedAny {
// Advise the kernel that the mmap is accessed randomly.
// We don't care much about errors with this.
_ = madvise(data, syscall.MADV_RANDOM)
// store the data, so we can unmap it when this generation
// gets finalized.
m.data = data
} else {
// unmap the data and don't stash the pointer in this
// generation. It's not being used. This generation
// doesn't need to exist, yay.
unmapErr := syswrap.Munmap(data)
if unmapErr != nil {
m.logger.Printf("error unmapping (probably harmless): %v", unmapErr)
}
}
}
// shouldClose comes from underlying syswrap.OpenFile, which checks
// a count of open files to hint at us when we need to start closing
// files to preserve open file descriptor limit.
if shouldClose {
err := m.closeFile()
if err != nil {
m.logger.Printf("closing file to preserve open files failed: %v\n", err)
}
}
// It's possible that the generation has no actual data to track,
// because nothing's mapped, in which case there won't be any bitmap
// sources following this, just the fragment source. (Bitmaps won't
// be attached to the source unless they're actually mapped to it,
// or generationDebug is true). That's okay. We pay a tiny cost
// for the finalizer, but we also get higher confidence that it really
// does get cleaned up.
return &m, nil
}

160
generation_debug.go Normal file
View file

@ -0,0 +1,160 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build generationdebug
package pilosa
import (
"fmt"
"math/rand"
"runtime"
"sort"
"sync"
"time"
)
const generationDebug = true
type lifespan struct {
from, to, finalized time.Time
}
var knownGenerations map[string]lifespan
var knownGenerationLock sync.Mutex
var timeZero time.Time
func registerGeneration(id string) string {
knownGenerationLock.Lock()
defer knownGenerationLock.Unlock()
if knownGenerations == nil {
knownGenerations = make(map[string]lifespan)
}
newSpan := lifespan{from: time.Now()}
origId := id
// if you have more than 65k of the same file open, maybe you have bigger
// problems than this.
for span, exists := knownGenerations[id]; exists; span, exists = knownGenerations[id] {
suffix := fmt.Sprintf("::%04x", rand.Int63n(65536))
if span.finalized != timeZero {
fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v, finalized %v\n",
id, suffix, span.from, span.to, span.finalized)
} else {
if span.to != timeZero {
fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v\n", id, suffix, span.from, span.to)
} else {
fmt.Printf("new generation %s: adding %s, already exists, created %v", id, suffix, span.from)
}
}
id = origId + suffix
}
fmt.Printf("new generation %s\n", id)
knownGenerations[id] = newSpan
return id
}
func endGeneration(id string) {
knownGenerationLock.Lock()
defer knownGenerationLock.Unlock()
span, exists := knownGenerations[id]
if !exists {
oops := fmt.Sprintf("ending generation %s: unknown", id)
panic(oops)
}
if span.finalized != timeZero || span.to != timeZero {
oops := fmt.Sprintf("ending generation %s: already died at %v, finalized at %v", id, span.to, span.finalized)
panic(oops)
}
span.to = time.Now()
knownGenerations[id] = span
}
// cancelGeneration marks the generation as finalized. In principle it's
// only used in cases where we just started a generation but something
// went wrong. it's not fancier than this because of the weird cases
// where the same generation shows up again, such as when closing and
// reopening an index so we don't know about previous instances of the
// same files.
func cancelGeneration(id string) {
knownGenerationLock.Lock()
defer knownGenerationLock.Unlock()
span, exists := knownGenerations[id]
if exists {
span.finalized = time.Now()
span.to = span.finalized
knownGenerations[id] = span
}
}
func finalizeGeneration(id string) {
knownGenerationLock.Lock()
defer knownGenerationLock.Unlock()
span, exists := knownGenerations[id]
if !exists {
oops := fmt.Sprintf("finalizing generation %s: unknown", id)
panic(oops)
}
if span.finalized != timeZero {
var oops string
if span.to != timeZero {
oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, but not dead", id, span.finalized)
} else {
oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, dead at %v", id, span.finalized, span.to)
}
panic(oops)
}
span.finalized = time.Now()
knownGenerations[id] = span
}
func reportGenerations() []string {
runtime.GC()
knownGenerationLock.Lock()
defer knownGenerationLock.Unlock()
var surviving []string
times := make([]int64, 0, len(knownGenerations))
for id, span := range knownGenerations {
if span.to == timeZero {
if span.finalized == timeZero {
surviving = append(surviving, fmt.Sprintf("%s: %v, not ended or finalized", id, span.from))
} else {
surviving = append(surviving, fmt.Sprintf("%s: %v, finalized %v, not ended", id, span.from, span.finalized))
}
} else {
if span.finalized == timeZero {
surviving = append(surviving, fmt.Sprintf("%s: %v to %v, not finalized", id, span.from, span.to))
} else {
times = append(times, int64(span.finalized.Sub(span.to)))
}
}
}
if len(times) > 0 {
sort.Slice(times, func(i, j int) bool { return times[i] < times[j] })
var total int64
for _, d := range times {
total += d
}
var mean, median, p90, p99, worst int64
mean = total / int64(len(times))
median = times[len(times)/2]
p90 = times[(len(times)*9)/10]
p99 = times[(len(times)*99)/100]
worst = times[len(times)-1]
surviving = append(surviving, fmt.Sprintf("%d finalized spans. lag: mean %v, median %v, p90 %v, p99 %v, worst %v",
len(times), time.Duration(mean), time.Duration(median), time.Duration(p90), time.Duration(p99), time.Duration(worst)))
}
return surviving
}

37
generation_nodebug.go Normal file
View file

@ -0,0 +1,37 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !generationdebug
package pilosa
const generationDebug = false
func registerGeneration(id string) string {
return id
}
func endGeneration(id string) {
}
func cancelGeneration(id string) {
}
func finalizeGeneration(id string) {
}
//lint:ignore U1000 this is conditional on a build flag, see generation_test.go.
func reportGenerations() []string { //nolint:unused,deadcode
return nil
}

39
generation_test.go Normal file
View file

@ -0,0 +1,39 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// +build generationdebug
package pilosa
import (
"fmt"
"os"
"testing"
)
func examineResults() {
results := reportGenerations()
if len(results) > 0 {
fmt.Printf("generations:\n")
for _, res := range results {
fmt.Printf(" %s\n", res)
}
}
}
func TestMain(m *testing.M) {
ret := m.Run()
examineResults()
os.Exit(ret)
}

3
go.mod
View file

@ -18,6 +18,7 @@ require (
github.com/gorilla/mux v1.7.0
github.com/hashicorp/memberlist v0.1.3
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b
github.com/opentracing/opentracing-go v1.1.0
github.com/pelletier/go-toml v1.2.0
github.com/pkg/errors v0.8.1
@ -44,4 +45,4 @@ require (
modernc.org/strutil v1.0.0
)
go 1.11
go 1.13

3
go.sum
View file

@ -86,6 +86,8 @@ github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y=
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
@ -96,6 +98,7 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU=
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

4
handler.go
View file

@ -48,6 +48,10 @@ type QueryRequest struct {
// Should we profile this query?
Profile bool
// Additional data associated with the query, in cases where there's
// row-style inputs for precomputed values.
EmbeddedData []*Row
}
// QueryResponse represent a response from a processed query.

8
holder.go
View file

@ -75,7 +75,7 @@ type Holder struct {
Logger logger.Logger
snapshotQueue chan *fragment
snapshotQueue snapshotQueue
// Manages replication from the primary node.
primaryTranslateNode *Node
@ -167,7 +167,7 @@ func (h *Holder) Open() error {
// Run snapshots asynchronously. The snapshotQueue will have a background
// task associated with it which flushes it and waits until this channel
// is closed, so we should always close this channel when done.
h.snapshotQueue = newSnapshotQueue(100, 2, h.Logger)
h.snapshotQueue = newSnapshotQueue(10, 2, h.Logger)
for _, fi := range fis {
// Skip files or hidden directories.
@ -203,6 +203,7 @@ func (h *Holder) Open() error {
go func() { defer h.wg.Done(); h.monitorCacheFlush() }()
h.Stats.Open()
h.snapshotQueue.ScanHolder(h)
h.opened.Close()
return nil
@ -222,8 +223,7 @@ func (h *Holder) Close() error {
}
}
if h.snapshotQueue != nil {
close(h.snapshotQueue)
// assuming the snapshotQueueWorker has already started, this is safe.
h.snapshotQueue.Stop()
h.snapshotQueue = nil
}

19
holder_test.go
View file

@ -197,7 +197,26 @@ func TestHolder_Open(t *testing.T) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := field.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 20); err != nil {
t.Fatal(err)
}
if err := h.Reopen(); err != nil {
t.Fatalf("unexpected error: %s", err)
}
})
}
func TestHolder_HasData(t *testing.T) {

6
index.go
View file

@ -58,7 +58,7 @@ type Index struct {
Stats stats.StatsClient
logger logger.Logger
snapshotQueue chan *fragment
snapshotQueue snapshotQueue
// Used for notifying holder when a field is added.
holder *Holder
@ -462,7 +462,9 @@ func (i *Index) newField(path, name string) (*Field, error) {
f.Stats = i.Stats
f.broadcaster = i.broadcaster
f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data"))
f.snapshotQueue = i.snapshotQueue
if i.snapshotQueue != nil {
f.snapshotQueue = i.snapshotQueue
}
f.OpenTranslateStore = i.OpenTranslateStore
return f, nil
}

2850
internal/private.pb.go
View file

File diff suppressed because it is too large Load diff

2962
internal/public.pb.go
View file

File diff suppressed because it is too large Load diff

10
internal/public.proto
View file

@ -6,6 +6,12 @@ message Row {
repeated uint64 Columns = 1;
repeated string Keys = 3;
repeated Attr Attrs = 2;
bytes Roaring = 4;
}
message SignedRow {
Row Pos = 1;
Row Neg = 2;
}
message RowIdentifiers {
@ -61,6 +67,7 @@ message QueryRequest {
bool Remote = 5;
bool ExcludeRowAttrs = 6;
bool ExcludeColumns = 7;
repeated Row EmbeddedData = 8;
}
message QueryResponse {
@ -79,6 +86,7 @@ message QueryResult {
repeated uint64 RowIDs = 7;
repeated GroupCount GroupCounts = 8;
RowIdentifiers RowIdentifiers = 9;
SignedRow SignedRow = 10;
}
message ImportRequest {
@ -120,4 +128,4 @@ message ImportRoaringRequestView {
message ImportRoaringRequest {
bool Clear = 1;
repeated ImportRoaringRequestView views = 2;
}
}

268
pql/ast.go
View file

@ -17,10 +17,13 @@ package pql
import (
"bytes"
"fmt"
"reflect"
"sort"
"strconv"
"strings"
"time"
"github.com/pilosa/pilosa/v2/ext"
)
// Query represents a PQL query.
@ -259,11 +262,250 @@ type callStackElem struct {
inList bool
}
// Call represents a function call in the AST.
// Some call types may require special handling, which needs to occur
// before distributing processing to individual shards.
type CallType byte
const (
// Normal calls can be executed per shard.
PrecallNone = CallType(iota)
// PreCallGlobal indicates a call which must be run globally *before*
// distributing the call to other shards. Example: A Distinct query,
// where every shard could potentially produce results for any shard,
// so you have to produce the results up front.
PrecallGlobal
// PreCallPerNode indicates a call which needs to be run per-shard
// in a way that lets it be done on each shard, but where it should
// be done prior to spawning per-shard goroutines. Example:
// A cross-index query, where each local shard may or may not need
// to get data from a remote node, but batches of shards can
// probably be gotten from the same remote node.
PrecallPerNode
)
// Call represents a function call in the AST. The Precomputed field
// is used by the executor to handle non-standard call types; it does
// these by actually executing them separately, then replacing them
// in the call tree with a new call using the special precomputed
// type, with the Precomputed field set to a map from shards to results.
type Call struct {
Name string
Args map[string]interface{}
Children []*Call
Name string
Args map[string]interface{}
Children []*Call
Type CallType
Precomputed map[uint64]interface{}
}
// callInfo defines the arguments allowed for a particular PQL call, and
// possibly things about its semantics. If allowUnknown is true, unfamiliar
// non-reserved names are allowed on the assumption that they're field names.
// Otherwise, only those names explicitly listed are allowed. Reserved args
// (those with a leading underscore) are never allowed unless explicitly
// present.
//
// The prototypes map maps from argument names to a value. If the value is
// non-nil, the argument will be checked for type-matching. So, for instance,
// `x: 10` would indicate that x must be an int.
type callInfo struct {
allowUnknown bool
prototypes map[string]interface{}
callType CallType
}
// We want to be able to accept either a string or int64 for
// field names. Special-case type:
type stringOrInt64Type struct{}
var stringOrInt64 stringOrInt64Type
var allowUnderField = callInfo{
allowUnknown: true,
prototypes: map[string]interface{}{
"_field": "",
},
}
var allowField = callInfo{
allowUnknown: false,
prototypes: map[string]interface{}{
"field": "",
},
}
var callInfoByFunc = map[string]callInfo{
// the easy cases: things that take arbitrary inputs, because they're
// taking field=value cases
"Bitmap": {allowUnknown: true},
"Count": {allowUnknown: true},
"Row": {allowUnknown: true},
"Range": {allowUnknown: true},
// allow only "field=X" cases with string field names
"Max": allowField,
"Min": allowField,
"Sum": allowField,
// only take other calls, should never have "args"
"Difference": {allowUnknown: false},
"Intersect": {allowUnknown: false},
"Not": {allowUnknown: false},
"ClearRow": {allowUnknown: true},
"Store": {allowUnknown: true},
"MinRow": allowField,
"MaxRow": allowField,
"Rows": {
allowUnknown: false,
prototypes: map[string]interface{}{
"_field": "",
"field": "",
"limit": int64(0),
"column": nil,
"previous": nil,
"from": nil,
"to": nil,
},
},
"Shift": {allowUnknown: false,
prototypes: map[string]interface{}{
"n": int64(0),
},
},
"Union": {allowUnknown: false},
"Xor": {allowUnknown: false},
// things that take _field
"TopN": allowUnderField,
// special cases:
"Clear": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_col": stringOrInt64,
},
},
"GroupBy": {
allowUnknown: false,
prototypes: map[string]interface{}{
"filter": nil,
"limit": int64(0),
"previous": nil,
},
},
"Options": {
allowUnknown: false,
prototypes: map[string]interface{}{
"excludeRowAttrs": true,
"excludeColumns": true,
"columnAttrs": true,
"shards": nil,
},
},
"Set": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_col": stringOrInt64,
"_timestamp": "",
},
},
"Precomputed": {
allowUnknown: true,
},
"SetBit": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_col": stringOrInt64,
},
},
"SetRowAttrs": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_field": "",
"_row": stringOrInt64,
},
},
"SetColumnAttrs": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_field": "",
"_col": stringOrInt64,
},
},
"IncludesColumn": {
allowUnknown: false,
prototypes: map[string]interface{}{
"column": stringOrInt64,
},
},
}
// RegisterPluginFuncs adds arg validation for plugin funcs. Not very good
// arg validation.
func RegisterPluginFuncs(ops []ext.BitmapOp) {
for _, op := range ops {
// ignore overlap for now. This should change.
if _, ok := callInfoByFunc[op.Name]; ok {
continue
}
ci := callInfo{allowUnknown: true}
if len(op.Reserved) > 0 {
// mark these as valid/known reserved words
ci.prototypes = make(map[string]interface{})
for _, res := range op.Reserved {
ci.prototypes[res] = nil
}
}
t := op.Func.BitmapOpType()
if t.Precall == ext.OpPrecallGlobal {
ci.callType = PrecallGlobal
}
callInfoByFunc[op.Name] = ci
}
}
// CheckCallInfo tries to validate that arguments are correct and valid for the
// given call. It does not guarantee checking all possible errors; for instance,
// if an argument is a field name, CheckCallInfo can't validate that the field
// exists. It also updates with information like whether the call is expected
// to require precalling.
func (c *Call) CheckCallInfo() error {
valid, ok := callInfoByFunc[c.Name]
if !ok {
return fmt.Errorf("no arg validation for '%s'", c.Name)
}
c.Type = valid.callType
for k, v := range c.Args {
acceptable, ok := valid.prototypes[k]
if !ok && !valid.allowUnknown {
return fmt.Errorf("'%s': unknown arg '%s'", c.String(), k)
}
if !ok && strings.HasPrefix(k, "_") {
return fmt.Errorf("'%s': unknown reserved arg '%s'", c.String(), k)
}
if acceptable == nil {
continue
}
// if the types are identical, that's fine
if reflect.TypeOf(acceptable) == reflect.TypeOf(v) {
continue
}
if reflect.TypeOf(acceptable) == reflect.TypeOf(stringOrInt64) {
switch v.(type) {
case string, int64:
continue
default:
return fmt.Errorf("'%s': arg '%s' needed a string or integer value, got %T.",
c.String(), k, v)
}
}
return fmt.Errorf("'%s': arg '%s' wrong type (got %T, expected %T)",
c.String(), k, v, acceptable)
}
// call-specific checking
for _, child := range c.Children {
if err := child.CheckCallInfo(); err != nil {
return err
}
}
return nil
}
// FieldArg determines which key-value pair contains the field and rowID,
@ -283,13 +525,29 @@ func IsReservedArg(name string) bool {
return true
}
switch name {
case "from", "to":
case "from", "to", "index":
return true
default:
return false
}
}
// CallIndex handles guessing whether we've been asked to apply this to a
// different index. An empty string means "no".
func (c *Call) CallIndex() string {
if index, ok := c.Args["_index"]; ok {
if index, ok := index.(string); ok {
return index
}
}
if index, ok := c.Args["index"]; ok && index != "" {
if index, ok := index.(string); ok {
return index
}
}
return ""
}
// BoolArg is for reading the value at key from call.Args as a bool. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. The value is assumed to be a bool. An error is

8
pql/parser.go
View file

@ -82,6 +82,14 @@ func (p *parser) Parse() (*Query, error) {
panic(v)
}
}
for _, call := range p.Query.Calls {
if call == nil {
return nil, fmt.Errorf("unexpected nil Call in query's call list")
}
if err := call.CheckCallInfo(); err != nil {
return nil, err
}
}
return &p.Query, nil
}

16
pql/parser_test.go
View file

@ -75,12 +75,12 @@ func TestParser_Parse(t *testing.T) {
// Parse with only arguments.
t.Run("ArgumentsOnly", func(t *testing.T) {
q, err := pql.ParseString(`MyCall( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\" )`)
q, err := pql.ParseString(`Row( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\" )`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "MyCall",
Name: "Row",
Args: map[string]interface{}{
"key": "value",
"foo": "bar",
@ -98,12 +98,12 @@ func TestParser_Parse(t *testing.T) {
// Parse with float arguments.
t.Run("WithFloatArgs", func(t *testing.T) {
q, err := pql.ParseString(`MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9)`)
q, err := pql.ParseString(`Row( key=12.25, foo= 13.167, bar=2., baz=0.9)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "MyCall",
Name: "Row",
Args: map[string]interface{}{
"key": 12.25,
"foo": 13.167,
@ -118,12 +118,12 @@ func TestParser_Parse(t *testing.T) {
// Parse with float arguments.
t.Run("WithNegativeArgs", func(t *testing.T) {
q, err := pql.ParseString(`MyCall( key=-12.25, foo= -13)`)
q, err := pql.ParseString(`Row( key=-12.25, foo= -13)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "MyCall",
Name: "Row",
Args: map[string]interface{}{
"key": -12.25,
"foo": int64(-13),
@ -173,12 +173,12 @@ func TestParser_Parse(t *testing.T) {
// Parse with condition arguments.
t.Run("WithCondition", func(t *testing.T) {
q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "MyCall",
Name: "Row",
Args: map[string]interface{}{
"key": "foo",
"x": &pql.Condition{Op: pql.EQ, Value: 12.25},

20
pql/pqlpeg_test.go
View file

@ -46,7 +46,7 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9
t.Fatalf("Failed, got: %s", q)
}
_, err = ParseString("C(a=falsen0)")
_, err = ParseString("Row(a=falsen0)")
if err != nil {
t.Fatalf("falsen0 should have been parsed as a string")
}
@ -109,15 +109,15 @@ func TestPEGWorking(t *testing.T) {
ncalls: 2},
{
name: "SetWithArbCall",
input: "Set(1, a=4)Blerg(z=ha)",
input: "Set(1, a=4)Row(z=ha)",
ncalls: 2},
{
name: "SetArbSet",
input: "Set(1, a=4)Blerg(z=ha)Set(2, z=99)",
input: "Set(1, a=4)Row(z=ha)Set(2, z=99)",
ncalls: 3},
{
name: "ArbSetArb",
input: "Arb(q=1, a=4)Set(1, z=9)Arb(z=99)",
input: "Row(q=1, a=4)Set(1, z=9)Row(z=99)",
ncalls: 3},
{
name: "SetStringArg",
@ -161,11 +161,11 @@ func TestPEGWorking(t *testing.T) {
ncalls: 1},
{
name: "double quoted args",
input: `B(a="zm''e")`,
input: `Row(a="zm''e")`,
ncalls: 1},
{
name: "single quoted args",
input: `B(a='zm""e')`,
input: `Row(a='zm""e')`,
ncalls: 1},
{
name: "SetRowAttrs",
@ -320,7 +320,7 @@ func TestPEGErrors(t *testing.T) {
input: "Set(, 1, a=4)"},
{
name: "StartinCommaArb",
input: "Zeeb(, a=4)"},
input: "Row(, a=4)"},
{
name: "SetRowAttrs0args",
input: "SetRowAttrs(blah, 9)"},
@ -585,11 +585,11 @@ func TestPQLDeepEquality(t *testing.T) {
}},
{
name: "Weird dash",
call: "Sum(field-=f)",
call: "Count(dashy-=f)",
exp: &Call{
Name: "Sum",
Name: "Count",
Args: map[string]interface{}{
"field-": "f",
"dashy-": "f",
},
}},
{

42
roaring/container_stash.go
View file

@ -267,9 +267,6 @@ func (c *Container) Freeze() *Container {
if c.flags&flagFrozen != 0 {
return c
}
// unmapOrClone should unmap-in-place because the existing
// container isn't frozen (or we'd already have returned it).
c = c.unmapOrClone()
c.flags |= flagFrozen
return c
}
@ -419,6 +416,45 @@ func (c *Container) bitmap() []uint64 {
return *(*[]uint64)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(c.pointer)), Len: int(c.len), Cap: int(c.cap)}))
}
// AsBitmap yields a 65k-bit bitmap, storing it in the target if a target
// is provided. The target should be zeroed, or this becomes an implicit
// union.
func (c *Container) AsBitmap(target []uint64) (out []uint64) {
if c.typeID == containerBitmap {
return c.bitmap()
}
// Reminder: len(nil) == 0.
if len(target) < 1024 {
out = make([]uint64, 1024)
} else {
out = target
}
if c.typeID == containerArray {
a := c.array()
for _, v := range a {
out[v/64] |= 1 << (v % 64)
}
return out
}
if c.typeID == containerRun {
runs := c.runs()
for _, r := range runs {
splatRun(out, r)
}
return out
}
// in theory this shouldn't happen?
return out
}
func splatRun(into []uint64, from interval16) {
// TODO this can be ~64x faster for long runs by setting maxBitmap instead of single bits
//note v must be int or will overflow
for v := int(from.start); v <= int(from.last); v++ {
into[v/64] |= (uint64(1) << uint(v%64))
}
}
// setBitmap stores a set of uint64s as data.
func (c *Container) setBitmap(bitmap []uint64) {
if c == nil || c.frozen() {

19
roaring/generation_debug.go Normal file
View file

@ -0,0 +1,19 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build generationdebug
package roaring
const generationDebug = true

19
roaring/generation_nodebug.go Normal file
View file

@ -0,0 +1,19 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !generationdebug
package roaring
const generationDebug = false

286
roaring/roaring.go
View file

@ -21,6 +21,7 @@ import (
"hash/fnv"
"io"
"math/bits"
"reflect"
"sort"
"unsafe"
@ -77,6 +78,46 @@ var containerTypeNames = map[byte]string{
var fullContainer = NewContainerRun([]interval16{{start: 0, last: maxContainerVal}}).Freeze()
// AdvisoryError is used for the special case where we probably want to *report*
// an error reading a file, but don't want to actually count the file as not
// being read. For instance, a partial ops-log entry is *probably* harmless;
// we probably crashed while writing (?) and as such didn't report the write
// as successful. We hope.
type AdvisoryError interface {
error
AdvisoryOnly()
}
type advisoryError struct {
e error
}
func (a advisoryError) Error() string {
return a.e.Error()
}
// This marks the error as safe to ignore.
func (a advisoryError) AdvisoryOnly() {
}
type FileShouldBeTruncatedError interface {
AdvisoryError
SuggestedLength() int64
}
type fileShouldBeTruncatedError struct {
advisoryError
offset int64
}
func (f *fileShouldBeTruncatedError) SuggestedLength() int64 {
return f.offset
}
func newFileShouldBeTruncatedError(err error, offset int64) *fileShouldBeTruncatedError {
return &fileShouldBeTruncatedError{advisoryError: advisoryError{e: err}, offset: offset}
}
type Containers interface {
// Get returns nil if the key does not exist.
Get(key uint64) *Container
@ -144,6 +185,7 @@ type ContainerIterator interface {
// Bitmap represents a roaring bitmap.
type Bitmap struct {
Containers Containers
Source Source
// User-defined flags.
Flags byte
@ -218,6 +260,7 @@ func (b *Bitmap) Freeze() *Bitmap {
// Create a copy of the bitmap structure.
other := &Bitmap{
Containers: b.Containers.Freeze(),
Source: b.Source,
}
return other
@ -391,6 +434,13 @@ func (b *Bitmap) Min() (uint64, bool) {
return v, !eof
}
// MinAt returns the lowest value in the bitmap at least equal to its argument.
// Second return value is true if containers exist in the bitmap.
func (b *Bitmap) MinAt(start uint64) (uint64, bool) {
v, eof := b.IteratorAt(start).Next()
return v, !eof
}
// Max returns the highest value in the bitmap.
// Returns zero if the bitmap is empty.
func (b *Bitmap) Max() uint64 {
@ -549,13 +599,21 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap {
hi0, hi1 := highbits(start), highbits(end)
citer, _ := b.Containers.Iterator(hi0)
other := NewSliceBitmap()
mappedAny := false
for citer.Next() {
k, c := citer.Value()
if k >= hi1 {
break
}
if c.Mapped() {
mappedAny = true
}
other.Containers.Put(off+(k-hi0), c.Freeze())
}
// if b.Source != nil && mappedAny {
if b.Source != nil && (generationDebug || mappedAny) {
other.Source = b.Source
}
return other
}
@ -594,6 +652,7 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
// Intersect returns the intersection of b and other.
func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
output := NewBitmap()
usedB, usedOther := false, false
iiter, _ := b.Containers.Iterator(0)
jiter, _ := other.Containers.Iterator(0)
i, j := iiter.Next(), jiter.Next()
@ -607,12 +666,27 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
j = jiter.Next()
kj, cj = jiter.Value()
} else { // ki == kj
output.Containers.Put(ki, intersect(ci, cj))
newC := intersect(ci, cj)
if newC == ci {
usedB = true
}
if newC == cj {
usedOther = true
}
output.Containers.Put(ki, newC)
i, j = iiter.Next(), jiter.Next()
ki, ci = iiter.Value()
kj, cj = jiter.Value()
}
}
switch {
case usedB && usedOther:
output.Source = MergeSources(b.Source, other.Source)
case usedB:
output.Source = b.Source
case usedOther:
output.Source = other.Source
}
return output
}
@ -640,25 +714,43 @@ func (b *Bitmap) UnionInPlace(others ...*Bitmap) {
func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) {
iiter, _ := b.Containers.Iterator(0)
jiter, _ := other.Containers.Iterator(0)
usedB, usedOther := false, false
i, j := iiter.Next(), jiter.Next()
ki, ci := iiter.Value()
kj, cj := jiter.Value()
for i || j {
if i && (!j || ki < kj) {
target.Containers.Put(ki, ci.Freeze())
usedB = true
i = iiter.Next()
ki, ci = iiter.Value()
} else if j && (!i || ki > kj) {
target.Containers.Put(kj, cj.Freeze())
usedOther = true
j = jiter.Next()
kj, cj = jiter.Value()
} else { // ki == kj
target.Containers.Put(ki, union(ci, cj))
newC := union(ci, cj)
target.Containers.Put(ki, newC)
if newC == ci {
usedB = true
}
if newC == cj {
usedOther = true
}
i, j = iiter.Next(), jiter.Next()
ki, ci = iiter.Value()
kj, cj = jiter.Value()
}
}
switch {
case usedB && usedOther:
target.Source = MergeSources(b.Source, other.Source)
case usedB:
target.Source = b.Source
case usedOther:
target.Source = other.Source
}
}
// unionInPlace stores the union of b and others into b. The others will
@ -752,7 +844,14 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) {
bitmapIters = make(handledIters, 0, requiredSliceSize)
}
var sources []Source
if b.Source != nil {
sources = append(sources, b.Source)
}
for _, other := range others {
if other.Source != nil {
sources = append(sources, other.Source)
}
otherIter, _ := other.Containers.Iterator(0)
if otherIter.Next() {
bitmapIters = append(bitmapIters, handledIter{
@ -762,6 +861,8 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) {
})
}
}
// new bitmap might have containers from any of those bitmaps in it
b.Source = MergeSources(sources...)
// Loop until we've exhausted every iter.
hasNext := true
@ -890,6 +991,7 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) {
// Difference returns the difference of b and other.
func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
output := NewBitmap()
output.Source = b.Source
iiter, _ := b.Containers.Iterator(0)
jiter, _ := other.Containers.Iterator(0)
@ -917,6 +1019,9 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
// Xor returns the bitwise exclusive or of b and other.
func (b *Bitmap) Xor(other *Bitmap) *Bitmap {
output := NewBitmap()
// Xor can end up with containers from either parent if the other
// had no container or an empty container.
output.Source = MergeSources(b.Source, other.Source)
iiter, _ := b.Containers.Iterator(0)
jiter, _ := other.Containers.Iterator(0)
@ -1433,7 +1538,11 @@ func (b *Bitmap) RemapRoaringStorage(data []byte) (mappedAny bool, returnErr err
var itrPointer *uint16
var itrErr error
if data != nil {
// If we got no data, we don't want to do the actual mapping, just
// the unmapping. If preferMapping is false, we also don't want to
// map to the data. We still need to do the UpdateEvery loop, we
// just won't have an iterator for it.
if data != nil && b.preferMapping {
itr, err = newRoaringIterator(data)
}
// don't return early: we still have to do the unmapping
@ -1617,6 +1726,12 @@ func (b *Bitmap) Iterator() *Iterator {
return itr
}
func (b *Bitmap) IteratorAt(start uint64) *Iterator {
itr := &Iterator{bitmap: b}
itr.Seek(start)
return itr
}
// Ops returns the number of write ops the bitmap is aware of in its ops
// log, and their total bit count.
func (b *Bitmap) Ops() (ops int, opN int) {
@ -1629,6 +1744,167 @@ func (b *Bitmap) SetOps(ops int, opN int) {
b.ops, b.opN = ops, opN
}
// RoaringToBitmaps yields a series of bitmaps with specified shard
// keys, based on a single roaring file, with splits at multiples of
// shardWidth, which should be a multiple of container size.
func RoaringToBitmaps(data []byte, shardWidth uint64) ([]*Bitmap, []uint64) {
if data == nil {
return nil, nil
}
var itr roaringIterator
var itrKey uint64
var itrCType byte
var itrN int
var itrLen int
var itrPointer *uint16
var itrErr error
currentShard := ^uint64(0)
var currentBitmap *Bitmap
var bitmaps []*Bitmap
var shards []uint64
keysPerShard := shardWidth >> 16
itr, err := newRoaringIterator(data)
if err != nil || itr == nil {
return nil, nil
}
itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next()
for itrErr == nil {
newC := &Container{
typeID: itrCType,
n: int32(itrN),
len: int32(itrLen),
cap: int32(itrLen),
pointer: itrPointer,
flags: flagMapped,
}
shard := itrKey / keysPerShard
if shard != currentShard {
if currentBitmap != nil {
bitmaps = append(bitmaps, currentBitmap)
shards = append(shards, currentShard)
}
currentBitmap = NewFileBitmap()
currentShard = shard
}
currentBitmap.Containers.Put(itrKey, newC)
itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next()
}
if currentBitmap != nil {
bitmaps = append(bitmaps, currentBitmap)
shards = append(shards, currentShard)
}
// we don't support ops logs for this
return bitmaps, shards
}
// BitmapsToRoaring renders a series of non-overlapping bitmaps as a
// unified roaring file.
func BitmapsToRoaring(bitmaps []*Bitmap) []byte {
count := int64(0)
size := int64(0)
for i, bm := range bitmaps {
c, s := bm.roaringSize()
// skip this bitmap during the next pass, since it's empty
if c == 0 {
bitmaps[i] = nil
continue
}
count += c
size += s
}
if count == 0 {
return nil
}
// we have count headers, which need 12 bytes, plus a magic number,
// plus offsets (4 bytes per container), plus size bytes of data to
// write.
out := make([]byte, headerBaseSize+(12*count)+(4*count)+size)
binary.LittleEndian.PutUint16(out[0:2], uint16(MagicNumber))
out[3] = byte(storageVersion)
binary.LittleEndian.PutUint32(out[4:8], uint32(count))
headerEnd := 8 + (12 * count)
offsetEnd := headerEnd + (4 * count)
headers := out[8:headerEnd]
offsets := out[headerEnd:offsetEnd]
data := out[offsetEnd:]
headerOffset := 0
offsetOffset := 0
dataOffset := 0
prevKey := uint64(0)
for _, bm := range bitmaps {
if bm == nil {
continue
}
citer, _ := bm.Containers.Iterator(0)
for citer.Next() {
k, c := citer.Value()
n := c.N()
if n == 0 {
continue
}
if roaringParanoia {
if k < prevKey {
panic("unsorted keys in multiple-bitmap roaring conversion")
}
}
// place header at header offset, and data at data
// offset
header := headers[headerOffset : headerOffset+12]
offset := offsets[offsetOffset : offsetOffset+4]
headerOffset += 12
offsetOffset += 4
binary.LittleEndian.PutUint64(header[0:8], k)
binary.LittleEndian.PutUint16(header[8:10], uint16(c.typeID))
binary.LittleEndian.PutUint16(header[10:12], uint16(n-1))
binary.LittleEndian.PutUint32(offset[0:4], uint32(dataOffset+int(offsetEnd)))
nextData := data[dataOffset:]
switch c.typeID {
case containerArray:
asUint16 := *(*[]uint16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&nextData[0])), Len: int(c.len), Cap: int(c.len)}))
copy(asUint16, c.array())
dataOffset += 2 * int(c.len)
case containerBitmap:
asUint64 := *(*[]uint64)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&nextData[0])), Len: 1024, Cap: 1024}))
copy(asUint64, c.bitmap())
dataOffset += 8192
case containerRun:
asInterval16 := *(*[]interval16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&nextData[2])), Len: int(c.len), Cap: int(c.len)}))
copy(asInterval16, c.runs())
binary.LittleEndian.PutUint16(nextData[0:2], uint16(c.len))
dataOffset += int(4*c.len) + 2
}
}
}
return out
}
// roaringSize yields the count of non-empty containers, and the size
// of the storage *only* -- not the headers.
func (b *Bitmap) roaringSize() (int64, int64) {
count := int64(0)
size := int64(0)
citer, _ := b.Containers.Iterator(0)
for citer.Next() {
_, c := citer.Value()
if c.N() == 0 {
continue
}
count++
switch c.typeID {
case containerArray:
size += 2 * int64(c.N())
case containerBitmap:
size += 8192
case containerRun:
// 2 bytes for the count of runs, plus 4 bytes per run
size += 2 + (4 * int64(c.len))
}
}
return count, size
}
// Info returns stats for the bitmap.
func (b *Bitmap) Info() bitmapInfo {
info := bitmapInfo{
@ -3271,9 +3547,9 @@ func intersectRunRun(a, b *Container) *Container {
output.setN(n)
runs := output.runs()
if n < ArrayMaxSize && int32(len(runs)) > n/2 {
output.runToArray()
output = output.runToArray()
} else if len(runs) > runMaxSize {
output.runToBitmap()
output = output.runToBitmap()
}
return output
}

98
roaring/source.go Normal file
View file

@ -0,0 +1,98 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package roaring
import (
"strings"
)
// A Source represents the source a given bitmap gets its data from,
// such as a memory-mapped file. When combining bitmaps, we might
// track them together in a single combined-source of some sort.
type Source interface {
ID() string
Dead() bool
}
// MergeSources combines sources. If you have two bitmaps, and you're
// combining them, then the combination's source is a combination of
// those two sources.
func MergeSources(sources ...Source) Source {
sourceCount := 0
totalCount := 0
var lastSource Source
for _, s := range sources {
if s == nil {
continue
}
lastSource = s
if s, ok := s.(combinedSource); ok {
sourceCount++
totalCount += len(s)
} else {
sourceCount++
totalCount++
}
}
// if there's no sources (this includes all sources being
// empty combinedSources), we don't have a source.
if totalCount == 0 {
return nil
}
// if there's exactly one source, combined or otherwise, that's
// fine, we'll just return it.
if sourceCount == 1 {
return lastSource
}
// make a new combinedSource, flattening any combinedSources
// already present.
newSources := make([]Source, 0, totalCount)
for _, s := range sources {
if s == nil {
continue
}
if s, ok := s.(combinedSource); ok {
newSources = append(newSources, s...)
} else {
newSources = append(newSources, s)
}
}
return combinedSource(newSources)
}
// SetSource tells the bitmap what source to associate with new things it
// creates. This is possibly logically incorrect.
func (b *Bitmap) SetSource(s Source) {
b.Source = s
}
type combinedSource []Source
func (c combinedSource) ID() string {
ids := make([]string, len(c))
for i := range c {
ids[i] = c[i].ID()
}
return strings.Join(ids, ",")
}
func (c combinedSource) Dead() bool {
for i := range c {
if c[i].Dead() {
return true
}
}
return false
}

10
roaring/unmarshal_binary.go
View file

@ -30,7 +30,9 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
return nil
}
statsHit("Bitmap/UnmarshalBinary")
b.opN = 0 // reset opN since we're reading new data.
// reset ops/opN since we're reading new data.
b.ops = 0
b.opN = 0
fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
if fileMagic == MagicNumber { // if pilosa roaring
return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring")
@ -205,15 +207,15 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
// Unmarshal the op and apply it.
var opr op
if err := opr.UnmarshalBinary(buf); err != nil {
// FIXME(benbjohnson): return error with position so file can be trimmed.
return err
return newFileShouldBeTruncatedError(err, int64(opsOffset))
}
opr.apply(b)
// Increase the op count.
b.ops++
b.opN += opr.count()
opsOffset += opr.size()
// Move the buffer forward.
buf = buf[opr.size():]
buf = data[opsOffset:]
}
return nil

178
row.go
View file

@ -18,6 +18,7 @@ import (
"encoding/json"
"sort"
"github.com/pilosa/pilosa/v2/ext"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)
@ -43,6 +44,55 @@ func NewRow(columns ...uint64) *Row {
return r
}
// NewRowFromBitmap divides a bitmap into rows, which it now calls shards. This
// transposes; data that was in any shard for Row 0 is now considered shard 0,
// etcetera.
func NewRowFromBitmap(b *roaring.Bitmap) *Row {
r := &Row{}
if b == nil {
return r
}
rowNum := uint64(0)
for col, ok := b.MinAt(rowNum * ShardWidth); ok; col, ok = b.MinAt(rowNum * ShardWidth) {
rowNum = col / ShardWidth
seg := rowSegment{
shard: rowNum,
data: b.OffsetRange(rowNum*ShardWidth, rowNum*ShardWidth, (rowNum+1)*ShardWidth),
writable: true,
}
seg.n = seg.data.Count()
r.segments = append(r.segments, seg)
rowNum++
}
return r
}
// NewRowFromRoaring parses a roaring data file as a row, dividing it into
// bitmaps and rowSegments based on shard width.
func NewRowFromRoaring(data []byte) *Row {
bitmaps, shards := roaring.RoaringToBitmaps(data, ShardWidth)
r := &Row{segments: make([]rowSegment, len(bitmaps))}
for i := range bitmaps {
segment := rowSegment{
shard: shards[i],
data: bitmaps[i],
writable: false,
n: bitmaps[i].Count(),
}
r.segments[i] = segment
}
return r
}
// Roaring returns the row treated as a unified roaring bitmap.
func (r *Row) Roaring() []byte {
bitmaps := make([]*roaring.Bitmap, len(r.segments))
for i := range r.segments {
bitmaps[i] = r.segments[i].data
}
return roaring.BitmapsToRoaring(bitmaps)
}
// IsEmpty returns true if the row doesn't contain any set bits.
func (r *Row) IsEmpty() bool {
if len(r.segments) == 0 {
@ -194,6 +244,65 @@ func (r *Row) Union(others ...*Row) *Row {
return &Row{segments: output}
}
// GenericBinaryOp returns the output of a generic op on r and other.
func (r *Row) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *Row, args map[string]interface{}) *Row {
var segments []rowSegment
itr := newMergeSegmentIterator(r.segments, other.segments)
for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
if s1 == nil {
segments = append(segments, *s0)
continue
} else if s0 == nil {
segments = append(segments, *s1)
continue
}
segments = append(segments, *s0.GenericBinaryOp(op, s1, args))
}
return &Row{segments: segments}
}
// GenericNaryOp returns the output of an nary op on r and others.
func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args map[string]interface{}) *Row {
segments := make([][]rowSegment, 0, len(others)+1)
if len(r.segments) > 0 {
segments = append(segments, r.segments)
}
nextSegs := make([][]rowSegment, 0, len(others)+1)
toProcess := make([]*rowSegment, 0, len(others)+1)
var output []rowSegment
for _, other := range others {
if len(other.segments) > 0 {
segments = append(segments, other.segments)
}
}
for len(segments) > 0 {
shard := segments[0][0].shard
for _, segs := range segments {
if segs[0].shard < shard {
shard = segs[0].shard
}
}
nextSegs = nextSegs[:0]
toProcess := toProcess[:0]
for _, segs := range segments {
if segs[0].shard == shard {
toProcess = append(toProcess, &segs[0])
segs = segs[1:]
}
if len(segs) > 0 {
nextSegs = append(nextSegs, segs)
}
}
// at this point, "toProcess" is a list of all the segments
// sharing the lowest ID, and nextSegs is a list of all the others.
// Swap the segment lists (so we don't have to reallocate it)
segments, nextSegs = nextSegs, segments
output = append(output, *toProcess[0].GenericNaryOp(op, toProcess[1:], args))
}
return &Row{segments: output}
}
// Difference returns the diff of r and other.
func (r *Row) Difference(other *Row) *Row {
var segments []rowSegment
@ -212,6 +321,17 @@ func (r *Row) Difference(other *Row) *Row {
return &Row{segments: segments}
}
// GenericUnary returns the results of a generic op on r.
func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row {
work := r
var segments []rowSegment
for _, segment := range work.segments {
opped := segment.GenericUnaryOp(op, args)
segments = append(segments, *opped)
}
return &Row{segments: segments}
}
// Shift returns the bitwise shift of r by n bits.
// Currently only positive shift values are supported.
func (r *Row) Shift(n int64) (*Row, error) {
@ -299,6 +419,15 @@ func (r *Row) Count() uint64 {
return n
}
// GenericCount applies an op to lots of things.
func (r *Row) GenericCount(op ext.BitmapOpUnaryCount, args map[string]interface{}) uint64 {
var n int64
for i := range r.segments {
n += op([]ext.Bitmap{WrapBitmap(r.segments[i].data)}, args)
}
return uint64(n)
}
// MarshalJSON returns a JSON-encoded byte slice of r.
func (r *Row) MarshalJSON() ([]byte, error) {
var o struct {
@ -359,7 +488,7 @@ type rowSegment struct {
}
func (s *rowSegment) Freeze() {
s.data.Freeze()
s.data = s.data.Freeze()
}
/*
@ -392,7 +521,7 @@ func (s *rowSegment) IntersectionCount(other *rowSegment) uint64 {
// Intersect returns the itersection of s and other.
func (s *rowSegment) Intersect(other *rowSegment) *rowSegment {
data := s.data.Intersect(other.data)
data.Freeze()
data = data.Freeze()
return &rowSegment{
data: data,
@ -419,10 +548,37 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment {
}
}
// GenericOp performs a generic op on s and other
func (s *rowSegment) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *rowSegment, args map[string]interface{}) *rowSegment {
data := op([]ext.Bitmap{WrapBitmap(s.data), WrapBitmap(other.data)}, args)
return &rowSegment{
data: UnwrapBitmap(data),
shard: s.shard,
n: data.Count(),
}
}
// GenericOp performs a generic op on s and others
func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSegment, args map[string]interface{}) *rowSegment {
bitmaps := make([]ext.Bitmap, len(others)+1)
bitmaps[0] = WrapBitmap(s.data)
for i, seg := range others {
bitmaps[i+1] = WrapBitmap(seg.data)
}
data := op(bitmaps, args)
return &rowSegment{
data: UnwrapBitmap(data),
shard: s.shard,
n: data.Count(),
}
}
// Difference returns the diff of s and other.
func (s *rowSegment) Difference(other *rowSegment) *rowSegment {
data := s.data.Difference(other.data)
data.Freeze()
data = data.Freeze()
return &rowSegment{
data: data,
@ -435,7 +591,7 @@ func (s *rowSegment) Difference(other *rowSegment) *rowSegment {
// Xor returns the xor of s and other.
func (s *rowSegment) Xor(other *rowSegment) *rowSegment {
data := s.data.Xor(other.data)
data.Freeze()
data = data.Freeze()
return &rowSegment{
data: data,
@ -452,7 +608,7 @@ func (s *rowSegment) Shift() (*rowSegment, error) {
if err != nil {
return nil, errors.Wrap(err, "shifting roaring data")
}
data.Freeze()
data = data.Freeze()
return &rowSegment{
data: data,
@ -462,6 +618,18 @@ func (s *rowSegment) Shift() (*rowSegment, error) {
}, nil
}
// GenericUnary returns s subject to op.
func (s *rowSegment) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *rowSegment {
//TODO deal with overflow
data := UnwrapBitmap(op([]ext.Bitmap{WrapBitmap(s.data)}, args))
return &rowSegment{
data: data,
shard: s.shard,
n: data.Count(),
}
}
// SetBit sets the i-th column of the row.
func (s *rowSegment) SetBit(i uint64) (changed bool) {
s.ensureWritable()

120
server.go
View file

@ -17,17 +17,21 @@ package pilosa
import (
"context"
"fmt"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"plugin"
"runtime"
"strconv"
"strings"
"sync"
"time"
"github.com/pilosa/pilosa/v2/ext"
"github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
"github.com/pkg/errors"
@ -57,6 +61,8 @@ type Server struct { // nolint: maligned
hosts []string
clusterDisabled bool
serializer Serializer
extensionPath string
extensions []*ext.ExtensionInfo
// External
systemInfo SystemInfo
@ -335,6 +341,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
if err != nil {
return nil, err
}
s.extensionPath = filepath.Join(path, ".extensions")
s.holder.Path = path
// s.holder.translateFile.Path = filepath.Join(path, ".keys")
@ -376,6 +383,30 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.cluster.broadcaster = s
s.cluster.maxWritesPerRequest = s.maxWritesPerRequest
s.holder.broadcaster = s
err = s.loadPlugins()
if err != nil {
s.logger.Printf("not all plugins loaded successfully")
}
if len(s.extensions) > 0 {
s.logger.Printf("loaded extensions:")
for _, ext := range s.extensions {
if ext == nil {
s.logger.Printf(" inexplicably, a nil extension?!?")
continue
}
s.logger.Printf(" %s %s: %s", ext.Name, ext.Version, ext.Description)
if ext.License != "" {
s.logger.Printf(" License: %s", ext.License)
}
if len(ext.BitmapOps) > 0 {
opList := make([]string, len(ext.BitmapOps))
for i := range ext.BitmapOps {
opList[i] = ext.BitmapOps[i].Name
}
s.logger.Printf(" Ops: %s", strings.Join(opList, ", "))
}
}
}
err = s.cluster.setup()
if err != nil {
@ -389,6 +420,95 @@ func (s *Server) InternalClient() InternalClient {
return s.defaultClient
}
func (s *Server) loadPlugins() error {
var anyError error
dir, err := os.Open(s.extensionPath)
if err != nil {
// don't complain about it not existing, that's fine.
if os.IsNotExist(err) {
s.logger.Printf("extension interface v0: no extensions directory.")
return nil
}
return errors.Wrap(err, "opening extension path:")
}
defer dir.Close()
for files, err := dir.Readdir(64); err != io.EOF; files, err = dir.Readdir(64) {
if err != nil {
return errors.Wrap(err, "searching extension directory:")
}
for _, file := range files {
name := file.Name()
// only .so files are likely plugins.
if !strings.HasSuffix(name, ".so") {
continue
}
// only regular files are candidates for loading.
mode := file.Mode()
if !mode.IsRegular() {
s.logger.Printf("extension file '%s' is not a regular file", name)
continue
}
err = s.loadPlugin(name)
if err != nil {
s.logger.Printf("loading extension %s: %v", name, err)
anyError = err
}
}
}
return anyError
}
func (s *Server) loadPlugin(name string) error {
path := filepath.Join(s.extensionPath, name)
p, err := plugin.Open(path)
if err != nil {
return err
}
pluginExtInfo, err := p.Lookup("ExtensionInfo")
if err != nil {
return fmt.Errorf("%s: no ExtensionInfo found", name)
}
extInfoFunc, ok := pluginExtInfo.(func(string) (*ext.ExtensionInfo, error))
if !ok {
return fmt.Errorf("%s: unexpected %T instead of ExtensionInfo object", name, pluginExtInfo)
}
extInfo, err := extInfoFunc("v0")
if err != nil {
return errors.Wrap(err, name)
}
if extInfo == nil {
return fmt.Errorf("%s: nil ExtensionInfo", name)
}
if extInfo.ExtensionAPI != "v0" {
return fmt.Errorf("%s: unsupported extension API %s", name, extInfo.ExtensionAPI)
}
s.extensions = append(s.extensions, extInfo)
bitmapOps := extInfo.BitmapOps
bmOps, countOps, fieldOps, unknownOps := 0, 0, 0, 0
for i := range bitmapOps {
// title-case the name
bitmapOps[i].Name = strings.Title(bitmapOps[i].Name)
typ := bitmapOps[i].Func.BitmapOpType()
switch {
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:
countOps++
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap:
bmOps++
case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap:
fieldOps++
default:
unknownOps++
}
}
err = s.executor.registerOps(bitmapOps)
if err != nil {
s.logger.Printf("warning: extension registration failed: %v", err)
} else {
pql.RegisterPluginFuncs(bitmapOps)
}
return nil
}
// UpAndDown brings the server up minimally and shuts it down
// again; basically, it exists for testing holder open and close.
func (s *Server) UpAndDown() error {

7
server/grpc.go
View file

@ -537,6 +537,13 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(r)}},
}}
case bool:
ci := []*pb.ColumnInfo{{Name: "changed", Datatype: "bool"}}
results <- &pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_BoolVal{BoolVal: r}},
}}
default:
logger.Printf("unhandled %T\n", r)
breakLoop = true

15
server/grpc_internal_test.go
View file

@ -205,6 +205,16 @@ func TestGRPC(t *testing.T) {
{uint64(123)},
},
},
// bool
{
true,
[]expHeader{
{"changed", "bool"},
},
[][]expColumn{
{true},
},
},
}
logger := logger.NopLogger
@ -244,6 +254,11 @@ func TestGRPC(t *testing.T) {
if val != v {
t.Fatalf("test %d expected column val: %v, but got: %v", ti, v, val)
}
case bool:
val := column.GetBoolVal()
if val != v {
t.Fatalf("test %d expected column val: %v, but got: %v", ti, v, val)
}
default:
t.Fatalf("test %d has unhandled data type: %T", ti, v)
}

405
snapshotqueue.go Normal file
View file

@ -0,0 +1,405 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"os"
"sync"
"time"
"github.com/pilosa/pilosa/v2/logger"
"github.com/pkg/errors"
)
// snapshotQueue is a thing which can handle enqueuing snapshots. A snapshot
// queue distinguishes between high-priority requests, which get satisfied
// by the next available worker, and regular requests, which get enqueued
// if there's space in the queue, and otherwise dropped. There's also a
// separate background task to scan a holder for fragments which may need
// snapshots, but which is processed only when the queue is empty, and only
// slowly. "Await" awaits an existing snapshot if one is already enqueued.
// "Immediate" tries to do one right away. (If one's already enqueued, this
// can leave it in the queue, which will ignore anything that shows up with
// the request flag cleared.)
//
// Await, Enqueue, and Immediate should be called only with the fragment lock
// held.
//
// ScanHolder spawns a new goroutine. You don't need to use `go` on it.
type snapshotQueue interface {
Immediate(*fragment) error
Enqueue(*fragment)
Await(*fragment) error
ScanHolder(*Holder)
Stop()
}
// queuelessSnapshotQueue isn't a snapshot queue, but it satisfies the
// interface.
type queuelessSnapshotQueue struct{}
func (q *queuelessSnapshotQueue) Enqueue(f *fragment) {
_ = f.snapshot()
}
func (q *queuelessSnapshotQueue) Await(f *fragment) error {
return nil
}
func (q *queuelessSnapshotQueue) Immediate(f *fragment) error {
return f.snapshot()
}
func (q *queuelessSnapshotQueue) ScanHolder(h *Holder) {
}
func (q *queuelessSnapshotQueue) Stop() {
}
// defaultSnapshotQueue is the fallback to use if none is available,
// and currently uses queueless -- it runs all snapshots immediately.
var defaultSnapshotQueue *queuelessSnapshotQueue
// newSnapshotQueue makes a new snapshot queue, of depth N, with
// w worker threads.
func newSnapshotQueue(n int, w int, l logger.Logger) snapshotQueue {
sq := prioritySnapshotQueue{normal: make(chan snapshotRequest, n), urgent: make(chan snapshotRequest), background: make(chan snapshotRequest), done: make(chan struct{}), logger: l}
if sq.logger == nil {
sq.logger = logger.NewStandardLogger(os.Stderr)
}
sq.spawnWorkers(w)
return &sq
}
type snapshotRequest struct {
frag *fragment
when time.Time
}
// prioritySnapshotQueue gives preference to "immediate" requests, and
// dispreference to "background" requests from ScanHolder. It timestamps
// requests, so it can discard a request if the most recent snapshot is
// newer than the request. The snapshotPending flag in the fragment is
// used to track that a given fragment thinks it has been successfully
// enqueued. Background requests are not considered enqueued, since
// they'll never get processed if there's anything else. In normal workloads,
// immediate/urgent snapshots should be rare, but we'll happily drop
// most requests on the floor; the scanner should pick them up once things
// are quiet.
type prioritySnapshotQueue struct {
logger logger.Logger
urgent chan snapshotRequest
normal chan snapshotRequest
background chan snapshotRequest
done chan struct{}
mu sync.RWMutex
scanWG, workerWG sync.WaitGroup
stats struct {
enqueued int64
skipped int64
}
}
func (sq *prioritySnapshotQueue) spawnWorkers(w int) {
sq.mu.Lock()
defer sq.mu.Unlock()
if sq.done == nil {
sq.logger.Printf("prioritySnapshotQueue worker: no done channel, already done?")
return
}
sq.workerWG.Add(w)
for i := 0; i < w; i++ {
go sq.worker(sq.urgent, sq.normal, sq.background, sq.done)
}
}
func (sq *prioritySnapshotQueue) worker(urgent, normal, background chan snapshotRequest, done chan struct{}) {
// We don't want a race condition on these. If they're non-nil when
// we get them, they should get closed at some point. If done is
// already nil, we shouldn't do anything.
defer sq.workerWG.Done()
ok := true
var req snapshotRequest
for ok {
req.frag = nil
select {
case req, ok = <-urgent:
default:
select {
case req, ok = <-urgent:
case req, ok = <-normal:
default:
select {
case req, ok = <-urgent:
case req, ok = <-normal:
case req, ok = <-background:
case _, ok = <-done:
}
}
}
if req.frag != nil {
sq.process(req)
}
}
}
// process actually runs a fragment. it will do this if either the fragment
// has a pending snapshot, or the force flag is set.
func (sq *prioritySnapshotQueue) process(req snapshotRequest) {
f := req.frag
f.mu.Lock()
defer f.mu.Unlock()
if f.snapshotStamp.Before(req.when) {
f.snapshotErr = f.snapshot()
if f.snapshotErr != nil {
fmt.Printf("snapshot error: %v\n", f.snapshotErr)
sq.logger.Printf("snapshot error: %v", f.snapshotErr)
}
f.snapshotPending = false
f.snapshotCond.Broadcast()
}
}
// Stop shuts down the snapshot queue. It first marks it as done, causing
// the background scanner(s), if any, to shut down, then waits for them, then
// closes and nils the queues. The background scanner has to get stopped
// because otherwise it might try to write to those closed queues.
func (sq *prioritySnapshotQueue) Stop() {
sq.mu.Lock()
defer sq.mu.Unlock()
close(sq.done)
// scanners need to be done before we close the other channels.
sq.scanWG.Wait()
sq.done = nil
close(sq.normal)
sq.normal = nil
close(sq.urgent)
sq.urgent = nil
close(sq.background)
sq.background = nil
sq.logger.Printf("snapshot queue: enqueued %d, skipped %d\n", sq.stats.enqueued, sq.stats.skipped)
}
// Enqueue tries to add a fragment to the queue, if the fragment is not already
// enqueued. You should hold a lock on the fragment when calling this.
func (sq *prioritySnapshotQueue) Enqueue(f *fragment) {
if f.snapshotPending {
return
}
sq.mu.Lock()
defer sq.mu.Unlock()
if sq.normal == nil {
sq.logger.Printf("requested snapshot after snapshot queue was closed")
return
}
// we have to set this before enqueing, because it's
// otherwise possible that we're at the head of the queue,
// and the recipient gets the fragment before we execute the
// line after the send.
f.snapshotPending = true
// try to enqueue snapshot
select {
case sq.normal <- snapshotRequest{frag: f, when: time.Now()}:
sq.stats.enqueued++
return
default:
sq.stats.skipped++
f.snapshotPending = false
return
}
}
// Await returns when f is not pending a snapshot. Call with the fragment lock
// held. Await waits on a condition variable inside f, associated with the
// fragment's lock, so this does not conflict with the lock being used for
// snapshots.
func (sq *prioritySnapshotQueue) Await(f *fragment) (err error) {
for f.snapshotPending {
f.snapshotCond.Wait()
}
err, f.snapshotErr = f.snapshotErr, nil
return err
}
// Immediate forces an immediate snapshot of the given fragment. Call with
// the fragment locked. If the queue is already closing, the fragment does
// not get snapshotted.
func (sq *prioritySnapshotQueue) Immediate(f *fragment) error {
sq.mu.RLock()
// no deferred unlock, because we want to unlock this before calling Await.
// Not because that needs this lock, but because once we're that far, we
// *don't* need this lock anymore so someone else should have it.
if sq.urgent == nil {
sq.mu.RUnlock()
sq.logger.Printf("requested immediate snapshot after snapshot queue was closed")
return errors.New("requested immediate snapshot after snapshot queue was closed")
}
f.snapshotPending = true
req := snapshotRequest{frag: f, when: time.Now()}
// if the fragment was already in the work queue, it's *possible*
// that the only available worker just picked it off the queue, and
// is now waiting on getting the fragment's lock, so it can run
// a snapshot. So we let go of the lock on the fragment, send the
// request, then request the fragment lock again, because Await will
// be sleeping on the condition variable associated with the lock,
// which means it needs to hold the lock so it can let it go during
// the wait... No, really, this made sense.
f.mu.Unlock()
sq.urgent <- req
sq.mu.RUnlock()
f.mu.Lock()
return sq.Await(f)
}
// needsSnapshot determines whether a fragment probably wants snapshotting.
// Specifically, it looks for fragments not already marked to receive
// snapshots, but which have a high enough opN to justify a snapshot. This
// is only used from the background scan.
func (sq *prioritySnapshotQueue) needsSnapshot(f *fragment) bool {
if f == nil {
return false
}
f.mu.Lock()
defer f.mu.Unlock()
if f.snapshotPending {
return false
}
if f.opN > f.MaxOpN {
return true
}
return false
}
// ScanHolder spawns a goroutine which iterates through the holder's
// indexes/fields/views/fragments, looking for fragments which have OpN
// high enough to justify a snapshot but don't seem to have one pending.
// It then dumps these in the low priority background queue.
func (sq *prioritySnapshotQueue) ScanHolder(h *Holder) {
sq.mu.Lock()
sq.scanWG.Add(1)
go sq.scanHolderWorker(h, sq.background, sq.done)
sq.mu.Unlock()
}
// scanHolderWorker is a background task that scans a holder looking for
// fragments which need snapshots taken. It's the cleanup task for snapshots
// that would have been requested by Enqueue, but the queue was full.
func (sq *prioritySnapshotQueue) scanHolderWorker(h *Holder, background chan snapshotRequest, done chan struct{}) {
defer sq.scanWG.Done()
var indexNames, fieldNames, viewNames []string
var fragNums []uint64
for {
// To avoid abusing things, cap activity rate; every time we finish
// the holder, or every couple hundred fragments considered, we
// pause for a bit.
counter := 0
hits := 0
h.mu.Lock()
indexNames = indexNames[:0]
for indexName := range h.indexes {
indexNames = append(indexNames, indexName)
}
h.mu.Unlock()
for _, indexName := range indexNames {
h.mu.Lock()
index := h.indexes[indexName]
h.mu.Unlock()
if index == nil {
continue
}
fieldNames = fieldNames[:0]
index.mu.Lock()
for fieldName := range index.fields {
fieldNames = append(fieldNames, fieldName)
}
index.mu.Unlock()
for _, fieldName := range fieldNames {
index.mu.Lock()
field := index.fields[fieldName]
index.mu.Unlock()
if field == nil {
continue
}
viewNames = viewNames[:0]
field.mu.Lock()
for viewName := range field.viewMap {
viewNames = append(viewNames, viewName)
}
field.mu.Unlock()
for _, viewName := range viewNames {
field.mu.Lock()
view := field.viewMap[viewName]
field.mu.Unlock()
if view == nil {
continue
}
fragNums := fragNums[:0]
view.mu.Lock()
for fragNum := range view.fragments {
fragNums = append(fragNums, fragNum)
}
view.mu.Unlock()
for _, fragNum := range fragNums {
view.mu.Lock()
frag := view.fragments[fragNum]
view.mu.Unlock()
if sq.needsSnapshot(frag) {
hits++
select {
case background <- snapshotRequest{frag: frag, when: time.Now()}:
sq.logger.Debugf("found fragment needing snapshot: %s\n", frag.path)
case <-done:
return
}
} else {
// Count fragments examined *without* finding anything that
// needed a snapshot. When we find things that need snapshots,
// the time it takes the workers to respond to us is enough
// of a delay to keep us from eating every CPU. So, if a lot
// of things need snapshots, and the workers aren't doing
// anything else, ScanHolder will mostly keep them saturated.
// If they're busy, we'll block forever in the write to the
// background queue. If there's nothing that needs snapshots,
// we pause frequently for a second or so at a time.
counter++
if counter == 100 {
select {
case <-time.After(1 * time.Second):
case <-done:
return
}
counter = 0
}
}
}
}
}
}
if hits > 0 {
sq.logger.Printf("background scan: %d fragments needed snapshots\n", hits)
hits = 0
} else {
sq.logger.Printf("background scan: no fragments needed snapshots, waiting\n")
// No reason to be active if we're not finding anything.
select {
case <-time.After(60 * time.Second):
case <-done:
return
}
}
}
}

6
test/logger.go
View file

@ -18,12 +18,14 @@ import (
"bytes"
"fmt"
"io/ioutil"
"sync"
)
// bufferLogger represents a test Logger that holds log messages
// in a buffer for review.
type bufferLogger struct {
buf *bytes.Buffer
mu sync.Mutex
}
// NewBufferLogger returns a new instance of BufferLogger.
@ -34,6 +36,8 @@ func NewBufferLogger() *bufferLogger {
}
func (b *bufferLogger) Printf(format string, v ...interface{}) {
b.mu.Lock()
defer b.mu.Unlock()
s := fmt.Sprintf(format, v...)
_, err := b.buf.WriteString(s)
if err != nil {
@ -44,5 +48,7 @@ func (b *bufferLogger) Printf(format string, v ...interface{}) {
func (b *bufferLogger) Debugf(format string, v ...interface{}) {}
func (b *bufferLogger) ReadAll() ([]byte, error) {
b.mu.Lock()
defer b.mu.Unlock()
return ioutil.ReadAll(b.buf)
}

3
utils_internal_test.go
View file

@ -301,6 +301,9 @@ func (t *ClusterCluster) Close() error {
if err != nil {
return err
}
// Make sure open indexes get shut down too. we wouldn't do
// this normally for a cluster, but we want to for test cases.
c.holder.Close()
}
return nil
}

8
view.go
View file

@ -59,7 +59,7 @@ type view struct {
stats stats.StatsClient
rowAttrStore AttrStore
logger logger.Logger
snapshotQueue chan *fragment
snapshotQueue snapshotQueue
}
// newView returns a new instance of View.
@ -309,7 +309,9 @@ func (v *view) newFragment(path string, shard uint64) *fragment {
frag.CacheSize = v.cacheSize
frag.Logger = v.logger
frag.stats = v.stats
frag.snapshotQueue = v.snapshotQueue
if v.snapshotQueue != nil {
frag.snapshotQueue = v.snapshotQueue
}
if v.fieldType == FieldTypeMutex {
frag.mutexVector = newRowsVector(frag)
} else if v.fieldType == FieldTypeBool {
@ -483,7 +485,7 @@ func upgradeViewBSIv2(v *view, bitDepth uint) (ok bool, _ error) {
if tmpPath, err := upgradeRoaringBSIv2(frag, bitDepth); err != nil {
return ok, errors.Wrap(err, "upgrading bsi v2")
} else if err := frag.closeStorage(true); err != nil {
} else if err := frag.closeStorage(); err != nil {
return ok, errors.Wrap(err, "closing after bsi v2 upgrade")
} else if err := os.Rename(tmpPath, frag.path); err != nil {
return ok, errors.Wrap(err, "renaming after bsi v2 upgrade")