Merge pull request #22 from seebs/fsckSnapshotExtension

This is a collection of changes that have been pending forever. It improves the snapshot queue performance, adds some amount of recovery for corrupt filles, reduces memory usage in the rowcache, and adds an extension interface. Yes, they should probably have happened separately over time, things happened.

Makefile

@@ -141,7 +141,7 @@ docker-test:
 
 # Run golangci-lint
 golangci-lint: require-golangci-lint
-	golangci-lint run
+	golangci-lint run --skip-files '.*\.peg\.go'
 
 # Run gometalinter with custom flags
 gometalinter: require-gometalinter vendor

api.go

@@ -152,6 +152,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
 		ExcludeRowAttrs: req.ExcludeRowAttrs, // NOTE: Kept for Pilosa 1.x compat.
 		ExcludeColumns:  req.ExcludeColumns,  // NOTE: Kept for Pilosa 1.x compat.
 		ColumnAttrs:     req.ColumnAttrs,     // NOTE: Kept for Pilosa 1.x compat.
+		EmbeddedData:    req.EmbeddedData,    // precomputed values that needed to be passed with the request
 	}
 	resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
 	if err != nil {

cluster_internal_test.go

@@ -158,6 +158,7 @@ func TestFragSources(t *testing.T) {
 	c5.addNodeBasicSorted(node3)
 
 	idx := newIndexWithTempPath("i")
+	defer idx.Close()
 	field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)

encoding/proto/proto.go

@@ -396,14 +396,19 @@ func encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) *internal.Import
 }
 
 func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest {
-	return &internal.QueryRequest{
+	r := &internal.QueryRequest{
 		Query:           m.Query,
 		Shards:          m.Shards,
 		ColumnAttrs:     m.ColumnAttrs,
 		Remote:          m.Remote,
 		ExcludeRowAttrs: m.ExcludeRowAttrs,
 		ExcludeColumns:  m.ExcludeColumns,
+		EmbeddedData:    make([]*internal.Row, len(m.EmbeddedData)),
 	}
+	for i := range m.EmbeddedData {
+		r.EmbeddedData[i] = encodeRow(m.EmbeddedData[i])
+	}
+	return r
 }
 
 func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
@@ -416,6 +421,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
 		pb.Results[i] = &internal.QueryResult{}
 
 		switch result := m.Results[i].(type) {
+		case pilosa.SignedRow:
+			pb.Results[i].Type = queryResultTypeSignedRow
+			pb.Results[i].SignedRow = encodeSignedRow(result)
 		case *pilosa.Row:
 			pb.Results[i].Type = queryResultTypeRow
 			pb.Results[i].Row = encodeRow(result)
@@ -446,7 +454,7 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
 		case nil:
 			pb.Results[i].Type = queryResultTypeNil
 		default:
-			panic(fmt.Errorf("unknown type: %d", pb.Results[i].Type))
+			panic(fmt.Errorf("unknown type: %T", m.Results[i]))
 		}
 	}
 
@@ -966,6 +974,10 @@ func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) {
 	m.Remote = pb.Remote
 	m.ExcludeRowAttrs = pb.ExcludeRowAttrs
 	m.ExcludeColumns = pb.ExcludeColumns
+	m.EmbeddedData = make([]*pilosa.Row, len(pb.EmbeddedData))
+	for i := range pb.EmbeddedData {
+		m.EmbeddedData[i] = decodeRow(pb.EmbeddedData[i])
+	}
 }
 
 func decodeImportRequest(pb *internal.ImportRequest, m *pilosa.ImportRequest) {
@@ -1068,10 +1080,13 @@ const (
 	queryResultTypeGroupCounts
 	queryResultTypeRowIdentifiers
 	queryResultTypePair
+	queryResultTypeSignedRow
 )
 
 func decodeQueryResult(pb *internal.QueryResult) interface{} {
 	switch pb.Type {
+	case queryResultTypeSignedRow:
+		return decodeSignedRow(pb.SignedRow)
 	case queryResultTypeRow:
 		return decodeRow(pb.Row)
 	case queryResultTypePairs:
@@ -1099,14 +1114,31 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} {
 // DecodeRow converts r from its internal representation.
 func decodeRow(pr *internal.Row) *pilosa.Row {
 	if pr == nil {
-		return nil
+		return pilosa.NewRow()
 	}
 
-	r := pilosa.NewRow()
+	var r *pilosa.Row
+	if len(pr.Roaring) > 0 {
+		r = pilosa.NewRowFromRoaring(pr.Roaring)
+	} else {
+		r = pilosa.NewRow()
+		for _, v := range pr.Columns {
+			r.SetBit(v)
+		}
+	}
 	r.Attrs = decodeAttrs(pr.Attrs)
 	r.Keys = pr.Keys
-	for _, v := range pr.Columns {
-		r.SetBit(v)
+
+	return r
+}
+
+func decodeSignedRow(pr *internal.SignedRow) pilosa.SignedRow {
+	if pr == nil {
+		return pilosa.SignedRow{}
+	}
+	r := pilosa.SignedRow{
+		Pos: decodeRow(pr.Pos),
+		Neg: decodeRow(pr.Neg),
 	}
 	return r
 }
@@ -1213,16 +1245,29 @@ func encodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet {
 	}
 }
 
+func encodeSignedRow(r pilosa.SignedRow) *internal.SignedRow {
+	ir := &internal.SignedRow{
+		Pos: encodeRow(r.Pos),
+		Neg: encodeRow(r.Neg),
+	}
+	return ir
+}
+
 func encodeRow(r *pilosa.Row) *internal.Row {
 	if r == nil {
 		return nil
 	}
 
-	return &internal.Row{
-		Columns: r.Columns(),
-		Keys:    r.Keys,
-		Attrs:   encodeAttrs(r.Attrs),
+	ir := &internal.Row{
+		Keys:  r.Keys,
+		Attrs: encodeAttrs(r.Attrs),
 	}
+	if false {
+		ir.Columns = r.Columns()
+	} else {
+		ir.Roaring = r.Roaring()
+	}
+	return ir
 }
 
 func encodeRowIdentifiers(r pilosa.RowIdentifiers) *internal.RowIdentifiers {

executor.go

@@ -22,7 +22,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/pilosa/pilosa/v2/ext"
 	"github.com/pilosa/pilosa/v2/pql"
+	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/shardwidth"
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
@@ -57,6 +59,12 @@ type executor struct {
 	workersWG      sync.WaitGroup
 	workerPoolSize int
 	work           chan job
+	// global registry to check for name clashes
+	additionalOps map[string]*ext.BitmapOp
+	// typed registries we can use in lookups
+	additionalBitmapOps map[string]ext.BitmapOpBitmap
+	additionalCountOps  map[string]ext.BitmapOpUnaryCount
+	additionalFieldOps  map[string]ext.BitmapOpBSIBitmap
 }
 
 // executorOption is a functional option type for pilosa.Executor
@@ -109,6 +117,38 @@ func (e *executor) Close() error {
 	return nil
 }
 
+func (e *executor) registerOps(ops []ext.BitmapOp) error {
+	if e.additionalOps == nil {
+		e.additionalOps = make(map[string]*ext.BitmapOp)
+		e.additionalBitmapOps = make(map[string]ext.BitmapOpBitmap)
+		e.additionalCountOps = make(map[string]ext.BitmapOpUnaryCount)
+		e.additionalFieldOps = make(map[string]ext.BitmapOpBSIBitmap)
+	}
+	for i, op := range ops {
+		name := op.Name
+		if _, exists := e.additionalOps[name]; exists {
+			return fmt.Errorf("op name '%s' already defined", name)
+		}
+		e.additionalOps[name] = &ops[i]
+		typ := ops[i].Func.BitmapOpType()
+		switch {
+		case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:
+			e.additionalCountOps[name] = ops[i].Func.(ext.BitmapOpUnaryCount)
+		case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap:
+			e.additionalBitmapOps[name] = ops[i].Func.(ext.BitmapOpBitmap)
+		case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap:
+			if fn, ok := ops[i].Func.(ext.BitmapOpBSIBitmapPrecall); ok {
+				e.additionalFieldOps[name] = ext.BitmapOpBSIBitmap(fn)
+			} else {
+				e.additionalFieldOps[name] = ops[i].Func.(ext.BitmapOpBSIBitmap)
+			}
+		default:
+			return fmt.Errorf("unsupported types for '%s': input type %d, output type %d", name, typ.Input, typ.Output)
+		}
+	}
+	return nil
+}
+
 // Execute executes a PQL query.
 func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute")
@@ -237,6 +277,97 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr
 	return ax, nil
 }
 
+// handlePreCalls traverses the call tree looking for calls that need
+// precomputed values. Right now, that's just Distinct.
+func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
+	if c.Name == "Precomputed" {
+		idx := c.Args["valueidx"].(int64)
+		if idx >= 0 && idx < int64(len(opt.EmbeddedData)) {
+			row := opt.EmbeddedData[idx]
+			c.Precomputed = make(map[uint64]interface{}, len(row.segments))
+			for _, segment := range row.segments {
+				c.Precomputed[segment.shard] = &Row{segments: []rowSegment{segment}}
+			}
+		} else {
+			return fmt.Errorf("no precomputed data! index %d, len %d", idx, len(opt.EmbeddedData))
+		}
+		return nil
+	}
+	newIndex := c.CallIndex()
+	// A cross-index query is handled by precall. This is inefficient,
+	// but we have to do it for now because shards might be different and
+	// we haven't implemented the local precalls that would be enough
+	// in some cases.
+	//
+	// This makes simple cross-index queries noticably inefficient.
+	//
+	// If you're here because of that: We should be using PrecallLocal
+	// in cases where the call isn't already PrecallGlobal, and
+	// PrecallLocal should wait until we're running on a specific node
+	// to do the farming-out of just the sub-queries it has to run
+	// for its local shards.
+	//
+	// As is, we have one node querying every node, then sending out
+	// all the data to every node, including the data that node already
+	// has. We could reduce the actual copying around dramatically,
+	// but only in the cases where local is good enough -- not something
+	// like Distinct, where you can't predict output shard for a result
+	// from the shard being queried.
+	if newIndex != "" && newIndex != index {
+		c.Type = pql.PrecallGlobal
+		index = newIndex
+	}
+	if c.Type == pql.PrecallNone {
+		// otherwise, handle the children
+		return e.handlePreCallChildren(ctx, index, c, shards, opt)
+	}
+	// We don't try to handle sub-calls from here. I'm not 100%
+	// sure that's right, but I think the fact that they're happening
+	// inside a precomputed call may mean they need different
+	// handling. In any event, the sub-calls will get handled by
+	// the executeCall when it gets to them...
+
+	// We set c to look like a normal call, and actually execute it:
+	c.Type = pql.PrecallNone
+	// possibly override call index.
+	v, err := e.executeCall(ctx, index, c, shards, opt)
+	if err != nil {
+		return err
+	}
+	var row *Row
+	switch r := v.(type) {
+	case *Row:
+		row = r
+	case SignedRow:
+		row = r.Pos
+	default:
+		return fmt.Errorf("precomputed call %s returned unexpected non-Row data: %T", c.Name, v)
+
+	}
+	c.Children = []*pql.Call{}
+	c.Name = "Precomputed"
+	c.Args = map[string]interface{}{"valueidx": len(opt.EmbeddedData)}
+	// stash a copy of the full results, which can be forwarded to other
+	// shards if the query has to go to them
+	opt.EmbeddedData = append(opt.EmbeddedData, row)
+	// and stash a copy locally, so local calls can use it
+	c.Precomputed = make(map[uint64]interface{}, len(row.segments))
+	for _, segment := range row.segments {
+		c.Precomputed[segment.shard] = &Row{segments: []rowSegment{segment}}
+	}
+	return nil
+}
+
+// handlePreCallChildren handles any pre-calls in the children of a given call.
+func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
+	for i := range c.Children {
+		if err := e.handlePreCalls(ctx, index, c.Children[i], shards, opt); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.execute")
 	defer span.Finish()
@@ -270,7 +401,27 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar
 			return nil, err
 		}
 
-		v, err := e.executeCall(ctx, index, call, shards, opt)
+		// If you actually make a top-level Distinct call, you
+		// want a SignedRow back. Otherwise, it's something else
+		// that will be using it as a row, and we only care
+		// about the positive values, because only positive values
+		// are valid column IDs. So we don't actually eat top-level
+		// pre calls.
+		err := e.handlePreCallChildren(ctx, index, call, shards, opt)
+		if err != nil {
+			return nil, err
+		}
+		var v interface{}
+		// Top-level calls don't need to precompute cross-index things,
+		// because we can just pick whatever index we want, but we
+		// still need to handle them. Since everything else was
+		// already precomputed by handlePreCallChildren, though,
+		// we don't need this logic in executeCall.
+		if newIndex := call.CallIndex(); newIndex != "" {
+			v, err = e.executeCall(ctx, newIndex, call, shards, opt)
+		} else {
+			v, err = e.executeCall(ctx, index, call, shards, opt)
+		}
 		if err != nil {
 			return nil, err
 		}
@@ -299,6 +450,14 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
 	}
 
 	// Special handling for mutation and top-n calls.
+	if op, ok := e.additionalCountOps[c.Name]; ok {
+		e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
+		return e.executeGenericCount(ctx, index, c, op, shards, opt)
+	}
+	if op, ok := e.additionalFieldOps[c.Name]; ok {
+		e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
+		return e.executeGenericField(ctx, index, c, op, shards, opt)
+	}
 	switch c.Name {
 	case "Sum":
 		e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
@@ -343,6 +502,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
 		return e.executeOptionsCall(ctx, index, c, shards, opt)
 	case "IncludesColumn":
 		return e.executeIncludesColumnCall(ctx, index, c, shards, opt)
+	case "Precomputed":
+		return e.executePrecomputedCall(ctx, index, c, shards, opt)
 	default:
 		e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
 		return e.executeBitmapCall(ctx, index, c, shards, opt)
@@ -501,6 +662,37 @@ func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, sh
 	return other, nil
 }
 
+// executeGenericField executes a generic call on a field. Note that in this
+// implementation, the operation is always a BSI op.
+func (e *executor) executeGenericField(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shards []uint64, opt *execOptions) (SignedRow, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericField")
+	span.LogKV("name", c.Name)
+	defer span.Finish()
+
+	if field := c.Args["field"]; field == "" {
+		return SignedRow{}, fmt.Errorf("plugin operation %s(): field required", c.Name)
+	}
+
+	// Execute calls in bulk on each remote node and merge.
+	mapFn := func(shard uint64) (interface{}, error) {
+		return e.executeGenericFieldShard(ctx, index, c, op, shard)
+	}
+
+	// Merge returned results at coordinating node.
+	reduceFn := func(prev, v interface{}) interface{} {
+		other, _ := prev.(SignedRow)
+		return other.union(v.(SignedRow))
+	}
+
+	result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
+	if err != nil {
+		return SignedRow{}, err
+	}
+	other, _ := result.(SignedRow)
+
+	return other, nil
+}
+
 // executeMin executes a Min() call.
 func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMin")
@@ -641,6 +833,41 @@ func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call,
 	return e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
 }
 
+// executePrecomputedCall pretends to execute a call that we have a precomputed value for.
+func (e *executor) executePrecomputedCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executePrecomputedCall")
+	defer span.Finish()
+
+	// Execute calls in bulk on each remote node and merge.
+	mapFn := func(shard uint64) (interface{}, error) {
+		if c.Precomputed != nil {
+			return c.Precomputed[shard], nil
+		}
+		// This might not be an error -- if there were no values, we will not have created
+		// the corresponding row.
+		return NewRow(), nil
+	}
+
+	// Merge returned results at coordinating node.
+	reduceFn := func(prev, v interface{}) interface{} {
+		other, _ := prev.(*Row)
+		if other == nil {
+			other = NewRow()
+		}
+		other.Merge(v.(*Row))
+		return other
+	}
+
+	other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
+	if err != nil {
+		return nil, errors.Wrap(err, "map reduce")
+	}
+
+	row, _ := other.(*Row)
+
+	return row, nil
+}
+
 // executeBitmapCall executes a call that returns a bitmap.
 func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCall")
@@ -719,6 +946,13 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard")
 	defer span.Finish()
 
+	if _, ok := e.additionalCountOps[c.Name]; ok {
+		return nil, fmt.Errorf("count op %s used as bitmap call", c.Name)
+	}
+	if op, ok := e.additionalBitmapOps[c.Name]; ok {
+		return e.executeGenericBitmapShard(ctx, index, c, op, shard)
+	}
+
 	switch c.Name {
 	case "Row", "Range":
 		return e.executeRowShard(ctx, index, c, shard)
@@ -734,11 +968,63 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *
 		return e.executeNotShard(ctx, index, c, shard)
 	case "Shift":
 		return e.executeShiftShard(ctx, index, c, shard)
+	case "Precomputed":
+		return e.executePrecomputedCallShard(ctx, index, c, shard)
 	default:
 		return nil, fmt.Errorf("unknown call: %s", c.Name)
 	}
 }
 
+// executeGenericFieldShard executes a generic/extension command on a
+// single shard. Note that in this implementation, the op is always
+// a BSI op.
+func (e *executor) executeGenericFieldShard(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shard uint64) (SignedRow, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericShard")
+	defer span.Finish()
+
+	var filter *Row
+	var filterBitmap *roaring.Bitmap
+	if len(c.Children) == 1 {
+		row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
+		if err != nil {
+			return SignedRow{}, errors.Wrap(err, "executing bitmap call")
+		}
+		filter = row
+		if filter != nil && len(filter.segments) > 0 {
+			filterBitmap = filter.segments[0].data
+		}
+	}
+
+	fieldName, _ := c.Args["field"].(string)
+
+	field := e.Holder.Field(index, fieldName)
+	if field == nil {
+		return SignedRow{}, nil
+	}
+
+	bsig := field.bsiGroup(fieldName)
+	if bsig == nil {
+		return SignedRow{}, nil
+	}
+
+	fragment := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
+	if fragment == nil {
+		return SignedRow{}, nil
+	}
+
+	var out ext.SignedBitmap
+	if filterBitmap != nil {
+		out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{WrapBitmap(filterBitmap)}, c.Args)
+	} else {
+		out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{}, c.Args)
+	}
+
+	return SignedRow{
+		Neg: NewRowFromBitmap(UnwrapBitmap(out.Neg)),
+		Pos: NewRowFromBitmap(UnwrapBitmap(out.Pos)),
+	}, nil
+}
+
 // executeSumCountShard calculates the sum and count for bsiGroups on a shard.
 func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSumCountShard")
@@ -1749,6 +2035,44 @@ func (e *executor) executeIntersectShard(ctx context.Context, index string, c *p
 	return other, nil
 }
 
+// executeGenericBitmapShard executes a generic bitmap call for a local shard.
+func (e *executor) executeGenericBitmapShard(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBitmap, shard uint64) (*Row, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericBitmapShard")
+	defer span.Finish()
+
+	if op.BitmapOpArity() == ext.OpArityUnary {
+		if len(c.Children) != 1 {
+			return nil, fmt.Errorf("%s needs exactly one row parameter", c.Name)
+		}
+		row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
+		if err != nil {
+			return nil, err
+		}
+		return row.GenericUnaryOp(op.BitmapOpFunc(), c.Args), nil
+	}
+
+	var err error
+	rows := make([]*Row, len(c.Children))
+	for i, input := range c.Children {
+		rows[i], err = e.executeBitmapCallShard(ctx, index, input, shard)
+		if err != nil {
+			return nil, err
+		}
+	}
+	var other *Row
+	switch op.BitmapOpArity() {
+	case ext.OpArityBinary:
+		other = rows[0]
+		for _, row := range rows[1:] {
+			other = other.GenericBinaryOp(op.BitmapOpFunc(), row, c.Args)
+		}
+	case ext.OpArityNary:
+		other = rows[0].GenericNaryOp(op.BitmapOpFunc(), rows[1:], c.Args)
+	}
+	other.invalidateCount()
+	return other, nil
+}
+
 // executeUnionShard executes a union() call for a local shard.
 func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard")
@@ -1793,6 +2117,25 @@ func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Cal
 	return other, nil
 }
 
+// executePrecomputedCallShard pretends to execute a precomputed call for a local shard.
+func (e *executor) executePrecomputedCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
+	if c.Precomputed != nil {
+		v := c.Precomputed[shard]
+		if v == nil {
+			return NewRow(), nil
+		}
+		if r, ok := v.(*Row); ok {
+			if r != nil {
+				return r, nil
+			} else {
+				return NewRow(), nil
+			}
+		}
+		return nil, fmt.Errorf("precomputed value is not a row: %T", v)
+	}
+	return nil, fmt.Errorf("per-shard: missing precomputed values for shard %d", shard)
+}
+
 // executeNotShard executes a not() call for a local shard.
 func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeNotShard")
@@ -1849,6 +2192,41 @@ func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.C
 	return row.Shift(n)
 }
 
+// executeGeneric executes a provided count-like call.
+func (e *executor) executeGenericCount(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpUnaryCount, shards []uint64, opt *execOptions) (uint64, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericCount")
+	defer span.Finish()
+
+	if len(c.Children) == 0 {
+		return 0, fmt.Errorf("%s() requires an input bitmap", c.Name)
+	} else if len(c.Children) > 1 {
+		return 0, fmt.Errorf("%s() only accepts a single bitmap input", c.Name)
+	}
+
+	// Execute calls in bulk on each remote node and merge.
+	mapFn := func(shard uint64) (interface{}, error) {
+		row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
+		if err != nil {
+			return 0, err
+		}
+		return row.GenericCount(op, c.Args), nil
+	}
+
+	// Merge returned results at coordinating node.
+	reduceFn := func(prev, v interface{}) interface{} {
+		other, _ := prev.(uint64)
+		return other + v.(uint64)
+	}
+
+	result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
+	if err != nil {
+		return 0, err
+	}
+	n, _ := result.(uint64)
+
+	return n, nil
+}
+
 // executeCount executes a count() call.
 func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount")
@@ -1946,7 +2324,7 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq
 		}
 
 		// Forward call to remote node otherwise.
-		res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
+		res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
 		if err != nil {
 			return false, err
 		}
@@ -2222,7 +2600,7 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.
 		}
 
 		// Forward call to remote node otherwise.
-		res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
+		res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
 		if err != nil {
 			return false, err
 		}
@@ -2257,7 +2635,7 @@ func (e *executor) executeSetValueField(ctx context.Context, index string, c *pq
 		}
 
 		// Forward call to remote node otherwise.
-		res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
+		res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
 		if err != nil {
 			return false, err
 		}
@@ -2311,7 +2689,7 @@ func (e *executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
 	resp := make(chan error, len(nodes))
 	for _, node := range nodes {
 		go func(node *Node) {
-			_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
+			_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
 			resp <- err
 		}(node)
 	}
@@ -2406,7 +2784,7 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
 	resp := make(chan error, len(nodes))
 	for _, node := range nodes {
 		go func(node *Node) {
-			_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil)
+			_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil, nil)
 			resp <- err
 		}(node)
 	}
@@ -2458,7 +2836,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
 	resp := make(chan error, len(nodes))
 	for _, node := range nodes {
 		go func(node *Node) {
-			_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil)
+			_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
 			resp <- err
 		}(node)
 	}
@@ -2474,15 +2852,16 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
 }
 
 // remoteExec executes a PQL query remotely for a set of shards on a node.
-func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64) (results []interface{}, err error) { // nolint: interfacer
+func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, embed []*Row) (results []interface{}, err error) { // nolint: interfacer
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeExec")
 	defer span.Finish()
 
 	// Encode request object.
 	pbreq := &QueryRequest{
-		Query:  q.String(),
-		Shards: shards,
-		Remote: true,
+		Query:        q.String(),
+		Shards:       shards,
+		Remote:       true,
+		EmbeddedData: embed,
 	}
 
 	pb, err := e.client.QueryNode(ctx, &node.URI, index, pbreq)
@@ -2577,6 +2956,43 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
 	}
 }
 
+// makeEmbeddedDataForShards produces new rows containing the rowSegments
+// that would correspond to a given set of shards.
+func makeEmbeddedDataForShards(allRows []*Row, shards []uint64) []*Row {
+	if len(allRows) == 0 || len(shards) == 0 {
+		return nil
+	}
+	newRows := make([]*Row, len(allRows))
+	for i, row := range allRows {
+		if row == nil || len(row.segments) == 0 {
+			continue
+		}
+		segments := row.segments
+		segmentIndex := 0
+		newRows[i] = &Row{}
+		for _, shard := range shards {
+			for segmentIndex < len(segments) && segments[segmentIndex].shard < shard {
+				segmentIndex++
+			}
+			// no more segments in this row
+			if segmentIndex >= len(segments) {
+				break
+			}
+			if segments[segmentIndex].shard == shard {
+				newRows[i].segments = append(newRows[i].segments, segments[segmentIndex])
+				segmentIndex++
+				if segmentIndex >= len(segments) {
+					// no more segments, we're done
+					break
+				}
+			}
+			// if we got here, segments[segmentIndex].shard exists
+			// but is greater than the current shard, so we continue.
+		}
+	}
+	return newRows
+}
+
 func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapper")
 	defer span.Finish()
@@ -2596,7 +3012,11 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
 			if n.ID == e.Node.ID {
 				resp.result, resp.err = e.mapperLocal(ctx, nodeShards, mapFn, reduceFn)
 			} else if !opt.Remote {
-				results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeShards)
+				var embeddedRowsForNode []*Row
+				if opt.EmbeddedData != nil {
+					embeddedRowsForNode = makeEmbeddedDataForShards(opt.EmbeddedData, nodeShards)
+				}
+				results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeShards, embeddedRowsForNode)
 				if len(results) > 0 {
 					resp.result = results[0]
 				}
@@ -3026,6 +3446,7 @@ type execOptions struct {
 	ExcludeRowAttrs bool
 	ExcludeColumns  bool
 	ColumnAttrs     bool
+	EmbeddedData    []*Row
 }
 
 // hasOnlySetRowAttrs returns true if calls only contains SetRowAttrs() calls.
@@ -3060,6 +3481,36 @@ func needsShards(calls []*pql.Call) bool {
 	return false
 }
 
+// SignedRow represents a signed *Row with two (neg/pos) *Rows.
+type SignedRow struct {
+	Neg *Row `json:"neg"`
+	Pos *Row `json:"pos"`
+}
+
+func (sr *SignedRow) union(other SignedRow) SignedRow {
+	ret := SignedRow{&Row{}, &Row{}}
+
+	// merge in sr
+	if sr != nil {
+		if sr.Neg != nil {
+			ret.Neg = ret.Neg.Union(sr.Neg)
+		}
+		if sr.Pos != nil {
+			ret.Pos = ret.Pos.Union(sr.Pos)
+		}
+	}
+
+	// merge in other
+	if other.Neg != nil {
+		ret.Neg = ret.Neg.Union(other.Neg)
+	}
+	if other.Pos != nil {
+		ret.Pos = ret.Pos.Union(other.Pos)
+	}
+
+	return ret
+}
+
 // ValCount represents a grouping of sum & count for Sum() and Average() calls.
 type ValCount struct {
 	Val   int64 `json:"value"`

ext/ext.go

@@ -0,0 +1,238 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package ext provides an EXPERIMENTAL AND TEMPORARY interface to use for
+// plugin extensions to Pilosa. DO NOT DEVELOP NEW PLUGINS WITH THIS. The
+// replacement design is already in process, but it needs more refinement
+// to address issues. This one has those issues, and more.
+//
+// In the current design, plugins will be loaded at runtime using the
+// go `plugin` package, so they should be built as a main package using
+// the plugin build mode.
+//
+// Plugins should not import other packages from Pilosa.
+//
+// To advertise their functionality, plugins define one or more of a
+// handful of symbols which will be checked for at plugin load and used
+// to register their functionality.
+//
+// The plugin interface will check for the following function(s). If the
+// functions exist, they must have the given signatures. If they return
+// a non-nil error, no ops are registered, and the error message will
+// be reported in the Pilosa server's logs.
+//
+// BitmapOps() ([]BitmapOp, error)
+//
+// These functions may be absent, and may return nil slices; in either
+// case, no ops are registered.
+package ext
+
+// The Bitmap type represents a Pilosa bitmap, and is used for bitmap
+// operations.
+type Bitmap interface {
+	// AddN and RemoveN can be used to add or remove values from a bitmap.
+	AddN(a ...uint64) (int, error)
+	RemoveN(a ...uint64) (int, error)
+
+	// Lookups
+	Max() uint64
+	Min() (uint64, bool)
+	Count() uint64
+	Any() bool
+	Contains(uint64) bool
+	Slice() []uint64
+	SliceRange(uint64, uint64) []uint64
+	// ContainerBits stores the next 1<<16 bits, starting at the provided
+	// bit index. It may use a provided []uint64 to store them, or may
+	// provide its own. Don't write to those bits. Offset must be a multiple
+	// of 1<<16.
+	ContainerBits(uint64, []uint64) []uint64
+
+	// These operators provide existing implemented binary ops.
+	Intersect(Bitmap) Bitmap
+	Union(Bitmap) Bitmap
+	IntersectionCount(Bitmap) uint64
+	Difference(Bitmap) Bitmap
+	Xor(Bitmap) Bitmap
+	Shift(int) (Bitmap, error)
+	Flip(uint64, uint64) Bitmap
+
+	// New() is an atrocity: it creates a new bitmap, unrelated to the
+	// existing bitmap. This lets you create a new bitmap without having
+	// imported any of the packages that have bitmap creation tools, because
+	// the bitmap wrapper type has to give you one.
+	New() Bitmap
+}
+
+// SignedBitmap represents a bitmap that can contain both positive and negative
+// values.
+type SignedBitmap struct {
+	Pos, Neg Bitmap
+}
+
+// A BitmapOp represents a new bitmap operation that should be exposed
+// in PQL.
+
+type BitmapOpInput byte
+type BitmapOpOutput byte
+type BitmapOpArity byte
+type BitmapOpPrecall byte
+type BitmapOpType struct {
+	Input   BitmapOpInput
+	Arity   BitmapOpArity
+	Output  BitmapOpOutput
+	Precall BitmapOpPrecall
+}
+
+const (
+	OpArityUnary = BitmapOpArity(iota)
+	OpArityBinary
+	OpArityNary
+)
+
+const (
+	// Unary: Exactly one bitmap.
+	OpInputBitmap = BitmapOpInput(iota)
+	// The really weird special case used for BSI, where we end up
+	// needing to do BSI computations. Arguments will be a
+	// single BitmapBSI, and a []Bitmap for other operands if any.
+	OpInputNaryBSI
+)
+
+const (
+	OpOutputCount = BitmapOpOutput(iota)
+	OpOutputBitmap
+	OpOutputSignedBitmap
+)
+
+const (
+	OpPrecallNone = BitmapOpPrecall(iota)
+	OpPrecallGlobal
+	OpPrecallLocal // unimplemented
+)
+
+// Regardless of arity, non-BSI functions should always take []Bitmap.
+type BitmapOpFunc interface {
+	BitmapOpType() BitmapOpType
+}
+
+// BitmapOpBitmap should actually always be func([]Bitmap) Bitmap, but
+// might be different kinds.
+type BitmapOpBitmap interface {
+	BitmapOpArity() BitmapOpArity
+	BitmapOpFunc() GenericBitmapOpBitmap
+}
+
+// the common underlying type of the other BitmapOpBitmap functions
+type GenericBitmapOpBitmap func([]Bitmap, map[string]interface{}) Bitmap
+
+// BitmapBSI represents the way a single BSI field is passed into a function
+// which takes a BSI field.
+type BitmapBSI struct {
+	FieldData  Bitmap
+	ShardWidth uint64
+	Offset     int64
+	Depth      uint
+}
+
+type BitmapOpBSIBitmap func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap
+
+func (b BitmapOpBSIBitmap) BitmapOpType() BitmapOpType {
+	return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Output: OpOutputSignedBitmap}
+}
+
+type BitmapOpBSIBitmapPrecall func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap
+
+func (b BitmapOpBSIBitmapPrecall) BitmapOpType() BitmapOpType {
+	return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Precall: OpPrecallGlobal, Output: OpOutputSignedBitmap}
+}
+
+type BitmapOpUnaryCount func([]Bitmap, map[string]interface{}) int64
+
+func (b BitmapOpUnaryCount) BitmapOpType() BitmapOpType {
+	return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputCount}
+}
+
+type BitmapOpUnaryBitmap func([]Bitmap, map[string]interface{}) Bitmap
+
+func (b BitmapOpUnaryBitmap) BitmapOpType() BitmapOpType {
+	return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputBitmap}
+}
+
+func (b BitmapOpUnaryBitmap) BitmapOpArity() BitmapOpArity {
+	return OpArityUnary
+}
+
+func (b BitmapOpUnaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap {
+	return GenericBitmapOpBitmap(b)
+}
+
+type BitmapOpBinaryBitmap func([]Bitmap, map[string]interface{}) Bitmap
+
+func (b BitmapOpBinaryBitmap) BitmapOpType() BitmapOpType {
+	return BitmapOpType{Input: OpInputBitmap, Arity: OpArityBinary, Output: OpOutputBitmap}
+}
+
+func (b BitmapOpBinaryBitmap) BitmapOpArity() BitmapOpArity {
+	return OpArityBinary
+}
+
+func (b BitmapOpBinaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap {
+	return GenericBitmapOpBitmap(b)
+}
+
+type BitmapOpNaryBitmap func([]Bitmap, map[string]interface{}) Bitmap
+
+func (b BitmapOpNaryBitmap) BitmapOpType() BitmapOpType {
+	return BitmapOpType{Input: OpInputBitmap, Arity: OpArityNary, Output: OpOutputBitmap}
+}
+
+func (b BitmapOpNaryBitmap) BitmapOpArity() BitmapOpArity {
+	return OpArityNary
+}
+
+func (b BitmapOpNaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap {
+	return GenericBitmapOpBitmap(b)
+}
+
+// BitmapOp represents an operation to be supported in PQL. Operations
+// on bitmaps should always take []Bitmap. Operations on InputNaryBSI should
+// take a []Bitmap, plus a Bitmap/shard-width/offset/depth.
+//
+// Reserved is a list of words to treat as reserved words in a prototype.
+// This is not currently used but might be later, and I want to have the
+// concept handy now.
+type BitmapOp struct {
+	Name     string
+	Func     BitmapOpFunc
+	Reserved []string
+}
+
+// ExtensionInfo tells us about the extension. The ExtensionAPI string
+// should be "v0". The version is a human-readable version, use something
+// that seems meaningful. Name and Description are reasonably self-explanatory,
+// I hope.
+//
+// Extensions should define a function:
+//   func ExtensionInfo(extensionAPI string) (*ExtensionInfo, error)
+// which reports their extension info if they think they can coexist with that
+// API string.
+type ExtensionInfo struct {
+	Name         string     // Extension name.
+	Description  string     // Short description.
+	Version      string     // Human-readable version info for extension.
+	ExtensionAPI string     // Extension API version. Should be v0 for now.
+	License      string     // License info.
+	BitmapOps    []BitmapOp // List of provided ops.
+}

ext/samples/.gitignore

@@ -0,0 +1 @@
+*/*.so

ext/samples/some/some.go

@@ -0,0 +1,125 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"math/bits"
+
+	"github.com/molecula/apophenia"
+	"github.com/pilosa/pilosa/v2/ext"
+)
+
+// This could be dynamically generated, but for now it's not.
+// nolint:unused,deadcode
+var extInfoTemplate = &ext.ExtensionInfo{
+	Name:         "some",
+	Description:  "some of the bits/all of the bits/none of the bits",
+	Version:      "0.01",
+	ExtensionAPI: "v0",
+	License:      "unreleased",
+	BitmapOps: []ext.BitmapOp{
+		{Name: "Some", Func: ext.BitmapOpUnaryBitmap(Some), Reserved: []string{"p", "seed"}},
+	},
+}
+
+// ExtensionInfo is the entry point used by the plugin code.
+func ExtensionInfo(api string) (*ext.ExtensionInfo, error) { // nolint:unused,deadcode
+	return extInfoTemplate, nil
+}
+
+const batchSize = 1024
+
+// Some returns some of the bits from its first input bitmap. Takes seed (int)
+// and p (float) values. Seed defaults to 0.
+func Some(inputs []ext.Bitmap, args map[string]interface{}) ext.Bitmap {
+	if len(inputs) == 0 || inputs[0] == nil {
+		return nil
+	}
+	input := inputs[0]
+	min, ok := input.Min()
+	// no bits found?
+	if !ok {
+		return nil
+	}
+	// start at multiple of 128 not greater than min.
+	min &^= 127
+	max := input.Max()
+	p, ok := args["p"].(float64)
+	if !ok {
+		return nil
+	}
+	// no bits or impossible probability range
+	if p <= 0 || p > 1 {
+		return nil
+	}
+	// every bit
+	if p == 1 {
+		return inputs[0]
+	}
+	// On failure, we default to 0.
+	seed, _ := args["seed"].(int64)
+	densityScale := uint64(256)
+	density := uint64(p * float64(densityScale))
+	for density == 0 {
+		densityScale <<= 1
+		density = uint64(p * float64(densityScale))
+		// too small
+		if densityScale > (1 << 32) {
+			return nil
+		}
+	}
+	w, err := apophenia.NewWeighted(apophenia.NewSequence(seed))
+	if err != nil {
+		return nil
+	}
+	someBits := input.New()
+	toAdd := make([]uint64, batchSize)
+	toAddN := 0
+	offset := apophenia.OffsetFor(apophenia.SequenceWeighted, 0, 0, 0)
+	for i := min; i < max; i += 128 {
+		offset.Lo = i
+		randomBits := w.Bits(offset, density, densityScale)
+		bit := uint64(0)
+		for randomBits.Lo != 0 {
+			next := uint64(bits.TrailingZeros64(randomBits.Lo) + 1)
+			randomBits.Lo >>= next
+			toAdd[toAddN] = next + bit + i
+			toAddN++
+			bit += next
+		}
+		bit = 64
+		for randomBits.Hi != 0 {
+			next := uint64(bits.TrailingZeros64(randomBits.Hi) + 1)
+			randomBits.Hi >>= next
+			toAdd[toAddN] = next + bit + i
+			toAddN++
+			bit += next
+		}
+		if toAddN > (batchSize - 128) {
+			// ignore error
+			_, _ = someBits.AddN(toAdd[:toAddN]...)
+			toAddN = 0
+		}
+	}
+	if toAddN > 0 {
+		_, _ = someBits.AddN(toAdd[:toAddN]...)
+	}
+	return input.Intersect(someBits)
+}
+
+func main() {
+	fmt.Printf("this is a plugin module only.\n")
+}

extension.go

@@ -0,0 +1,96 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+
+	"github.com/pilosa/pilosa/v2/ext"
+	"github.com/pilosa/pilosa/v2/roaring"
+)
+
+// WrapBitmap yields an extension-Bitmap from a roaring Bitmap.
+func WrapBitmap(bm *roaring.Bitmap) ext.Bitmap {
+	return wrappedBitmap{bm}
+}
+
+// wrappedBitmap is a very shallow glue shim to convert a roaring Bitmap to
+// an extension Bitmap.
+type wrappedBitmap struct{ *roaring.Bitmap }
+
+// UnwrapBitmap converts an extension-bitmap to its underlying roaring Bitmap.
+func UnwrapBitmap(bm ext.Bitmap) *roaring.Bitmap {
+	if inner, ok := bm.(wrappedBitmap); ok {
+		if inner.Bitmap != nil {
+			return inner.Bitmap
+		}
+		return roaring.NewFileBitmap()
+	}
+	return roaring.NewFileBitmap()
+}
+
+func (b wrappedBitmap) Intersect(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Intersect(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) Union(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Union(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) IntersectionCount(other ext.Bitmap) uint64 {
+	return b.Bitmap.IntersectionCount(other.(wrappedBitmap).Bitmap)
+}
+
+func (b wrappedBitmap) Difference(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Difference(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) Xor(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Xor(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) Shift(n int) (ext.Bitmap, error) {
+	shifted, err := b.Bitmap.Shift(n)
+	return wrappedBitmap{shifted}, err
+}
+
+func (b wrappedBitmap) Flip(start, last uint64) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Flip(start, last)}
+}
+
+func (b wrappedBitmap) New() ext.Bitmap {
+	return WrapBitmap(roaring.NewFileBitmap())
+}
+
+// ContainerBits tries to get one container's worth of bits.
+func (b wrappedBitmap) ContainerBits(offset uint64, target []uint64) (out []uint64) {
+	// it's an error to call this with a non-container-aligned offset
+	if offset&0xFFFF != 0 {
+		return nil
+	}
+	if b.Bitmap == nil {
+		fmt.Printf("ContainerBits on bitmap with no contents\n")
+		return nil
+	}
+	if b.Bitmap.Containers == nil {
+		fmt.Printf("ContainerBits on bitmap with nil Containers\n")
+		return nil
+	}
+	c := b.Bitmap.Containers.Get(offset >> 16)
+	if c == nil {
+		return nil
+	}
+	return c.AsBitmap(target)
+}

field.go

@@ -91,8 +91,7 @@ type Field struct {
 
 	logger logger.Logger
 
-	snapshotQueue chan *fragment
-
+	snapshotQueue snapshotQueue
 	// Instantiates new translation store on open.
 	OpenTranslateStore OpenTranslateStoreFunc
 }
@@ -316,18 +315,30 @@ func (f *Field) mergeRemoteAvailableShards(b *roaring.Bitmap) {
 
 // loadAvailableShards reads remoteAvailableShards data for the field, if any.
 func (f *Field) loadAvailableShards() error {
-	bm := roaring.NewBitmap()
 	// Read data from meta file.
 	path := filepath.Join(f.path, ".available.shards")
 	buf, err := ioutil.ReadFile(path)
+	// doesn't exist: this is fine
 	if os.IsNotExist(err) {
 		return nil
-	} else if err != nil {
-		return errors.Wrap(err, "reading available shards")
-	} else {
-		if err := bm.UnmarshalBinary(buf); err != nil {
-			return errors.Wrap(err, "unmarshaling")
+	}
+	// some other problem:
+	if err != nil {
+		f.logger.Printf("available shards file present but unreadable, discarding: %v", err)
+		err = os.Remove(path)
+		if err != nil {
+			return errors.Wrap(err, "deleting corrupt available shards list")
 		}
+		return nil
+	}
+	bm := roaring.NewBitmap()
+	if err = bm.UnmarshalBinary(buf); err != nil {
+		f.logger.Printf("available shards file corrupt, discarding: %v", err)
+		err = os.Remove(path)
+		if err != nil {
+			return errors.Wrap(err, "deleting corrupt available shards list")
+		}
+		return nil
 	}
 	// Merge bitmap from file into field.
 	f.mergeRemoteAvailableShards(bm)
@@ -912,7 +923,9 @@ func (f *Field) newView(path, name string) *view {
 	view.rowAttrStore = f.rowAttrStore
 	view.stats = f.Stats
 	view.broadcaster = f.broadcaster
-	view.snapshotQueue = f.snapshotQueue
+	if f.snapshotQueue != nil {
+		view.snapshotQueue = f.snapshotQueue
+	}
 	return view
 }
 

field_internal_test.go

@@ -19,6 +19,7 @@ import (
 	"io/ioutil"
 	"math"
 	"os"
+	"path/filepath"
 	"reflect"
 	"testing"
 	"time"
@@ -366,6 +367,62 @@ func TestField_PersistAvailableShards(t *testing.T) {
 
 }
 
+func TestField_CorruptAvailableShards(t *testing.T) {
+	f := MustOpenField(OptFieldTypeDefault())
+
+	// bm represents remote available shards.
+	bm := roaring.NewBitmap(1, 2, 3)
+
+	if err := f.AddRemoteAvailableShards(bm); err != nil {
+		t.Fatal(err)
+	}
+
+	path := filepath.Join(f.path, ".available.shards")
+
+	avail, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := avail.Write([]byte{23})
+	if err != nil || n != 1 {
+		t.Fatal(err)
+	}
+	avail.Close()
+
+	// Reload field and verify that shard data is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
+		t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
+	}
+}
+
+func TestField_TruncatedAvailableShards(t *testing.T) {
+	f := MustOpenField(OptFieldTypeDefault())
+
+	// bm represents remote available shards.
+	bm := roaring.NewBitmap(1, 2, 3)
+
+	if err := f.AddRemoteAvailableShards(bm); err != nil {
+		t.Fatal(err)
+	}
+
+	path := filepath.Join(f.path, ".available.shards")
+
+	avail, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	avail.Close()
+
+	// Reload field and verify that shard data is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
+		t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
+	}
+}
+
 // Ensure that persisting available shards having a smaller footprint (for example,
 // when going from a bitmap to a smaller, RLE representation) succeeds.
 func TestField_PersistAvailableShardsFootprint(t *testing.T) {

fragment.go

@@ -43,7 +43,6 @@ import (
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/shardwidth"
 	"github.com/pilosa/pilosa/v2/stats"
-	"github.com/pilosa/pilosa/v2/syswrap"
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
 )
@@ -108,21 +107,16 @@ type fragment struct {
 	shard uint64
 
 	// File-backed storage
-	path               string
-	flags              byte // user-defined flags passed to roaring
-	file               *os.File
-	storage            *roaring.Bitmap
-	storageData        []byte
-	totalOpN           int64 // total opN values
-	totalOps           int64 // total ops (across all snapshots)
-	opN                int   // number of ops since snapshot (may be approximate for imports)
-	ops                int   // number of higher-level operations, as opposed to bit changes
-	snapshotsRequested int   // number of times we've requested a snapshot
-	snapshotsTaken     int   // number of actual snapshot operations
-	snapshotting       bool  // set to true when requesting a snapshot, set to false after snapshot completes
-	snapshotCond       sync.Cond
-	snapshotDelays     int
-	snapshotDelayTime  time.Duration
+	path            string
+	flags           byte // user-defined flags passed to roaring
+	gen             generation
+	storage         *roaring.Bitmap
+	opN             int  // number of ops since snapshot (may be approximate for imports)
+	ops             int  // number of higher-level operations, as opposed to bit changes
+	snapshotPending bool // set to true when requesting a snapshot, set to false after snapshot completes
+	snapshotCond    sync.Cond
+	snapshotErr     error     // error yielded by the last snapshot operation
+	snapshotStamp   time.Time // timestamp of last snapshot
 
 	// Cache for row counts.
 	CacheType string // passed in by field
@@ -156,7 +150,7 @@ type fragment struct {
 
 	stats stats.StatsClient
 
-	snapshotQueue chan *fragment
+	snapshotQueue snapshotQueue
 }
 
 // newFragment returns a new instance of Fragment.
@@ -174,7 +168,8 @@ func newFragment(path, index, field, view string, shard uint64, flags byte) *fra
 		Logger: logger.NopLogger,
 		MaxOpN: defaultFragmentMaxOpN,
 
-		stats: stats.NopStatsClient,
+		stats:         stats.NopStatsClient,
+		snapshotQueue: defaultSnapshotQueue,
 	}
 	f.snapshotCond = sync.Cond{L: &f.mu}
 	return f
@@ -183,62 +178,6 @@ func newFragment(path, index, field, view string, shard uint64, flags byte) *fra
 // cachePath returns the path to the fragment's cache data.
 func (f *fragment) cachePath() string { return f.path + cacheExt }
 
-// newSnapshotQueue makes a new snapshot queue, of depth N, and spawns a
-// goroutine for it.
-func newSnapshotQueue(n int, w int, l logger.Logger) chan *fragment {
-	ch := make(chan *fragment, n)
-	for i := 0; i < w; i++ {
-		go snapshotQueueWorker(ch, l)
-	}
-	return ch
-}
-
-func snapshotQueueWorker(snapshotQueue chan *fragment, l logger.Logger) {
-	for f := range snapshotQueue {
-		err := f.protectedSnapshot(true)
-		if err != nil {
-			l.Printf("snapshot error: %v", err)
-		}
-		f.snapshotCond.Broadcast()
-	}
-}
-
-// enqueueSnapshot requests that the fragment be snapshotted at some point
-// in the future, if this has not already been requested. Call this only when
-// the mutex is held.
-func (f *fragment) enqueueSnapshot() {
-	f.snapshotsRequested++
-	if f.snapshotting {
-		return
-	}
-	f.snapshotting = true
-	if f.snapshotQueue != nil {
-		select {
-		case f.snapshotQueue <- f:
-		default:
-			before := time.Now()
-			// wait forever, but notice that we're waiting
-			f.snapshotQueue <- f
-			f.snapshotDelays++
-			f.snapshotDelayTime += time.Since(before)
-			if f.snapshotDelays >= 10 {
-				f.Logger.Printf("snapshotting %s: last ten enqueue delays took %v", f.path, f.snapshotDelayTime)
-				f.snapshotDelays = 0
-				f.snapshotDelayTime = 0
-			}
-		}
-	} else {
-		// in testing, for instance, there may be no holder, thus no one
-		// to handle these snapshots.
-		err := f.snapshot()
-		if err != nil {
-			f.Logger.Printf("snapshot failed: %v", err)
-		}
-		f.snapshotting = false
-		f.snapshotCond.Broadcast()
-	}
-}
-
 // Open opens the underlying storage.
 func (f *fragment) Open() error {
 	f.mu.Lock()
@@ -254,6 +193,10 @@ func (f *fragment) Open() error {
 		// Fill cache with rows persisted to disk.
 		f.Logger.Debugf("open cache for index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
 		if err := f.openCache(); err != nil {
+			e2 := f.closeStorage()
+			if e2 != nil {
+				return errors.Wrapf(err, "closing storage: %v, after opening cache", e2)
+			}
 			return errors.Wrap(err, "opening cache")
 		}
 
@@ -273,48 +216,103 @@ func (f *fragment) Open() error {
 	return nil
 }
 
-func (f *fragment) reopen() (mustClose bool, err error) {
-	if f.file == nil {
-		// Open the data file to be mmap'd and used as an ops log.
-		f.file, mustClose, err = syswrap.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
-		if err != nil {
-			return mustClose, fmt.Errorf("open file: %s", err)
-		}
-		f.storage.OpWriter = f.file
+// emptyStorage is the common case for importStorage/applyStorage where they
+// get no data. It tries to write the current storage to the provided file,
+// which is assumed to be the file they didn't get any data from.
+func (f *fragment) emptyStorage(file *os.File) (bool, error) {
+	// No data. We'll mark this for no mapping, clear any existing
+	// mapped containers, and set the Source to nil. We also have no
+	// ops.
+	f.opN = 0
+	f.ops = 0
+	f.storage.SetOps(0, 0)
+
+	f.storage.PreferMapping(false)
+	_, err := f.storage.RemapRoaringStorage(nil)
+	f.storage.SetSource(nil)
+	if err != nil {
+		return false, fmt.Errorf("applying/importing storage: no data, and clearing old mapping also failed: %v", err)
 	}
-	return mustClose, nil
+	// Write the existing storage out to the file so it's
+	// a valid Roaring file thereafter. nothing to unmarshal.
+	// In the unlikely event that this happened even though we
+	// had significant data, we're not mapping it, but that's
+	// harmless even if it's not maximally efficient.
+	bi := bufio.NewWriter(file)
+	if _, err = f.storage.WriteTo(bi); err != nil {
+		return false, fmt.Errorf("init storage file: %s", err)
+	}
+	bi.Flush()
+	return false, nil
 }
 
-// openStorage opens the storage bitmap. Usually you also want to read in
-// the storage, but in the case where we just wrote that file, such as
-// unprotectedWriteToFragment, we could also just... not. If we didn't
-// have existing storage, we probably need to unmarshal the data. If the
-// file we're asked to open is empty, we probably don't.
-//
-// If we already had mapped storage previously, we want to unmap that, and
-// possibly remap it from the file, but we don't need a full unmarshal, just
-// an update of mapped pointers.
-//
-// unmarshalData is somewhat overloaded. it tells us whether or not we
-// need to actually create a bitmap from the data (if the data exists to
-// do this from).
-//
-// usually unmarshalData is only set to false when we're in the middle of
-// a snapshot, and unprotectedWriteToFragment just wrote the in-memory data
-// out.
-//
-// If we have existing storage data, and we successfully get new data,
-// we will unmap the existing storage data.
-//
-// This function's design is probably a problem -- it is trying to handle
-// both cases where there was existing data before, and cases where we
-// just wrote the data.
-func (f *fragment) openStorage(unmarshalData bool) error {
-	oldStorageData := f.storageData
-	// there's a few places where we might encounter an error, but need
-	// to continue past it through other error checks, before returning it.
-	var lastError error
+// importStorage attempts to import data from storage -- for instance,
+// reading in a roaring bitmap from media.
+func (f *fragment) importStorage(data []byte, file *os.File, newGen generation, mapped bool) (bool, error) {
+	f.storage.PreferMapping(mapped)
+	if len(data) == 0 {
+		return f.emptyStorage(file)
+	}
 
+	// UnmarshalBinary will have remapped the storage to newGen if it
+	// succeeded, or if it fails but the error is advisory-only. So we
+	// optimistically set the source here, but if there's a non-advisory
+	// error, we'll unmap it and then set the source to nil.
+	f.storage.SetSource(newGen)
+	if err := f.storage.UnmarshalBinary(data); err != nil {
+		// roaring can report advisory-only errors...
+		cause := errors.Cause(err)
+		_, ok := cause.(roaring.AdvisoryError)
+		if !ok {
+			_, e2 := f.storage.RemapRoaringStorage(nil)
+			f.storage.SetSource(nil)
+			if e2 != nil {
+				return false, fmt.Errorf("unmarshal storage: file=%s, err=%s, clearing old mapping also failed: %v", file.Name(), err, e2)
+			}
+			return false, fmt.Errorf("unmarshal storage: file=%s, err=%s", file.Name(), err)
+		}
+		f.Logger.Printf("warning: unmarshal storage, file=%s, err=%v", file.Name(), err)
+		trunc, ok := cause.(roaring.FileShouldBeTruncatedError)
+		if ok {
+			// generation code looks for a FileShouldBeTruncatedError
+			return false, trunc
+		}
+	}
+	f.ops, f.opN = f.storage.Ops()
+	// For now, we assume that UnmarshalBinary will have mapped at least
+	// one container if we told it the storage was mapped and it didn't
+	// error out. This might be wrong in occasional trivial cases, but
+	// it should be harmless.
+	return mapped, nil
+}
+
+// applyStorage applies storage to a fragment that may already have
+// usable data. For instance, this would try to remap existing containers
+// to use a new storage as backing store.
+func (f *fragment) applyStorage(data []byte, file *os.File, newGen generation, mapped bool) (bool, error) {
+	if len(data) == 0 {
+		return f.emptyStorage(file)
+	}
+	// Tell storage to prefer mapping if and only if we think the data
+	// is mmapped and valid.
+	f.storage.PreferMapping(mapped)
+	f.storage.SetSource(newGen)
+	// RemapRoaringStorage will fix any mapped containers to point either
+	// to the provided data (if PreferMapping was called with true and
+	// data is provided and there's a corresponding container) or to
+	// allocated storage, so when it's done, there's nothing in it that
+	// is mapped to anything *other than* the provided data.
+	return f.storage.RemapRoaringStorage(data)
+}
+
+// openStorage opens the storage bitmap.
+//
+// This has been massively reworked recently, and now hands a lot of
+// file management off to the generation object and the Done method
+// of that object. Similarly, the bitmap mapping/remapping
+// logic is now mostly in importStorage (reading in a bitmap) and applyStorage
+// (remapping an existing bitmap to match a new backing store).
+func (f *fragment) openStorage(unmarshalData bool) error {
 	// Create a roaring bitmap to serve as storage for the shard.
 	if f.storage == nil {
 		f.storage = roaring.NewFileBitmap()
@@ -323,139 +321,23 @@ func (f *fragment) openStorage(unmarshalData bool) error {
 		// unmarshal this data in order to have any.
 		unmarshalData = true
 	}
-	// Open the data file to be mmap'd and used as an ops log.
-	file, mustClose, err := syswrap.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
-	if err != nil {
-		return fmt.Errorf("open file: %s", err)
-	}
-	f.file = file
-	if mustClose {
-		defer f.safeClose()
-	}
-
-	// Lock the underlying file.
-	if err := syscall.Flock(int(f.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
-		return fmt.Errorf("flock: %s", err)
-	}
-
-	// data is the data we would unmarshal from, if we're unmarshalling; it might
-	// be obtained by calling ReadAll on a file.
-	//
-	// newStorageData is the data we should map things to. it is set only if
-	// mmapped; if we didn't mmap (say, we couldn't), we won't want to unmap
-	// the ioutil byte slice. (Theoretically, we shouldn't be using the mapped
-	// flag in that case...)
-	var data []byte
-	var newStorageData []byte
-
-	// If the file is empty then initialize it with an empty bitmap.
-	fi, err := f.file.Stat()
-	if err != nil {
-		return errors.Wrap(err, "statting file before")
-	} else if fi.Size() == 0 {
-		bi := bufio.NewWriter(f.file)
-		var err error
-		if _, err = f.storage.WriteTo(bi); err != nil {
-			return fmt.Errorf("init storage file: %s", err)
-		}
-		bi.Flush()
-		_, err = f.file.Stat()
-		if err != nil {
-			return errors.Wrap(err, "statting file after")
-		}
-		// there's nothing here, we're not going to try to unmarshal it.
-		unmarshalData = false
-		f.rowCache = &simpleCache{make(map[uint64]*Row)}
-	} else {
-		// Mmap the underlying file so it can be zero copied.
-		data, err = syswrap.Mmap(int(f.file.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
-		if err == syswrap.ErrMaxMapCountReached {
-			f.Logger.Debugf("maximum number of maps reached, reading file instead")
-			if unmarshalData {
-				data, err = ioutil.ReadAll(file)
-				if err != nil {
-					return errors.Wrap(err, "failure file readall")
-				}
-			}
-		} else if err != nil {
-			return errors.Wrap(err, "mmap failed")
-		} else {
-			newStorageData = data
-		}
-	}
-
+	f.rowCache = &simpleCache{make(map[uint64]*Row)}
+	var storageOp func([]byte, *os.File, generation, bool) (bool, error)
 	if unmarshalData {
-		f.storageData = newStorageData
-		// We're about to either re-read the bitmap, or fail to do so
-		// and unconditionally unmap the existing stuff. Either way, we
-		// want to unmap the old storage data after we're done here, but
-		// we can't unmap it yet because it's still live until sometime
-		// later, but we can't unmap it later, because we could return
-		// early... this is what defer is for.
-		if oldStorageData != nil {
-			defer func() {
-				unmapErr := syswrap.Munmap(oldStorageData)
-				if unmapErr != nil {
-					f.Logger.Printf("unmap of old storage failed: %s", err)
-				}
-			}()
-		}
-		// set the preference for mapping based on whether the data's mmapped
-		f.storage.PreferMapping(newStorageData != nil)
-		// so we have a problem here: if this fails, it's unclear whether
-		// *either* or *both* of old and new storage data might be in use.
-		// So we call the thing that should unconditionally unmap both of them...
-		if err := f.storage.UnmarshalBinary(data); err != nil {
-			_, e2 := f.storage.RemapRoaringStorage(nil)
-			if e2 != nil {
-				return fmt.Errorf("unmarshal storage: file=%s, err=%s, clearing old mapping also failed: %v", f.file.Name(), err, e2)
-			}
-			return fmt.Errorf("unmarshal storage: file=%s, err=%s", f.file.Name(), err)
-		}
-		f.rowCache = &simpleCache{make(map[uint64]*Row)}
-		f.ops, f.opN = f.storage.Ops()
+		storageOp = f.importStorage
 	} else {
-		// we're moving to new storage, so instead of using the OpN
-		// derived from reading that storage, we notify the bitmap that
-		// OpN is now effectively zero.
-		f.opN = 0
-		f.ops = 0
-		f.storage.SetOps(0, 0)
-		// if oldStorageData is nil, this just tries to unmap any bits that
-		// are currently mapped. otherwise, it will point them at this
-		// storage (if the containers match).
-		var mappedAny bool
-		mappedAny, lastError = f.storage.RemapRoaringStorage(newStorageData)
-		if oldStorageData != nil {
-			unmapErr := syswrap.Munmap(oldStorageData)
-			if unmapErr != nil {
-				f.Logger.Printf("unmap of old storage failed: %s", err)
-			}
-		}
-		if mappedAny {
-			// Advise the kernel that the mmap is accessed randomly.
-			if err := madvise(newStorageData, syscall.MADV_RANDOM); err != nil {
-				lastError = fmt.Errorf("madvise: %s", err)
-			}
-		} else {
-			// if we did map data, but for some reason none of it got used
-			// as backing store, we can unmap it, and set the slice to nil,
-			// so we don't keep the now-invalid slice in f.storageData.
-			if newStorageData != nil {
-				unmapErr := syswrap.Munmap(newStorageData)
-				if unmapErr != nil {
-					lastError = fmt.Errorf("unmapping unused storage data: %s", err)
-				}
-				newStorageData = nil
-			}
-		}
-		f.storageData = newStorageData
+		storageOp = f.applyStorage
 	}
-
-	// Attach the file to the bitmap to act as a write-ahead log.
-	f.storage.OpWriter = f.file
-
-	return lastError
+	var err error
+	f.gen, err = newGeneration(f.gen, f.path, unmarshalData, storageOp, f.Logger)
+	if generationDebug {
+		// We might have already done this anyway, if we think we
+		// mapped stuff, but when debugging we want to do it
+		// unconditionally, because the test cases otherwise won't
+		// exercise this code well.
+		f.storage.SetSource(f.gen)
+	}
+	return err
 }
 
 // openCache initializes the cache from row ids persisted to disk.
@@ -504,31 +386,12 @@ func (f *fragment) openCache() error {
 func (f *fragment) Close() error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	for f.snapshotting {
+	for f.snapshotPending {
 		f.snapshotCond.Wait()
 	}
 	return f.close()
 }
 
-// awaitSnapshot lets us delay until the snapshot gets written, preventing tests
-// from misleadingly showing amazingly fast performance because the snapshots they
-// trigger haven't happened yet.
-func (f *fragment) awaitSnapshot() {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	for f.snapshotting {
-		f.snapshotCond.Wait()
-	}
-}
-
-// unprotectedAwaitSnapshot assumes you already hold the lock, and waits for
-// the snapshot fairy to come along.
-func (f *fragment) unprotectedAwaitSnapshot() {
-	for f.snapshotting {
-		f.snapshotCond.Wait()
-	}
-}
-
 func (f *fragment) close() error {
 	// Flush cache if closing gracefully.
 	if err := f.flushCache(); err != nil {
@@ -537,7 +400,7 @@ func (f *fragment) close() error {
 	}
 
 	// Close underlying storage.
-	if err := f.closeStorage(true); err != nil {
+	if err := f.closeStorage(); err != nil {
 		f.Logger.Printf("fragment: error closing storage: err=%s, path=%s", err, f.path)
 		return errors.Wrap(err, "closing storage")
 	}
@@ -548,54 +411,17 @@ func (f *fragment) close() error {
 	return nil
 }
 
-// safeClose is unprotected.
-func (f *fragment) safeClose() error {
-	// Flush file, unlock & close.
-	if f.file != nil {
-		if err := f.file.Sync(); err != nil {
-			return fmt.Errorf("sync: %s", err)
-		}
-		if err := syscall.Flock(int(f.file.Fd()), syscall.LOCK_UN); err != nil {
-			return fmt.Errorf("unlock: %s", err)
-		}
-		if err := syswrap.CloseFile(f.file); err != nil {
-			return fmt.Errorf("close file: %s", err)
-		}
-	}
-	f.file = nil
-	f.storage.OpWriter = nil
-
-	return nil
-}
-
-// closeStorage attempts to close storage, including unmapping the old
-// storage if includeMap is true. This would normally make sense if you're
-// expecting to be done using the fragment, or to reload it. But it's also
-// okay to just leave stuff mmapped; you don't have to keep the file
-// descriptor open. So in some cases, we'll just leave the old mmapping
-// in place, rather than regenerating everything from the new file.
-func (f *fragment) closeStorage(includeMap bool) error {
-	// Clear the storage bitmap so it doesn't access the closed mmap.
-
-	//f.storage = roaring.NewBitmap()
-
-	// Unmap the file.
-	if includeMap && f.storageData != nil {
-		if err := syswrap.Munmap(f.storageData); err != nil {
-			return fmt.Errorf("munmap: %s", err)
-		}
-		f.storageData = nil
-	}
-
-	if err := f.safeClose(); err != nil {
-		return err
-	}
-
+// closeStorage marks the current generation as done. It is not necessary
+// to call this before openStorage.
+func (f *fragment) closeStorage() error {
 	// opN is determined by how many bit set/clear operations are in the storage
 	// write log, so once the storage is closed it should be 0. Opening new
 	// storage will set opN appropriately.
 	f.opN = 0
 
+	if f.gen != nil {
+		f.gen.Done()
+	}
 	return nil
 }
 
@@ -648,22 +474,17 @@ func (f *fragment) rowFromStorage(rowID uint64) *Row {
 func (f *fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	mustClose, err := f.reopen()
-	if err != nil {
-		return false, errors.Wrap(err, "reopening")
-	}
-	if mustClose {
-		defer f.safeClose()
-	}
-
-	// handle mutux field type
-	if f.mutexVector != nil {
-		if err := f.handleMutex(rowID, columnID); err != nil {
-			return changed, errors.Wrap(err, "handling mutex")
+	err = f.gen.Transaction(&f.storage.OpWriter, func() error {
+		// handle mutux field type
+		if f.mutexVector != nil {
+			if err := f.handleMutex(rowID, columnID); err != nil {
+				return errors.Wrap(err, "handling mutex")
+			}
 		}
-	}
-
-	return f.unprotectedSetBit(rowID, columnID)
+		changed, err = f.unprotectedSetBit(rowID, columnID)
+		return err
+	})
+	return changed, err
 }
 
 // handleMutex will clear an existing row and store the new row
@@ -727,17 +548,14 @@ func (f *fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err
 
 // clearBit clears a bit for a given column & row within the fragment.
 // This updates both the on-disk storage and the in-cache bitmap.
-func (f *fragment) clearBit(rowID, columnID uint64) (bool, error) {
+func (f *fragment) clearBit(rowID, columnID uint64) (changed bool, err error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	mustClose, err := f.reopen()
-	if err != nil {
-		return false, errors.Wrap(err, "reopening")
-	}
-	if mustClose {
-		defer f.safeClose()
-	}
-	return f.unprotectedClearBit(rowID, columnID)
+	err = f.gen.Transaction(&f.storage.OpWriter, func() error {
+		changed, err = f.unprotectedClearBit(rowID, columnID)
+		return err
+	})
+	return changed, err
 }
 
 // unprotectedClearBit TODO should be replaced by an invocation of
@@ -783,17 +601,14 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er
 
 // setRow replaces an existing row (specified by rowID) with the given
 // Row. This updates both the on-disk storage and the in-cache bitmap.
-func (f *fragment) setRow(row *Row, rowID uint64) (bool, error) {
+func (f *fragment) setRow(row *Row, rowID uint64) (changed bool, err error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	mustClose, err := f.reopen()
-	if err != nil {
-		return false, errors.Wrap(err, "reopening")
-	}
-	if mustClose {
-		defer f.safeClose()
-	}
-	return f.unprotectedSetRow(row, rowID)
+	err = f.gen.Transaction(&f.storage.OpWriter, func() error {
+		changed, err = f.unprotectedSetRow(row, rowID)
+		return err
+	})
+	return changed, err
 }
 
 func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err error) {
@@ -834,7 +649,7 @@ func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err
 	f.rowCache.Add(rowID, nil)
 
 	// Snapshot storage.
-	f.enqueueSnapshot()
+	f.snapshotQueue.Enqueue(f)
 	f.stats.Count("setRow", 1, 1.0)
 
 	return changed, nil
@@ -842,17 +657,14 @@ func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err
 
 // ClearRow clears a row for a given rowID within the fragment.
 // This updates both the on-disk storage and the in-cache bitmap.
-func (f *fragment) clearRow(rowID uint64) (bool, error) {
+func (f *fragment) clearRow(rowID uint64) (changed bool, err error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	mustClose, err := f.reopen()
-	if err != nil {
-		return false, errors.Wrap(err, "reopening")
-	}
-	if mustClose {
-		defer f.safeClose()
-	}
-	return f.unprotectedClearRow(rowID)
+	err = f.gen.Transaction(&f.storage.OpWriter, func() error {
+		changed, err = f.unprotectedClearRow(rowID)
+		return err
+	})
+	return changed, err
 }
 
 func (f *fragment) unprotectedClearRow(rowID uint64) (changed bool, err error) {
@@ -878,7 +690,7 @@ func (f *fragment) unprotectedClearRow(rowID uint64) (changed bool, err error) {
 	f.rowCache.Add(rowID, nil)
 
 	// Snapshot storage.
-	f.enqueueSnapshot()
+	f.snapshotQueue.Enqueue(f)
 
 	f.stats.Count("clearRow", 1, 1.0)
 
@@ -978,67 +790,62 @@ func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64
 func (f *fragment) setValueBase(columnID uint64, bitDepth uint, value int64, clear bool) (changed bool, err error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	mustClose, err := f.reopen()
-	if err != nil {
-		return false, errors.Wrap(err, "reopening")
-	}
-	if mustClose {
-		defer f.safeClose()
-	}
+	err = f.gen.Transaction(&f.storage.OpWriter, func() error {
+		// Convert value to an unsigned representation.
+		uvalue := uint64(value)
+		if value < 0 {
+			uvalue = uint64(-value)
+		}
 
-	// Convert value to an unsigned representation.
-	uvalue := uint64(value)
-	if value < 0 {
-		uvalue = uint64(-value)
-	}
+		for i := uint(0); i < bitDepth; i++ {
+			if uvalue&(1<<i) != 0 {
+				if c, err := f.unprotectedSetBit(uint64(bsiOffsetBit+i), columnID); err != nil {
+					return err
+				} else if c {
+					changed = true
+				}
+			} else {
+				if c, err := f.unprotectedClearBit(uint64(bsiOffsetBit+i), columnID); err != nil {
+					return err
+				} else if c {
+					changed = true
+				}
+			}
+		}
 
-	for i := uint(0); i < bitDepth; i++ {
-		if uvalue&(1<<i) != 0 {
-			if c, err := f.unprotectedSetBit(uint64(bsiOffsetBit+i), columnID); err != nil {
-				return changed, err
+		// Mark value as set (or cleared).
+		if clear {
+			if c, err := f.unprotectedClearBit(uint64(bsiExistsBit), columnID); err != nil {
+				return errors.Wrap(err, "clearing not-null")
 			} else if c {
 				changed = true
 			}
 		} else {
-			if c, err := f.unprotectedClearBit(uint64(bsiOffsetBit+i), columnID); err != nil {
-				return changed, err
+			if c, err := f.unprotectedSetBit(uint64(bsiExistsBit), columnID); err != nil {
+				return errors.Wrap(err, "marking not-null")
 			} else if c {
 				changed = true
 			}
 		}
-	}
 
-	// Mark value as set (or cleared).
-	if clear {
-		if c, err := f.unprotectedClearBit(uint64(bsiExistsBit), columnID); err != nil {
-			return changed, errors.Wrap(err, "clearing not-null")
-		} else if c {
-			changed = true
+		// Mark sign bit (or clear).
+		if value >= 0 || clear {
+			if c, err := f.unprotectedClearBit(uint64(bsiSignBit), columnID); err != nil {
+				return errors.Wrap(err, "clearing sign")
+			} else if c {
+				changed = true
+			}
+		} else {
+			if c, err := f.unprotectedSetBit(uint64(bsiSignBit), columnID); err != nil {
+				return errors.Wrap(err, "marking sign")
+			} else if c {
+				changed = true
+			}
 		}
-	} else {
-		if c, err := f.unprotectedSetBit(uint64(bsiExistsBit), columnID); err != nil {
-			return changed, errors.Wrap(err, "marking not-null")
-		} else if c {
-			changed = true
-		}
-	}
 
-	// Mark sign bit (or clear).
-	if value >= 0 || clear {
-		if c, err := f.unprotectedClearBit(uint64(bsiSignBit), columnID); err != nil {
-			return changed, errors.Wrap(err, "clearing sign")
-		} else if c {
-			changed = true
-		}
-	} else {
-		if c, err := f.unprotectedSetBit(uint64(bsiSignBit), columnID); err != nil {
-			return changed, errors.Wrap(err, "marking sign")
-		} else if c {
-			changed = true
-		}
-	}
-
-	return changed, nil
+		return nil
+	})
+	return changed, err
 }
 
 // importSetValue is a more efficient SetValue just for imports.
@@ -2066,52 +1873,46 @@ func (f *fragment) bulkImportStandard(rowIDs, columnIDs []uint64, options *Impor
 // snapshot of the fragment or just do in-memory updates while appending
 // operations to the op log.
 func (f *fragment) importPositions(set, clear []uint64, rowSet map[uint64]struct{}) error {
-	mustClose, err := f.reopen()
-	if err != nil {
-		return errors.Wrap(err, "reopening")
-	}
-	if mustClose {
-		defer f.safeClose()
-	}
-
-	if len(set) > 0 {
-		f.stats.Count("ImportingN", int64(len(set)), 1)
-		changedN, err := f.storage.AddN(set...) // TODO benchmark Add/RemoveN behavior with sorted/unsorted positions
-		if err != nil {
-			return errors.Wrap(err, "adding positions")
+	err := f.gen.Transaction(&f.storage.OpWriter, func() error {
+		if len(set) > 0 {
+			f.stats.Count("ImportingN", int64(len(set)), 1)
+			changedN, err := f.storage.AddN(set...) // TODO benchmark Add/RemoveN behavior with sorted/unsorted positions
+			if err != nil {
+				return errors.Wrap(err, "adding positions")
+			}
+			f.stats.Count("ImportedN", int64(changedN), 1)
+			f.incrementOpN(changedN)
 		}
-		f.stats.Count("ImportedN", int64(changedN), 1)
-		f.incrementOpN(changedN)
-	}
 
-	if len(clear) > 0 {
-		f.stats.Count("ClearingN", int64(len(clear)), 1)
-		changedN, err := f.storage.RemoveN(clear...)
-		if err != nil {
-			return errors.Wrap(err, "clearing positions")
+		if len(clear) > 0 {
+			f.stats.Count("ClearingN", int64(len(clear)), 1)
+			changedN, err := f.storage.RemoveN(clear...)
+			if err != nil {
+				return errors.Wrap(err, "clearing positions")
+			}
+			f.stats.Count("ClearedN", int64(changedN), 1)
+			f.incrementOpN(changedN)
 		}
-		f.stats.Count("ClearedN", int64(changedN), 1)
-		f.incrementOpN(changedN)
-	}
 
-	// Update cache counts for all affected rows.
-	for rowID := range rowSet {
-		// Invalidate block checksum.
-		delete(f.checksums, int(rowID/HashBlockSize))
+		// Update cache counts for all affected rows.
+		for rowID := range rowSet {
+			// Invalidate block checksum.
+			delete(f.checksums, int(rowID/HashBlockSize))
+
+			if f.CacheType != CacheTypeNone {
+				n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth)
+				f.cache.BulkAdd(rowID, n)
+			}
+
+			f.rowCache.Add(rowID, nil)
+		}
 
 		if f.CacheType != CacheTypeNone {
-			n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth)
-			f.cache.BulkAdd(rowID, n)
+			f.cache.Recalculate()
 		}
-
-		f.rowCache.Add(rowID, nil)
-	}
-
-	if f.CacheType != CacheTypeNone {
-		f.cache.Recalculate()
-	}
-
-	return nil
+		return nil
+	})
+	return err
 }
 
 // bulkImportMutex performs a bulk import on a fragment while ensuring
@@ -2197,7 +1998,6 @@ func (f *fragment) importValueSmallWrite(columnIDs []uint64, values []int64, bit
 		}
 		return nil
 	}(); err != nil {
-		_ = f.closeStorage(true)
 		_ = f.openStorage(true)
 		return err
 	}
@@ -2245,23 +2045,23 @@ func (f *fragment) importValue(columnIDs []uint64, values []int64, bitDepth uint
 		}
 		return nil
 	}(); err != nil {
-		_ = f.closeStorage(true)
 		_ = f.openStorage(true)
 		return err
 	}
-	// We don't actually care, except we want our stats to be accurate.
-	f.incrementOpN(totalChanges)
+	// Keep stats accurate. We don't call incrementOpN here because it may
+	// or may not enqueue a request, which would then be in the queue
+	// taking up space and otherwise being a possible nuisance, when we're
+	// about to force a snapshot anyway.
+	f.opN += totalChanges
+	f.ops++
 
 	// Reset the rowCache.
 	f.rowCache = &simpleCache{make(map[uint64]*Row)}
 
-	// in theory, this should probably have happened anyway, but if enough
+	// in theory, this should probably have been queued anyway, but if enough
 	// of the bits matched existing bits, we'll be under our opN estimate, and
 	// we want to ensure that the snapshot happens.
-	f.enqueueSnapshot()
-	f.unprotectedAwaitSnapshot()
-
-	return nil
+	return f.snapshotQueue.Immediate(f)
 }
 
 // importRoaring imports from the official roaring data format defined at
@@ -2276,7 +2076,13 @@ func (f *fragment) importRoaring(ctx context.Context, data []byte, clear bool) e
 	defer f.mu.Unlock()
 	span.Finish()
 	span, ctx = tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits")
-	changed, rowSet, err := f.storage.ImportRoaringBits(data, clear, true, rowSize)
+	var changed int
+	var rowSet map[uint64]int
+	err := f.gen.Transaction(&f.storage.OpWriter, func() (err error) {
+		changed, rowSet, err = f.storage.ImportRoaringBits(data, clear, true, rowSize)
+		return err
+	})
+
 	span.Finish()
 	if err != nil {
 		return err
@@ -2315,14 +2121,14 @@ func (f *fragment) incrementOpN(changed int) {
 	f.opN += changed
 	f.ops++
 	if f.opN > f.MaxOpN {
-		f.enqueueSnapshot()
+		f.snapshotQueue.Enqueue(f)
 	}
 }
 
 // Snapshot writes the storage bitmap to disk and reopens it. This may
 // coexist with existing background-queue snapshotting; it does not remove
 // things from the queue. You probably don't want to do this; use
-// enqueueSnapshot/awaitSnapshot.
+// the snapshotQueue's Enqueue/Await.
 func (f *fragment) Snapshot() error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -2335,25 +2141,13 @@ func track(start time.Time, message string, stats stats.StatsClient, logger logg
 	stats.Histogram("snapshot", elapsed.Seconds(), 1.0)
 }
 
-// protectedSnapshot grabs the lock and unconditionally calls snapshot(). If
-// fromQueue is true, the snapshotting state is also cleared.
-func (f *fragment) protectedSnapshot(fromQueue bool) error {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	err := f.snapshot()
-	if fromQueue {
-		f.snapshotting = false
-	}
-	return err
-}
-
 // snapshot does the actual snapshot operation. it does not check or care
-// about f.snapshotting.
+// about f.snapshotPending.
 func (f *fragment) snapshot() error {
-	f.totalOpN += int64(f.opN)
-	f.totalOps += int64(f.ops)
-	f.snapshotsTaken++
 	_, err := unprotectedWriteToFragment(f, f.storage)
+	if err == nil {
+		f.snapshotStamp = time.Now()
+	}
 	return err
 }
 
@@ -2370,22 +2164,24 @@ func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) (n int64, err e
 	if err != nil {
 		return n, fmt.Errorf("create snapshot file: %s", err)
 	}
-	defer file.Close()
+	// No deferred close, because we want to close it sooner than the
+	// end of this function.
 
 	// Write storage to snapshot.
 	bw := bufio.NewWriter(file)
 	if n, err = bm.WriteTo(bw); err != nil {
+		file.Close()
 		return n, fmt.Errorf("snapshot write to: %s", err)
 	}
 
 	if err := bw.Flush(); err != nil {
+		file.Close()
 		return n, fmt.Errorf("flush: %s", err)
 	}
 
-	// Close current storage.
-	if err := f.closeStorage(false); err != nil {
-		return n, fmt.Errorf("close storage: %s", err)
-	}
+	// we close the file here so we don't still have it open when trying
+	// to open it in a moment.
+	file.Close()
 
 	// Move snapshot to data file location.
 	if err := os.Rename(snapshotPath, f.path); err != nil {
@@ -2585,11 +2381,6 @@ func (f *fragment) readStorageFromArchive(r io.Reader) error {
 		return errors.Wrap(err, "copying")
 	}
 
-	// Close current storage.
-	if err := f.closeStorage(true); err != nil {
-		return errors.Wrap(err, "closing")
-	}
-
 	// Move snapshot to data file location.
 	if err := os.Rename(path, f.path); err != nil {
 		return errors.Wrap(err, "renaming")

fragment_internal_test.go

@@ -1397,6 +1397,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
 
 	// Read into another fragment.
 	f1 := mustOpenFragment("i", "f", viewStandard, 0, "")
+	defer f1.Clean(t)
 	if rn, err := f1.ReadFrom(&buf); err != nil {
 		t.Fatal(err)
 	} else if wn != rn {
@@ -2069,7 +2070,11 @@ func BenchmarkImportRoaring(b *testing.B) {
 					b.StartTimer()
 					err := f.importRoaringT(data, false)
 					if err != nil {
-						f.awaitSnapshot()
+						// we don't actually particularly
+						// care whether this succeeds,
+						// but if it's happening we want
+						// it to be done.
+						_ = f.snapshotQueue.Await(f)
 						f.Clean(b)
 						b.Fatalf("import error: %v", err)
 					}
@@ -2108,7 +2113,9 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) {
 							j := j
 							eg.Go(func() error {
 								err := frags[j].importRoaringT(data[j], false)
-								frags[j].awaitSnapshot()
+								// error unimportant if it happened, but we want
+								// any snapshots to have finished.
+								_ = frags[j].snapshotQueue.Await(frags[j])
 								return err
 							})
 						}
@@ -2146,11 +2153,13 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
 								// is excessive. force storage into snapshotted state, then use import
 								// to generate an op log and/or snapshot.
 								_, _, err := frags[j].storage.ImportRoaringBits(data, false, false, 0)
-								frags[j].enqueueSnapshot()
-								frags[j].awaitSnapshot()
 								if err != nil {
 									b.Fatalf("importing roaring: %v", err)
 								}
+								err = frags[j].snapshotQueue.Immediate(frags[j])
+								if err != nil {
+									b.Fatalf("snapshot after import: %v", err)
+								}
 							}
 							eg := errgroup.Group{}
 							b.StartTimer()
@@ -2158,7 +2167,10 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
 								j := j
 								eg.Go(func() error {
 									err := frags[j].importRoaringT(updata, false)
-									frags[j].awaitSnapshot()
+									err2 := frags[j].snapshotQueue.Await(frags[j])
+									if err == nil {
+										err = err2
+									}
 									return err
 								})
 							}
@@ -2220,20 +2232,38 @@ func BenchmarkImportRoaringUpdate(b *testing.B) {
 						// is excessive. force storage into snapshotted state, then use import
 						// to generate an op log and/or snapshot.
 						_, _, err := f.storage.ImportRoaringBits(data, false, false, 0)
-						f.enqueueSnapshot()
-						f.awaitSnapshot()
 						if err != nil {
 							b.Errorf("import error: %v", err)
 						}
+						err = f.snapshotQueue.Immediate(f)
+						if err != nil {
+							b.Errorf("snapshot after import error: %v", err)
+						}
 						b.StartTimer()
 						err = f.importRoaringT(updata, false)
-						f.awaitSnapshot()
 						if err != nil {
 							f.Clean(b)
 							b.Errorf("import error: %v", err)
 						}
+						err = f.snapshotQueue.Await(f)
+						if err != nil {
+							b.Errorf("snapshot after import error: %v", err)
+						}
 						b.StopTimer()
-						stat, _ := f.file.Stat()
+						var stat os.FileInfo
+						var statTarget io.Writer
+						err = f.gen.Transaction(&statTarget, func() error {
+							targetFile, ok := statTarget.(*os.File)
+							if ok {
+								stat, _ = targetFile.Stat()
+							} else {
+								b.Errorf("couldn't stat file")
+							}
+							return nil
+						})
+						if err != nil {
+							b.Errorf("transaction error: %v", err)
+						}
 						fileSize[name] = stat.Size()
 						f.Clean(b)
 					}
@@ -2382,6 +2412,7 @@ func TestGetZipfRowsSliceRoaring(t *testing.T) {
 			t.Fatalf("suspect distribution from getZipfRowsSliceRoaring")
 		}
 	}
+	f.Clean(t)
 }
 
 // getZipfRowsSliceRoaring generates a random fragment with the given number of
@@ -2527,16 +2558,28 @@ func (f *fragment) sanityCheck(t testing.TB) {
 }
 
 func (f *fragment) Clean(t testing.TB) {
-	f.awaitSnapshot()
+	f.mu.Lock()
+	err := f.snapshotQueue.Await(f)
+	f.mu.Unlock()
+	if err != nil {
+		t.Fatalf("snapshot failed before sanity check: %v", err)
+	}
 	f.sanityCheck(t)
+	if f.storage != nil && f.storage.Source != nil {
+		if f.storage.Source.Dead() {
+			t.Fatalf("cleaning up fragment %s, source %s, source already dead", f.path, f.storage.Source.ID())
+		}
+	}
 	errc := f.Close()
+	// prevent double-closes of generation during testing.
+	f.gen = nil
 	errf := os.Remove(f.path)
 	errp := os.Remove(f.cachePath())
 	if errc != nil || errf != nil {
 		t.Fatal("cleaning up fragment: ", errc, errf, errp)
 	}
 	if f.snapshotQueue != nil {
-		close(f.snapshotQueue)
+		f.snapshotQueue.Stop()
 		f.snapshotQueue = nil
 	}
 	// not all fragments have cache files
@@ -2559,7 +2602,7 @@ func (f *fragment) CleanKeep(t testing.TB) {
 		t.Fatal("closing fragment: ", errc, errp)
 	}
 	if f.snapshotQueue != nil {
-		close(f.snapshotQueue)
+		f.snapshotQueue.Stop()
 		f.snapshotQueue = nil
 	}
 	// not all fragments have cache files
@@ -3047,10 +3090,10 @@ func TestFragmentRowIterator(t *testing.T) {
 
 func TestUnionInPlaceMapped(t *testing.T) {
 	f := mustOpenFragment("i", "f", "v", 0, CacheTypeNone)
+	// note: clean has to be deferred first, because it has to run with
+	// the lock *not* held, because it is sometimes so it has to grab the
+	// lock...
 	defer f.Clean(t)
-	// I know this doesn't actually matter in our current context, but
-	// strictly speaking, we do say you have to hold the lock while calling
-	// unprotectedWriteToFragment...
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	r0 := rand.New(rand.NewSource(2))
@@ -3083,8 +3126,15 @@ func TestUnionInPlaceMapped(t *testing.T) {
 	f.storage.UnionInPlace(setBM1)
 	countUnion := f.storage.Count()
 	// UnionInPlace produces no ops log, we have to make it snapshot, to
-	// ensure that the on-disk representation is correct.
-	f.enqueueSnapshot()
+	// ensure that the on-disk representation is correct. Note, UIP is
+	// not used for things that are modifying real fragments, usually;
+	// it's used only in computation of things that usually don't go to
+	// disk, which is why we handle this specially in testing and not
+	// generically.
+	err = f.snapshotQueue.Immediate(f)
+	if err != nil {
+		t.Fatalf("snapshot after union-in-place: %v", err)
+	}
 
 	if count0 != countF {
 		t.Fatalf("writing bitmap to storage changed count: %d => %d", count0, countF)
@@ -3272,7 +3322,7 @@ func TestImportClearRestart(t *testing.T) {
 				f2.MaxOpN = maxOpN
 				f2.CacheType = f.CacheType
 
-				err = f.closeStorage(true)
+				err = f.closeStorage()
 				if err != nil {
 					t.Fatalf("closing storage: %v", err)
 				}
@@ -3306,7 +3356,7 @@ func TestImportClearRestart(t *testing.T) {
 				f3.MaxOpN = maxOpN
 				f3.CacheType = f.CacheType
 
-				err = f2.closeStorage(true)
+				err = f2.closeStorage()
 				if err != nil {
 					t.Fatalf("f2 closing storage: %v", err)
 				}
@@ -3354,6 +3404,7 @@ func check(t *testing.T, f *fragment, exp map[uint64]map[uint64]struct{}) {
 
 func TestImportValueConcurrent(t *testing.T) {
 	f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0)
+	defer f.Clean(t)
 	eg := &errgroup.Group{}
 	for i := 0; i < 4; i++ {
 		i := i

generation.go

@@ -0,0 +1,407 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"runtime"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/syswrap"
+	"github.com/pkg/errors"
+)
+
+// generation represents one "generation" of opening a data file.
+// This is what determines when it's safe to unmap a data file, if it
+// got mapped, and handles closing/reopening files if we need to
+// manage file handle availability. It's an interface because this
+// lets us write simpler code for specific cases, rather than handling
+// the whole matrix of mapped/unmapped, staying open/being reopened,
+// etcetera.
+//
+// You create a generation by calling newGeneration with a file
+// path. If it succeeds in opening that path, it calls a provided
+// setup function with the data from the generation, and a flag
+// indicating whether the data is mmapped. If the setup function
+// fails, newGeneration cleans things up and closes. Otherwise,
+// it returns a generation.
+//
+// The generation itself uses runtime.SetFinalizer to clean up when
+// the last reference to it goes away. You should store a pointer
+// to the generation in any object which is reliant on the generation.
+//
+// When you anticipate a generation should be done (for instance,
+// opening a new generation), the old one gets marked done, which
+// stashes a timestamp in it. Later operations can check whether
+// the timestamp is a while back, and if so, complain that something
+// might be wrong.
+//
+// In some cases, we don't have enough open file limit to keep every
+// file actually open. To address this, use the `Transaction` function,
+// which ensures that the file is open, stores a reference to it in
+// a provided `*io.Writer`, and then restores the previous value of
+// the io.Writer when it's done. For instance, for a bitmap, this might
+// be used with `&b.OpWriter`.
+//
+// newGeneration takes an optional previous generation; it calls
+// that generation's Done function after running the provided setup,
+// and bumps the generation count.
+type generation interface {
+	// Transaction runs the given transaction with the generation's
+	// file open. If the **os.File parameter is
+	// non-nil, the generation's file will be open, and stored
+	// into that pointer, during the execution of func, after
+	// which the previous contents are restored. Otherwise
+	// the file may or may not be open during the operation.
+	Transaction(*io.Writer, func() error) error
+	// Done() should be called exactly once, to indicate that a
+	// generation is expected not to be in use for long -- for instance,
+	// when a new generation replaces it.
+	Done()
+	// Generation count.
+	Generation() int64
+	// ID indicates the source -- path and generation number -- that
+	// this generation represents.
+	ID() string
+	Dead() bool
+}
+
+type mmapGeneration struct {
+	mu         sync.Mutex // mutex guards modifiers of generation, not of data
+	transMu    sync.Mutex // guards transactions, specifically
+	path       string
+	id         string
+	file       *os.File
+	data       []byte
+	generation int64     // generation counter
+	dead       bool      // we think this generation is dead
+	deadSince  time.Time // when this generation was marked dead
+	retries    int       // for cases where we're retrying
+	logger     logger.Logger
+}
+
+func (m *mmapGeneration) Dead() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.dead
+}
+
+func (m *mmapGeneration) ID() string {
+	return m.id
+}
+
+func (m *mmapGeneration) Generation() int64 {
+	return m.generation
+}
+
+// Transaction runs an exclusive call, ensuring that the file is open if
+// the *io.Writer parameter is present.
+func (m *mmapGeneration) Transaction(fileP *io.Writer, fn func() error) (transactionErr error) {
+	m.transMu.Lock()
+	defer m.transMu.Unlock()
+	// HEY LOOK CAREFULLY AT THIS BIT:
+	// We can't just defer this unlock. We specifically want to be
+	// sure to unlock the regular mutex *before* this function is over,
+	// and if we error out trying to open the file, we want to do it
+	// even sooner. If we deferred this, the transaction would block
+	// *everything*, including things like sanity checks against the
+	// generation being Dead(), but also including the deferred
+	// re-close-the-file.
+	m.mu.Lock()
+	// if we've been asked for a file pointer, we need to ensure that
+	// our file is open, and that the file pointer to it is stored in
+	// the requested location, then revert that when we're done.
+	// if we aren't asked for a file pointer, nothing needs the file
+	// open.
+	if m.dead {
+		elapsed := time.Since(m.deadSince)
+		m.logger.Printf("WARNING: transaction against %s, which has been dead for %v\n", m.id, elapsed)
+	}
+	if fileP != nil {
+		if m.file == nil {
+			// we ignore the shouldClose response here; if this
+			// fragment was previously not being kept open, we're
+			// going to stick with that.
+			_, err := m.openFile()
+			if err != nil {
+				m.mu.Unlock()
+				return err
+			}
+			defer func() {
+				// report a close error if we have no other error to report
+				m.mu.Lock()
+				defer m.mu.Unlock()
+				err := m.closeFile()
+				if transactionErr == nil {
+					transactionErr = err
+				}
+			}()
+		}
+		var fileStash io.Writer
+		fileStash, *fileP = *fileP, m.file
+		defer func() {
+			*fileP = fileStash
+		}()
+	}
+	// We are done locking the generation itself for now.
+	m.mu.Unlock()
+	return fn()
+}
+
+// Done marks the generation done, and closes its file, but may not unmap it.
+// It's still conceptually possible to end up doing a Transaction against a
+// done generation, but it's a red flag.
+func (m *mmapGeneration) Done() {
+	if m == nil {
+		return
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.dead {
+		oops := fmt.Sprintf("generation %s, marked done again at %v, previously marked dead at %v",
+			m.id, time.Now(), m.deadSince)
+		panic(oops)
+	}
+	m.dead = true
+	m.deadSince = time.Now()
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("error closing generation %s: %v", m.id, err)
+	}
+	// If we're not debugging, the finalizer won't have been enabled
+	// previously. Finalizers have non-zero cost, so having them not be
+	// created until they're needed seems rewarding?
+	if !generationDebug {
+		runtime.SetFinalizer(m, generationFinalizer)
+	}
+	endGeneration(m.id)
+	// note, Done() doesn't close the file; only the finalizer actually
+	// does the shutdown.
+}
+
+// Try to close the file if it's currently open.
+func (m *mmapGeneration) closeFile() error {
+	var lastErr error
+	// report the most serious error encountered, but still close
+	// file even if something else failed.
+	if m.file != nil {
+		if err := m.file.Sync(); err != nil {
+			lastErr = fmt.Errorf("sync: %s", err)
+		}
+		if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_UN); err != nil {
+			lastErr = fmt.Errorf("unlock: %s", err)
+		}
+		if err := syswrap.CloseFile(m.file); err != nil {
+			lastErr = fmt.Errorf("close file: %s", err)
+		}
+		m.file = nil
+	}
+	return lastErr
+}
+
+// openFile ensures the file is open and locked, or fails. If it does
+// open the file, it will also report the "you need to close this file
+// when you're done" flag from syswrap.
+func (m *mmapGeneration) openFile() (shouldClose bool, err error) {
+	if m.file != nil {
+		return false, nil
+	}
+	m.file, shouldClose, err = syswrap.OpenFile(m.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		return false, err
+	}
+	// do we actually want this in every openFile? I don't know.
+	if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		m.file.Close()
+		m.file = nil
+		return false, fmt.Errorf("flock: %s", err)
+	}
+	return shouldClose, nil
+}
+
+func generationFinalizer(m *mmapGeneration) {
+	m.mu.Lock()
+	if !m.dead {
+		m.logger.Printf("finalizing generation %s which isn't dead yet\n",
+			m.id)
+	}
+	m.mu.Unlock()
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("finalizing generation, closing file: %v\n", err)
+	}
+	if m.data != nil {
+		err := syswrap.Munmap(m.data)
+		if err != nil {
+			m.logger.Printf("finalizing generation, munmap: %v\n", err)
+		}
+		m.data = nil
+	}
+	finalizeGeneration(m.id)
+}
+
+// Cancel closes a generation out entirely. It cancels any finalizer,
+// unmaps any data, ends generation tracking, and closes any files.
+// It does each of these separately whether or not the others need to be done,
+// or succeed. It's used to handle failures from newGeneration; it makes sure
+// the generation isn't holding any resources and doesn't need to be cleaned
+// up otherwise.
+//
+// Mostly a helper function because there's several cases where newGeneration
+// might fail.
+func (m *mmapGeneration) Cancel() {
+	if m.data != nil {
+		_ = syswrap.Munmap(m.data)
+		m.data = nil
+	}
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("error cancelling generation %s: %v", m.id, err)
+	}
+	runtime.SetFinalizer(m, nil)
+	m.dead = true
+	m.deadSince = time.Now()
+	cancelGeneration(m.id)
+}
+
+// newGeneration creates a new generation using the given file path. It
+// then calls the provided setup function with the allocated storage, a
+// file handle, the new generation, and a flag indicatting whether the storage
+// is memory-mapped. If the setup function returns a non-nil error, the
+// generation is cleaned up, and newGeneration fails. The setup function
+// also returns a boolean indicating whether it used the mapping; if it
+// didn't, newGeneration discards the mapping and returns a nil generation.
+//
+// If generationDebug is enabled, we track the generation even if no mapping
+// is actually in use, so we can verify that the tracking is working.
+//
+// On failure, newGeneration returns nil values for generation and func,
+// and an error. On success, the func returned is the close func to use
+// when the generation is no longer needed by the caller.
+func newGeneration(existing generation, path string, readData bool, setup func([]byte, *os.File, generation, bool) (bool, error), logger logger.Logger) (generation, error) {
+	m := mmapGeneration{path: path, logger: logger}
+	if existing != nil {
+		m.generation = existing.Generation() + 1
+		// we might keep a previous generation around just for its generation count.
+		if !existing.Dead() {
+			defer existing.Done()
+		}
+	}
+	shouldClose, err := m.openFile()
+	if err != nil {
+		return nil, err
+	}
+	m.id = fmt.Sprintf("%s:%d", m.path, m.generation)
+	// possibly assign new generation ID if this one's been used, which can
+	// happen with reopens, especially during testing.
+	m.id = registerGeneration(m.id)
+	// if debugging, we always want the finalizer on so we notice if a
+	// generation is finalized without being closed. for non-debugging
+	// use, we only need it when the generation is closed.
+	if generationDebug {
+		runtime.SetFinalizer(&m, generationFinalizer)
+	}
+	// Mmap the underlying file so it can be zero copied.
+	var mapped bool
+	var data []byte
+	fi, err := m.file.Stat()
+	if err == nil && fi.Size() > 0 {
+		data, err = syswrap.Mmap(int(m.file.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
+		if err == syswrap.ErrMaxMapCountReached {
+			// I have no idea where/how to display this message.
+			m.logger.Printf("maximum number of maps reached, reading file '%s' instead", m.path)
+		} else if err != nil {
+			m.Cancel()
+			return nil, errors.Wrap(err, "mmap failed")
+		} else {
+			mapped = true
+		}
+	}
+	if data == nil && readData {
+		data, err = ioutil.ReadAll(m.file)
+		if err != nil {
+			m.Cancel()
+			return nil, errors.Wrap(err, "failure file readall")
+		}
+	}
+	// if we got here, data's the expected data, so let's try to use it
+	mappedAny, err := setup(data, m.file, &m, mapped)
+
+	// if the setup failed, we unmap data if we previously mapped it,
+	// and exit. Note that having no data, or having only trivial
+	// data (like a zero-container Roaring file) isn't "failed".
+	if err != nil {
+		m.Cancel()
+		// Unless, that is, we think the file probably ought to
+		// be truncated: For instance, if a bitmap has a corrupted
+		// ops log, we could truncate that part of it and retry.
+		if err, ok := err.(roaring.FileShouldBeTruncatedError); ok && m.retries < 1 {
+			m.logger.Printf("file %s read partially, but should-be-truncated at %d bytes\n", m.path, err.SuggestedLength())
+			// close this generation, then try again. once.
+			m.retries++
+			err := os.Truncate(m.path, err.SuggestedLength())
+			if err != nil {
+				m.logger.Printf("truncating file failed [but retrying anyway]: %v\n", err)
+			}
+			return newGeneration(&m, path, readData, setup, logger)
+		}
+		return nil, err
+	}
+
+	if mapped {
+		// when generationDebug is on, we want to track this even
+		// if it's not being used.
+		if generationDebug || mappedAny {
+			// Advise the kernel that the mmap is accessed randomly.
+			// We don't care much about errors with this.
+			_ = madvise(data, syscall.MADV_RANDOM)
+			// store the data, so we can unmap it when this generation
+			// gets finalized.
+			m.data = data
+		} else {
+			// unmap the data and don't stash the pointer in this
+			// generation. It's not being used. This generation
+			// doesn't need to exist, yay.
+			unmapErr := syswrap.Munmap(data)
+			if unmapErr != nil {
+				m.logger.Printf("error unmapping (probably harmless): %v", unmapErr)
+			}
+		}
+	}
+	// shouldClose comes from underlying syswrap.OpenFile, which checks
+	// a count of open files to hint at us when we need to start closing
+	// files to preserve open file descriptor limit.
+	if shouldClose {
+		err := m.closeFile()
+		if err != nil {
+			m.logger.Printf("closing file to preserve open files failed: %v\n", err)
+		}
+	}
+	// It's possible that the generation has no actual data to track,
+	// because nothing's mapped, in which case there won't be any bitmap
+	// sources following this, just the fragment source. (Bitmaps won't
+	// be attached to the source unless they're actually mapped to it,
+	// or generationDebug is true). That's okay. We pay a tiny cost
+	// for the finalizer, but we also get higher confidence that it really
+	// does get cleaned up.
+	return &m, nil
+}

generation_debug.go

@@ -0,0 +1,160 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build generationdebug
+
+package pilosa
+
+import (
+	"fmt"
+	"math/rand"
+	"runtime"
+	"sort"
+	"sync"
+	"time"
+)
+
+const generationDebug = true
+
+type lifespan struct {
+	from, to, finalized time.Time
+}
+
+var knownGenerations map[string]lifespan
+var knownGenerationLock sync.Mutex
+
+var timeZero time.Time
+
+func registerGeneration(id string) string {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	if knownGenerations == nil {
+		knownGenerations = make(map[string]lifespan)
+	}
+	newSpan := lifespan{from: time.Now()}
+	origId := id
+
+	// if you have more than 65k of the same file open, maybe you have bigger
+	// problems than this.
+	for span, exists := knownGenerations[id]; exists; span, exists = knownGenerations[id] {
+		suffix := fmt.Sprintf("::%04x", rand.Int63n(65536))
+		if span.finalized != timeZero {
+			fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v, finalized %v\n",
+				id, suffix, span.from, span.to, span.finalized)
+		} else {
+			if span.to != timeZero {
+				fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v\n", id, suffix, span.from, span.to)
+			} else {
+				fmt.Printf("new generation %s: adding %s, already exists, created %v", id, suffix, span.from)
+			}
+		}
+		id = origId + suffix
+	}
+	fmt.Printf("new generation %s\n", id)
+	knownGenerations[id] = newSpan
+	return id
+}
+
+func endGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if !exists {
+		oops := fmt.Sprintf("ending generation %s: unknown", id)
+		panic(oops)
+	}
+	if span.finalized != timeZero || span.to != timeZero {
+		oops := fmt.Sprintf("ending generation %s: already died at %v, finalized at %v", id, span.to, span.finalized)
+		panic(oops)
+	}
+	span.to = time.Now()
+	knownGenerations[id] = span
+}
+
+// cancelGeneration marks the generation as finalized. In principle it's
+// only used in cases where we just started a generation but something
+// went wrong. it's not fancier than this because of the weird cases
+// where the same generation shows up again, such as when closing and
+// reopening an index so we don't know about previous instances of the
+// same files.
+func cancelGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if exists {
+		span.finalized = time.Now()
+		span.to = span.finalized
+		knownGenerations[id] = span
+	}
+}
+
+func finalizeGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if !exists {
+		oops := fmt.Sprintf("finalizing generation %s: unknown", id)
+		panic(oops)
+	}
+	if span.finalized != timeZero {
+		var oops string
+		if span.to != timeZero {
+			oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, but not dead", id, span.finalized)
+		} else {
+			oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, dead at %v", id, span.finalized, span.to)
+		}
+		panic(oops)
+	}
+	span.finalized = time.Now()
+	knownGenerations[id] = span
+}
+
+func reportGenerations() []string {
+	runtime.GC()
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	var surviving []string
+	times := make([]int64, 0, len(knownGenerations))
+	for id, span := range knownGenerations {
+		if span.to == timeZero {
+			if span.finalized == timeZero {
+				surviving = append(surviving, fmt.Sprintf("%s: %v, not ended or finalized", id, span.from))
+			} else {
+				surviving = append(surviving, fmt.Sprintf("%s: %v, finalized %v, not ended", id, span.from, span.finalized))
+			}
+		} else {
+			if span.finalized == timeZero {
+				surviving = append(surviving, fmt.Sprintf("%s: %v to %v, not finalized", id, span.from, span.to))
+			} else {
+				times = append(times, int64(span.finalized.Sub(span.to)))
+			}
+		}
+	}
+	if len(times) > 0 {
+		sort.Slice(times, func(i, j int) bool { return times[i] < times[j] })
+		var total int64
+		for _, d := range times {
+			total += d
+		}
+		var mean, median, p90, p99, worst int64
+		mean = total / int64(len(times))
+		median = times[len(times)/2]
+		p90 = times[(len(times)*9)/10]
+		p99 = times[(len(times)*99)/100]
+		worst = times[len(times)-1]
+		surviving = append(surviving, fmt.Sprintf("%d finalized spans. lag: mean %v, median %v, p90 %v, p99 %v, worst %v",
+			len(times), time.Duration(mean), time.Duration(median), time.Duration(p90), time.Duration(p99), time.Duration(worst)))
+	}
+	return surviving
+}

generation_nodebug.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !generationdebug
+
+package pilosa
+
+const generationDebug = false
+
+func registerGeneration(id string) string {
+	return id
+}
+
+func endGeneration(id string) {
+}
+
+func cancelGeneration(id string) {
+}
+
+func finalizeGeneration(id string) {
+}
+
+//lint:ignore U1000 this is conditional on a build flag, see generation_test.go.
+func reportGenerations() []string { //nolint:unused,deadcode
+	return nil
+}

generation_test.go

@@ -0,0 +1,39 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// +build generationdebug
+
+package pilosa
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+func examineResults() {
+	results := reportGenerations()
+	if len(results) > 0 {
+		fmt.Printf("generations:\n")
+		for _, res := range results {
+			fmt.Printf("  %s\n", res)
+		}
+	}
+}
+
+func TestMain(m *testing.M) {
+	ret := m.Run()
+	examineResults()
+	os.Exit(ret)
+}

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/gorilla/mux v1.7.0
 	github.com/hashicorp/memberlist v0.1.3
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pelletier/go-toml v1.2.0
 	github.com/pkg/errors v0.8.1
@@ -44,4 +45,4 @@ require (
 	modernc.org/strutil v1.0.0
 )
 
-go 1.11
+go 1.13

go.sum

@@ -86,6 +86,8 @@ github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y=
+github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
@@ -96,6 +98,7 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU=
 github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

handler.go

@@ -48,6 +48,10 @@ type QueryRequest struct {
 
 	// Should we profile this query?
 	Profile bool
+
+	// Additional data associated with the query, in cases where there's
+	// row-style inputs for precomputed values.
+	EmbeddedData []*Row
 }
 
 // QueryResponse represent a response from a processed query.

holder.go

@@ -75,7 +75,7 @@ type Holder struct {
 
 	Logger logger.Logger
 
-	snapshotQueue chan *fragment
+	snapshotQueue snapshotQueue
 
 	// Manages replication from the primary node.
 	primaryTranslateNode     *Node
@@ -167,7 +167,7 @@ func (h *Holder) Open() error {
 	// Run snapshots asynchronously. The snapshotQueue will have a background
 	// task associated with it which flushes it and waits until this channel
 	// is closed, so we should always close this channel when done.
-	h.snapshotQueue = newSnapshotQueue(100, 2, h.Logger)
+	h.snapshotQueue = newSnapshotQueue(10, 2, h.Logger)
 
 	for _, fi := range fis {
 		// Skip files or hidden directories.
@@ -203,6 +203,7 @@ func (h *Holder) Open() error {
 	go func() { defer h.wg.Done(); h.monitorCacheFlush() }()
 
 	h.Stats.Open()
+	h.snapshotQueue.ScanHolder(h)
 
 	h.opened.Close()
 	return nil
@@ -222,8 +223,7 @@ func (h *Holder) Close() error {
 		}
 	}
 	if h.snapshotQueue != nil {
-		close(h.snapshotQueue)
-		// assuming the snapshotQueueWorker has already started, this is safe.
+		h.snapshotQueue.Stop()
 		h.snapshotQueue = nil
 	}
 

holder_test.go

@@ -197,7 +197,26 @@ func TestHolder_Open(t *testing.T) {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
+	t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
+		h := test.MustOpenHolder()
+		defer h.Close()
 
+		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+			t.Fatal(err)
+		} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
+			t.Fatal(err)
+		} else if _, err := field.SetBit(0, 0, nil); err != nil {
+			t.Fatal(err)
+		} else if err := h.Holder.Close(); err != nil {
+			t.Fatal(err)
+		} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 20); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := h.Reopen(); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
 }
 
 func TestHolder_HasData(t *testing.T) {

index.go

@@ -58,7 +58,7 @@ type Index struct {
 	Stats       stats.StatsClient
 
 	logger        logger.Logger
-	snapshotQueue chan *fragment
+	snapshotQueue snapshotQueue
 
 	// Used for notifying holder when a field is added.
 	holder *Holder
@@ -462,7 +462,9 @@ func (i *Index) newField(path, name string) (*Field, error) {
 	f.Stats = i.Stats
 	f.broadcaster = i.broadcaster
 	f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data"))
-	f.snapshotQueue = i.snapshotQueue
+	if i.snapshotQueue != nil {
+		f.snapshotQueue = i.snapshotQueue
+	}
 	f.OpenTranslateStore = i.OpenTranslateStore
 	return f, nil
 }

internal/public.proto

@@ -6,6 +6,12 @@ message Row {
 	repeated uint64 Columns = 1;
 	repeated string Keys = 3;
 	repeated Attr Attrs = 2;
+	bytes Roaring = 4;
+}
+
+message SignedRow {
+	Row Pos = 1;
+	Row Neg = 2;
 }
 
 message RowIdentifiers {
@@ -61,6 +67,7 @@ message QueryRequest {
 	bool Remote = 5;
 	bool ExcludeRowAttrs = 6;
 	bool ExcludeColumns = 7;
+	repeated Row EmbeddedData = 8;
 }
 
 message QueryResponse {
@@ -79,6 +86,7 @@ message QueryResult {
 	repeated uint64 RowIDs = 7;
 	repeated GroupCount GroupCounts = 8;
 	RowIdentifiers RowIdentifiers = 9;
+	SignedRow SignedRow = 10;
 }
 
 message ImportRequest {
@@ -120,4 +128,4 @@ message ImportRoaringRequestView {
 message ImportRoaringRequest {
 	bool Clear = 1;
 	repeated ImportRoaringRequestView views = 2;
-}
+}

pql/ast.go

@@ -17,10 +17,13 @@ package pql
 import (
 	"bytes"
 	"fmt"
+	"reflect"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/pilosa/pilosa/v2/ext"
 )
 
 // Query represents a PQL query.
@@ -259,11 +262,250 @@ type callStackElem struct {
 	inList    bool
 }
 
-// Call represents a function call in the AST.
+// Some call types may require special handling, which needs to occur
+// before distributing processing to individual shards.
+type CallType byte
+
+const (
+	// Normal calls can be executed per shard.
+	PrecallNone = CallType(iota)
+	// PreCallGlobal indicates a call which must be run globally *before*
+	// distributing the call to other shards. Example: A Distinct query,
+	// where every shard could potentially produce results for any shard,
+	// so you have to produce the results up front.
+	PrecallGlobal
+	// PreCallPerNode indicates a call which needs to be run per-shard
+	// in a way that lets it be done on each shard, but where it should
+	// be done prior to spawning per-shard goroutines. Example:
+	// A cross-index query, where each local shard may or may not need
+	// to get data from a remote node, but batches of shards can
+	// probably be gotten from the same remote node.
+	PrecallPerNode
+)
+
+// Call represents a function call in the AST. The Precomputed field
+// is used by the executor to handle non-standard call types; it does
+// these by actually executing them separately, then replacing them
+// in the call tree with a new call using the special precomputed
+// type, with the Precomputed field set to a map from shards to results.
 type Call struct {
-	Name     string
-	Args     map[string]interface{}
-	Children []*Call
+	Name        string
+	Args        map[string]interface{}
+	Children    []*Call
+	Type        CallType
+	Precomputed map[uint64]interface{}
+}
+
+// callInfo defines the arguments allowed for a particular PQL call, and
+// possibly things about its semantics. If allowUnknown is true, unfamiliar
+// non-reserved names are allowed on the assumption that they're field names.
+// Otherwise, only those names explicitly listed are allowed. Reserved args
+// (those with a leading underscore) are never allowed unless explicitly
+// present.
+//
+// The prototypes map maps from argument names to a value. If the value is
+// non-nil, the argument will be checked for type-matching. So, for instance,
+// `x: 10` would indicate that x must be an int.
+type callInfo struct {
+	allowUnknown bool
+	prototypes   map[string]interface{}
+	callType     CallType
+}
+
+// We want to be able to accept either a string or int64 for
+// field names. Special-case type:
+type stringOrInt64Type struct{}
+
+var stringOrInt64 stringOrInt64Type
+
+var allowUnderField = callInfo{
+	allowUnknown: true,
+	prototypes: map[string]interface{}{
+		"_field": "",
+	},
+}
+
+var allowField = callInfo{
+	allowUnknown: false,
+	prototypes: map[string]interface{}{
+		"field": "",
+	},
+}
+
+var callInfoByFunc = map[string]callInfo{
+	// the easy cases: things that take arbitrary inputs, because they're
+	// taking field=value cases
+	"Bitmap": {allowUnknown: true},
+	"Count":  {allowUnknown: true},
+	"Row":    {allowUnknown: true},
+	"Range":  {allowUnknown: true},
+
+	// allow only "field=X" cases with string field names
+	"Max": allowField,
+	"Min": allowField,
+	"Sum": allowField,
+
+	// only take other calls, should never have "args"
+	"Difference": {allowUnknown: false},
+	"Intersect":  {allowUnknown: false},
+	"Not":        {allowUnknown: false},
+	"ClearRow":   {allowUnknown: true},
+	"Store":      {allowUnknown: true},
+	"MinRow":     allowField,
+	"MaxRow":     allowField,
+	"Rows": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"_field":   "",
+			"field":    "",
+			"limit":    int64(0),
+			"column":   nil,
+			"previous": nil,
+			"from":     nil,
+			"to":       nil,
+		},
+	},
+	"Shift": {allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"n": int64(0),
+		},
+	},
+	"Union": {allowUnknown: false},
+	"Xor":   {allowUnknown: false},
+
+	// things that take _field
+	"TopN": allowUnderField,
+	// special cases:
+	"Clear": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_col": stringOrInt64,
+		},
+	},
+	"GroupBy": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"filter":   nil,
+			"limit":    int64(0),
+			"previous": nil,
+		},
+	},
+	"Options": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"excludeRowAttrs": true,
+			"excludeColumns":  true,
+			"columnAttrs":     true,
+			"shards":          nil,
+		},
+	},
+	"Set": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_col":       stringOrInt64,
+			"_timestamp": "",
+		},
+	},
+	"Precomputed": {
+		allowUnknown: true,
+	},
+	"SetBit": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_col": stringOrInt64,
+		},
+	},
+	"SetRowAttrs": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_field": "",
+			"_row":   stringOrInt64,
+		},
+	},
+	"SetColumnAttrs": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_field": "",
+			"_col":   stringOrInt64,
+		},
+	},
+	"IncludesColumn": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"column": stringOrInt64,
+		},
+	},
+}
+
+// RegisterPluginFuncs adds arg validation for plugin funcs. Not very good
+// arg validation.
+func RegisterPluginFuncs(ops []ext.BitmapOp) {
+	for _, op := range ops {
+		// ignore overlap for now. This should change.
+		if _, ok := callInfoByFunc[op.Name]; ok {
+			continue
+		}
+		ci := callInfo{allowUnknown: true}
+		if len(op.Reserved) > 0 {
+			// mark these as valid/known reserved words
+			ci.prototypes = make(map[string]interface{})
+			for _, res := range op.Reserved {
+				ci.prototypes[res] = nil
+			}
+		}
+		t := op.Func.BitmapOpType()
+		if t.Precall == ext.OpPrecallGlobal {
+			ci.callType = PrecallGlobal
+		}
+		callInfoByFunc[op.Name] = ci
+	}
+}
+
+// CheckCallInfo tries to validate that arguments are correct and valid for the
+// given call. It does not guarantee checking all possible errors; for instance,
+// if an argument is a field name, CheckCallInfo can't validate that the field
+// exists. It also updates with information like whether the call is expected
+// to require precalling.
+func (c *Call) CheckCallInfo() error {
+	valid, ok := callInfoByFunc[c.Name]
+	if !ok {
+		return fmt.Errorf("no arg validation for '%s'", c.Name)
+	}
+	c.Type = valid.callType
+	for k, v := range c.Args {
+		acceptable, ok := valid.prototypes[k]
+		if !ok && !valid.allowUnknown {
+			return fmt.Errorf("'%s': unknown arg '%s'", c.String(), k)
+		}
+		if !ok && strings.HasPrefix(k, "_") {
+			return fmt.Errorf("'%s': unknown reserved arg '%s'", c.String(), k)
+		}
+		if acceptable == nil {
+			continue
+		}
+		// if the types are identical, that's fine
+		if reflect.TypeOf(acceptable) == reflect.TypeOf(v) {
+			continue
+		}
+		if reflect.TypeOf(acceptable) == reflect.TypeOf(stringOrInt64) {
+			switch v.(type) {
+			case string, int64:
+				continue
+			default:
+				return fmt.Errorf("'%s': arg '%s' needed a string or integer value, got %T.",
+					c.String(), k, v)
+			}
+		}
+		return fmt.Errorf("'%s': arg '%s' wrong type (got %T, expected %T)",
+			c.String(), k, v, acceptable)
+	}
+	// call-specific checking
+	for _, child := range c.Children {
+		if err := child.CheckCallInfo(); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // FieldArg determines which key-value pair contains the field and rowID,
@@ -283,13 +525,29 @@ func IsReservedArg(name string) bool {
 		return true
 	}
 	switch name {
-	case "from", "to":
+	case "from", "to", "index":
 		return true
 	default:
 		return false
 	}
 }
 
+// CallIndex handles guessing whether we've been asked to apply this to a
+// different index. An empty string means "no".
+func (c *Call) CallIndex() string {
+	if index, ok := c.Args["_index"]; ok {
+		if index, ok := index.(string); ok {
+			return index
+		}
+	}
+	if index, ok := c.Args["index"]; ok && index != "" {
+		if index, ok := index.(string); ok {
+			return index
+		}
+	}
+	return ""
+}
+
 // BoolArg is for reading the value at key from call.Args as a bool. If the
 // key is not in Call.Args, the value of the returned bool will be false, and
 // the error will be nil. The value is assumed to be a bool. An error is

pql/parser.go

@@ -82,6 +82,14 @@ func (p *parser) Parse() (*Query, error) {
 			panic(v)
 		}
 	}
+	for _, call := range p.Query.Calls {
+		if call == nil {
+			return nil, fmt.Errorf("unexpected nil Call in query's call list")
+		}
+		if err := call.CheckCallInfo(); err != nil {
+			return nil, err
+		}
+	}
 
 	return &p.Query, nil
 }

pql/parser_test.go

@@ -75,12 +75,12 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with only arguments.
 	t.Run("ArgumentsOnly", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\"  )`)
+		q, err := pql.ParseString(`Row( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\"  )`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
 					"key":    "value",
 					"foo":    "bar",
@@ -98,12 +98,12 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with float arguments.
 	t.Run("WithFloatArgs", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9)`)
+		q, err := pql.ParseString(`Row( key=12.25, foo= 13.167, bar=2., baz=0.9)`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
 					"key": 12.25,
 					"foo": 13.167,
@@ -118,12 +118,12 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with float arguments.
 	t.Run("WithNegativeArgs", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall( key=-12.25, foo= -13)`)
+		q, err := pql.ParseString(`Row( key=-12.25, foo= -13)`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
 					"key": -12.25,
 					"foo": int64(-13),
@@ -173,12 +173,12 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with condition arguments.
 	t.Run("WithCondition", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
+		q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
 					"key": "foo",
 					"x":   &pql.Condition{Op: pql.EQ, Value: 12.25},

pql/pqlpeg_test.go

@@ -46,7 +46,7 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9
 		t.Fatalf("Failed, got: %s", q)
 	}
 
-	_, err = ParseString("C(a=falsen0)")
+	_, err = ParseString("Row(a=falsen0)")
 	if err != nil {
 		t.Fatalf("falsen0 should have been parsed as a string")
 	}
@@ -109,15 +109,15 @@ func TestPEGWorking(t *testing.T) {
 			ncalls: 2},
 		{
 			name:   "SetWithArbCall",
-			input:  "Set(1, a=4)Blerg(z=ha)",
+			input:  "Set(1, a=4)Row(z=ha)",
 			ncalls: 2},
 		{
 			name:   "SetArbSet",
-			input:  "Set(1, a=4)Blerg(z=ha)Set(2, z=99)",
+			input:  "Set(1, a=4)Row(z=ha)Set(2, z=99)",
 			ncalls: 3},
 		{
 			name:   "ArbSetArb",
-			input:  "Arb(q=1, a=4)Set(1, z=9)Arb(z=99)",
+			input:  "Row(q=1, a=4)Set(1, z=9)Row(z=99)",
 			ncalls: 3},
 		{
 			name:   "SetStringArg",
@@ -161,11 +161,11 @@ func TestPEGWorking(t *testing.T) {
 			ncalls: 1},
 		{
 			name:   "double quoted args",
-			input:  `B(a="zm''e")`,
+			input:  `Row(a="zm''e")`,
 			ncalls: 1},
 		{
 			name:   "single quoted args",
-			input:  `B(a='zm""e')`,
+			input:  `Row(a='zm""e')`,
 			ncalls: 1},
 		{
 			name:   "SetRowAttrs",
@@ -320,7 +320,7 @@ func TestPEGErrors(t *testing.T) {
 			input: "Set(, 1, a=4)"},
 		{
 			name:  "StartinCommaArb",
-			input: "Zeeb(, a=4)"},
+			input: "Row(, a=4)"},
 		{
 			name:  "SetRowAttrs0args",
 			input: "SetRowAttrs(blah, 9)"},
@@ -585,11 +585,11 @@ func TestPQLDeepEquality(t *testing.T) {
 			}},
 		{
 			name: "Weird dash",
-			call: "Sum(field-=f)",
+			call: "Count(dashy-=f)",
 			exp: &Call{
-				Name: "Sum",
+				Name: "Count",
 				Args: map[string]interface{}{
-					"field-": "f",
+					"dashy-": "f",
 				},
 			}},
 		{

roaring/container_stash.go

@@ -267,9 +267,6 @@ func (c *Container) Freeze() *Container {
 	if c.flags&flagFrozen != 0 {
 		return c
 	}
-	// unmapOrClone should unmap-in-place because the existing
-	// container isn't frozen (or we'd already have returned it).
-	c = c.unmapOrClone()
 	c.flags |= flagFrozen
 	return c
 }
@@ -419,6 +416,45 @@ func (c *Container) bitmap() []uint64 {
 	return *(*[]uint64)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(c.pointer)), Len: int(c.len), Cap: int(c.cap)}))
 }
 
+// AsBitmap yields a 65k-bit bitmap, storing it in the target if a target
+// is provided. The target should be zeroed, or this becomes an implicit
+// union.
+func (c *Container) AsBitmap(target []uint64) (out []uint64) {
+	if c.typeID == containerBitmap {
+		return c.bitmap()
+	}
+	// Reminder: len(nil) == 0.
+	if len(target) < 1024 {
+		out = make([]uint64, 1024)
+	} else {
+		out = target
+	}
+	if c.typeID == containerArray {
+		a := c.array()
+		for _, v := range a {
+			out[v/64] |= 1 << (v % 64)
+		}
+		return out
+	}
+	if c.typeID == containerRun {
+		runs := c.runs()
+		for _, r := range runs {
+			splatRun(out, r)
+		}
+		return out
+	}
+	// in theory this shouldn't happen?
+	return out
+}
+
+func splatRun(into []uint64, from interval16) {
+	// TODO this can be ~64x faster for long runs by setting maxBitmap instead of single bits
+	//note v must be int or will overflow
+	for v := int(from.start); v <= int(from.last); v++ {
+		into[v/64] |= (uint64(1) << uint(v%64))
+	}
+}
+
 // setBitmap stores a set of uint64s as data.
 func (c *Container) setBitmap(bitmap []uint64) {
 	if c == nil || c.frozen() {

roaring/generation_debug.go

@@ -0,0 +1,19 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build generationdebug
+
+package roaring
+
+const generationDebug = true

roaring/generation_nodebug.go

@@ -0,0 +1,19 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !generationdebug
+
+package roaring
+
+const generationDebug = false

roaring/roaring.go

@@ -21,6 +21,7 @@ import (
 	"hash/fnv"
 	"io"
 	"math/bits"
+	"reflect"
 	"sort"
 	"unsafe"
 
@@ -77,6 +78,46 @@ var containerTypeNames = map[byte]string{
 
 var fullContainer = NewContainerRun([]interval16{{start: 0, last: maxContainerVal}}).Freeze()
 
+// AdvisoryError is used for the special case where we probably want to *report*
+// an error reading a file, but don't want to actually count the file as not
+// being read. For instance, a partial ops-log entry is *probably* harmless;
+// we probably crashed while writing (?) and as such didn't report the write
+// as successful. We hope.
+type AdvisoryError interface {
+	error
+	AdvisoryOnly()
+}
+
+type advisoryError struct {
+	e error
+}
+
+func (a advisoryError) Error() string {
+	return a.e.Error()
+}
+
+// This marks the error as safe to ignore.
+func (a advisoryError) AdvisoryOnly() {
+}
+
+type FileShouldBeTruncatedError interface {
+	AdvisoryError
+	SuggestedLength() int64
+}
+
+type fileShouldBeTruncatedError struct {
+	advisoryError
+	offset int64
+}
+
+func (f *fileShouldBeTruncatedError) SuggestedLength() int64 {
+	return f.offset
+}
+
+func newFileShouldBeTruncatedError(err error, offset int64) *fileShouldBeTruncatedError {
+	return &fileShouldBeTruncatedError{advisoryError: advisoryError{e: err}, offset: offset}
+}
+
 type Containers interface {
 	// Get returns nil if the key does not exist.
 	Get(key uint64) *Container
@@ -144,6 +185,7 @@ type ContainerIterator interface {
 // Bitmap represents a roaring bitmap.
 type Bitmap struct {
 	Containers Containers
+	Source     Source
 
 	// User-defined flags.
 	Flags byte
@@ -218,6 +260,7 @@ func (b *Bitmap) Freeze() *Bitmap {
 	// Create a copy of the bitmap structure.
 	other := &Bitmap{
 		Containers: b.Containers.Freeze(),
+		Source:     b.Source,
 	}
 
 	return other
@@ -391,6 +434,13 @@ func (b *Bitmap) Min() (uint64, bool) {
 	return v, !eof
 }
 
+// MinAt returns the lowest value in the bitmap at least equal to its argument.
+// Second return value is true if containers exist in the bitmap.
+func (b *Bitmap) MinAt(start uint64) (uint64, bool) {
+	v, eof := b.IteratorAt(start).Next()
+	return v, !eof
+}
+
 // Max returns the highest value in the bitmap.
 // Returns zero if the bitmap is empty.
 func (b *Bitmap) Max() uint64 {
@@ -549,13 +599,21 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap {
 	hi0, hi1 := highbits(start), highbits(end)
 	citer, _ := b.Containers.Iterator(hi0)
 	other := NewSliceBitmap()
+	mappedAny := false
 	for citer.Next() {
 		k, c := citer.Value()
 		if k >= hi1 {
 			break
 		}
+		if c.Mapped() {
+			mappedAny = true
+		}
 		other.Containers.Put(off+(k-hi0), c.Freeze())
 	}
+	// if b.Source != nil && mappedAny {
+	if b.Source != nil && (generationDebug || mappedAny) {
+		other.Source = b.Source
+	}
 	return other
 }
 
@@ -594,6 +652,7 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
 // Intersect returns the intersection of b and other.
 func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
 	output := NewBitmap()
+	usedB, usedOther := false, false
 	iiter, _ := b.Containers.Iterator(0)
 	jiter, _ := other.Containers.Iterator(0)
 	i, j := iiter.Next(), jiter.Next()
@@ -607,12 +666,27 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
 			j = jiter.Next()
 			kj, cj = jiter.Value()
 		} else { // ki == kj
-			output.Containers.Put(ki, intersect(ci, cj))
+			newC := intersect(ci, cj)
+			if newC == ci {
+				usedB = true
+			}
+			if newC == cj {
+				usedOther = true
+			}
+			output.Containers.Put(ki, newC)
 			i, j = iiter.Next(), jiter.Next()
 			ki, ci = iiter.Value()
 			kj, cj = jiter.Value()
 		}
 	}
+	switch {
+	case usedB && usedOther:
+		output.Source = MergeSources(b.Source, other.Source)
+	case usedB:
+		output.Source = b.Source
+	case usedOther:
+		output.Source = other.Source
+	}
 	return output
 }
 
@@ -640,25 +714,43 @@ func (b *Bitmap) UnionInPlace(others ...*Bitmap) {
 func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) {
 	iiter, _ := b.Containers.Iterator(0)
 	jiter, _ := other.Containers.Iterator(0)
+	usedB, usedOther := false, false
 	i, j := iiter.Next(), jiter.Next()
 	ki, ci := iiter.Value()
 	kj, cj := jiter.Value()
 	for i || j {
 		if i && (!j || ki < kj) {
 			target.Containers.Put(ki, ci.Freeze())
+			usedB = true
 			i = iiter.Next()
 			ki, ci = iiter.Value()
 		} else if j && (!i || ki > kj) {
 			target.Containers.Put(kj, cj.Freeze())
+			usedOther = true
 			j = jiter.Next()
 			kj, cj = jiter.Value()
 		} else { // ki == kj
-			target.Containers.Put(ki, union(ci, cj))
+			newC := union(ci, cj)
+			target.Containers.Put(ki, newC)
+			if newC == ci {
+				usedB = true
+			}
+			if newC == cj {
+				usedOther = true
+			}
 			i, j = iiter.Next(), jiter.Next()
 			ki, ci = iiter.Value()
 			kj, cj = jiter.Value()
 		}
 	}
+	switch {
+	case usedB && usedOther:
+		target.Source = MergeSources(b.Source, other.Source)
+	case usedB:
+		target.Source = b.Source
+	case usedOther:
+		target.Source = other.Source
+	}
 }
 
 // unionInPlace stores the union of b and others into b. The others will
@@ -752,7 +844,14 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) {
 		bitmapIters = make(handledIters, 0, requiredSliceSize)
 	}
 
+	var sources []Source
+	if b.Source != nil {
+		sources = append(sources, b.Source)
+	}
 	for _, other := range others {
+		if other.Source != nil {
+			sources = append(sources, other.Source)
+		}
 		otherIter, _ := other.Containers.Iterator(0)
 		if otherIter.Next() {
 			bitmapIters = append(bitmapIters, handledIter{
@@ -762,6 +861,8 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) {
 			})
 		}
 	}
+	// new bitmap might have containers from any of those bitmaps in it
+	b.Source = MergeSources(sources...)
 
 	// Loop until we've exhausted every iter.
 	hasNext := true
@@ -890,6 +991,7 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) {
 // Difference returns the difference of b and other.
 func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
 	output := NewBitmap()
+	output.Source = b.Source
 
 	iiter, _ := b.Containers.Iterator(0)
 	jiter, _ := other.Containers.Iterator(0)
@@ -917,6 +1019,9 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
 // Xor returns the bitwise exclusive or of b and other.
 func (b *Bitmap) Xor(other *Bitmap) *Bitmap {
 	output := NewBitmap()
+	// Xor can end up with containers from either parent if the other
+	// had no container or an empty container.
+	output.Source = MergeSources(b.Source, other.Source)
 
 	iiter, _ := b.Containers.Iterator(0)
 	jiter, _ := other.Containers.Iterator(0)
@@ -1433,7 +1538,11 @@ func (b *Bitmap) RemapRoaringStorage(data []byte) (mappedAny bool, returnErr err
 	var itrPointer *uint16
 	var itrErr error
 
-	if data != nil {
+	// If we got no data, we don't want to do the actual mapping, just
+	// the unmapping. If preferMapping is false, we also don't want to
+	// map to the data. We still need to do the UpdateEvery loop, we
+	// just won't have an iterator for it.
+	if data != nil && b.preferMapping {
 		itr, err = newRoaringIterator(data)
 	}
 	// don't return early: we still have to do the unmapping
@@ -1617,6 +1726,12 @@ func (b *Bitmap) Iterator() *Iterator {
 	return itr
 }
 
+func (b *Bitmap) IteratorAt(start uint64) *Iterator {
+	itr := &Iterator{bitmap: b}
+	itr.Seek(start)
+	return itr
+}
+
 // Ops returns the number of write ops the bitmap is aware of in its ops
 // log, and their total bit count.
 func (b *Bitmap) Ops() (ops int, opN int) {
@@ -1629,6 +1744,167 @@ func (b *Bitmap) SetOps(ops int, opN int) {
 	b.ops, b.opN = ops, opN
 }
 
+// RoaringToBitmaps yields a series of bitmaps with specified shard
+// keys, based on a single roaring file, with splits at multiples of
+// shardWidth, which should be a multiple of container size.
+func RoaringToBitmaps(data []byte, shardWidth uint64) ([]*Bitmap, []uint64) {
+	if data == nil {
+		return nil, nil
+	}
+	var itr roaringIterator
+	var itrKey uint64
+	var itrCType byte
+	var itrN int
+	var itrLen int
+	var itrPointer *uint16
+	var itrErr error
+	currentShard := ^uint64(0)
+	var currentBitmap *Bitmap
+	var bitmaps []*Bitmap
+	var shards []uint64
+	keysPerShard := shardWidth >> 16
+
+	itr, err := newRoaringIterator(data)
+	if err != nil || itr == nil {
+		return nil, nil
+	}
+
+	itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next()
+	for itrErr == nil {
+		newC := &Container{
+			typeID:  itrCType,
+			n:       int32(itrN),
+			len:     int32(itrLen),
+			cap:     int32(itrLen),
+			pointer: itrPointer,
+			flags:   flagMapped,
+		}
+		shard := itrKey / keysPerShard
+		if shard != currentShard {
+			if currentBitmap != nil {
+				bitmaps = append(bitmaps, currentBitmap)
+				shards = append(shards, currentShard)
+			}
+			currentBitmap = NewFileBitmap()
+			currentShard = shard
+		}
+		currentBitmap.Containers.Put(itrKey, newC)
+		itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next()
+	}
+	if currentBitmap != nil {
+		bitmaps = append(bitmaps, currentBitmap)
+		shards = append(shards, currentShard)
+	}
+	// we don't support ops logs for this
+	return bitmaps, shards
+}
+
+// BitmapsToRoaring renders a series of non-overlapping bitmaps as a
+// unified roaring file.
+func BitmapsToRoaring(bitmaps []*Bitmap) []byte {
+	count := int64(0)
+	size := int64(0)
+	for i, bm := range bitmaps {
+		c, s := bm.roaringSize()
+		// skip this bitmap during the next pass, since it's empty
+		if c == 0 {
+			bitmaps[i] = nil
+			continue
+		}
+		count += c
+		size += s
+	}
+	if count == 0 {
+		return nil
+	}
+	// we have count headers, which need 12 bytes, plus a magic number,
+	// plus offsets (4 bytes per container), plus size bytes of data to
+	// write.
+	out := make([]byte, headerBaseSize+(12*count)+(4*count)+size)
+	binary.LittleEndian.PutUint16(out[0:2], uint16(MagicNumber))
+	out[3] = byte(storageVersion)
+	binary.LittleEndian.PutUint32(out[4:8], uint32(count))
+	headerEnd := 8 + (12 * count)
+	offsetEnd := headerEnd + (4 * count)
+	headers := out[8:headerEnd]
+	offsets := out[headerEnd:offsetEnd]
+	data := out[offsetEnd:]
+	headerOffset := 0
+	offsetOffset := 0
+	dataOffset := 0
+	prevKey := uint64(0)
+	for _, bm := range bitmaps {
+		if bm == nil {
+			continue
+		}
+		citer, _ := bm.Containers.Iterator(0)
+		for citer.Next() {
+			k, c := citer.Value()
+			n := c.N()
+			if n == 0 {
+				continue
+			}
+			if roaringParanoia {
+				if k < prevKey {
+					panic("unsorted keys in multiple-bitmap roaring conversion")
+				}
+			}
+			// place header at header offset, and data at data
+			// offset
+			header := headers[headerOffset : headerOffset+12]
+			offset := offsets[offsetOffset : offsetOffset+4]
+			headerOffset += 12
+			offsetOffset += 4
+			binary.LittleEndian.PutUint64(header[0:8], k)
+			binary.LittleEndian.PutUint16(header[8:10], uint16(c.typeID))
+			binary.LittleEndian.PutUint16(header[10:12], uint16(n-1))
+			binary.LittleEndian.PutUint32(offset[0:4], uint32(dataOffset+int(offsetEnd)))
+			nextData := data[dataOffset:]
+			switch c.typeID {
+			case containerArray:
+				asUint16 := *(*[]uint16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&nextData[0])), Len: int(c.len), Cap: int(c.len)}))
+				copy(asUint16, c.array())
+				dataOffset += 2 * int(c.len)
+			case containerBitmap:
+				asUint64 := *(*[]uint64)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&nextData[0])), Len: 1024, Cap: 1024}))
+				copy(asUint64, c.bitmap())
+				dataOffset += 8192
+			case containerRun:
+				asInterval16 := *(*[]interval16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&nextData[2])), Len: int(c.len), Cap: int(c.len)}))
+				copy(asInterval16, c.runs())
+				binary.LittleEndian.PutUint16(nextData[0:2], uint16(c.len))
+				dataOffset += int(4*c.len) + 2
+			}
+		}
+	}
+	return out
+}
+
+// roaringSize yields the count of non-empty containers, and the size
+// of the storage *only* -- not the headers.
+func (b *Bitmap) roaringSize() (int64, int64) {
+	count := int64(0)
+	size := int64(0)
+	citer, _ := b.Containers.Iterator(0)
+	for citer.Next() {
+		_, c := citer.Value()
+		if c.N() == 0 {
+			continue
+		}
+		count++
+		switch c.typeID {
+		case containerArray:
+			size += 2 * int64(c.N())
+		case containerBitmap:
+			size += 8192
+		case containerRun:
+			// 2 bytes for the count of runs, plus 4 bytes per run
+			size += 2 + (4 * int64(c.len))
+		}
+	}
+	return count, size
+}
+
 // Info returns stats for the bitmap.
 func (b *Bitmap) Info() bitmapInfo {
 	info := bitmapInfo{
@@ -3271,9 +3547,9 @@ func intersectRunRun(a, b *Container) *Container {
 	output.setN(n)
 	runs := output.runs()
 	if n < ArrayMaxSize && int32(len(runs)) > n/2 {
-		output.runToArray()
+		output = output.runToArray()
 	} else if len(runs) > runMaxSize {
-		output.runToBitmap()
+		output = output.runToBitmap()
 	}
 	return output
 }

roaring/source.go

@@ -0,0 +1,98 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package roaring
+
+import (
+	"strings"
+)
+
+// A Source represents the source a given bitmap gets its data from,
+// such as a memory-mapped file. When combining bitmaps, we might
+// track them together in a single combined-source of some sort.
+type Source interface {
+	ID() string
+	Dead() bool
+}
+
+// MergeSources combines sources. If you have two bitmaps, and you're
+// combining them, then the combination's source is a combination of
+// those two sources.
+func MergeSources(sources ...Source) Source {
+	sourceCount := 0
+	totalCount := 0
+	var lastSource Source
+	for _, s := range sources {
+		if s == nil {
+			continue
+		}
+		lastSource = s
+		if s, ok := s.(combinedSource); ok {
+			sourceCount++
+			totalCount += len(s)
+		} else {
+			sourceCount++
+			totalCount++
+		}
+	}
+	// if there's no sources (this includes all sources being
+	// empty combinedSources), we don't have a source.
+	if totalCount == 0 {
+		return nil
+	}
+	// if there's exactly one source, combined or otherwise, that's
+	// fine, we'll just return it.
+	if sourceCount == 1 {
+		return lastSource
+	}
+	// make a new combinedSource, flattening any combinedSources
+	// already present.
+	newSources := make([]Source, 0, totalCount)
+	for _, s := range sources {
+		if s == nil {
+			continue
+		}
+		if s, ok := s.(combinedSource); ok {
+			newSources = append(newSources, s...)
+		} else {
+			newSources = append(newSources, s)
+		}
+	}
+	return combinedSource(newSources)
+}
+
+// SetSource tells the bitmap what source to associate with new things it
+// creates. This is possibly logically incorrect.
+func (b *Bitmap) SetSource(s Source) {
+	b.Source = s
+}
+
+type combinedSource []Source
+
+func (c combinedSource) ID() string {
+	ids := make([]string, len(c))
+	for i := range c {
+		ids[i] = c[i].ID()
+	}
+	return strings.Join(ids, ",")
+}
+
+func (c combinedSource) Dead() bool {
+	for i := range c {
+		if c[i].Dead() {
+			return true
+		}
+	}
+	return false
+}

roaring/unmarshal_binary.go

@@ -30,7 +30,9 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		return nil
 	}
 	statsHit("Bitmap/UnmarshalBinary")
-	b.opN = 0 // reset opN since we're reading new data.
+	// reset ops/opN since we're reading new data.
+	b.ops = 0
+	b.opN = 0
 	fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
 	if fileMagic == MagicNumber { // if pilosa roaring
 		return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring")
@@ -205,15 +207,15 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
 		// Unmarshal the op and apply it.
 		var opr op
 		if err := opr.UnmarshalBinary(buf); err != nil {
-			// FIXME(benbjohnson): return error with position so file can be trimmed.
-			return err
+			return newFileShouldBeTruncatedError(err, int64(opsOffset))
 		}
 		opr.apply(b)
 		// Increase the op count.
 		b.ops++
 		b.opN += opr.count()
+		opsOffset += opr.size()
 		// Move the buffer forward.
-		buf = buf[opr.size():]
+		buf = data[opsOffset:]
 	}
 
 	return nil

row.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"sort"
 
+	"github.com/pilosa/pilosa/v2/ext"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pkg/errors"
 )
@@ -43,6 +44,55 @@ func NewRow(columns ...uint64) *Row {
 	return r
 }
 
+// NewRowFromBitmap divides a bitmap into rows, which it now calls shards. This
+// transposes; data that was in any shard for Row 0 is now considered shard 0,
+// etcetera.
+func NewRowFromBitmap(b *roaring.Bitmap) *Row {
+	r := &Row{}
+	if b == nil {
+		return r
+	}
+	rowNum := uint64(0)
+	for col, ok := b.MinAt(rowNum * ShardWidth); ok; col, ok = b.MinAt(rowNum * ShardWidth) {
+		rowNum = col / ShardWidth
+		seg := rowSegment{
+			shard:    rowNum,
+			data:     b.OffsetRange(rowNum*ShardWidth, rowNum*ShardWidth, (rowNum+1)*ShardWidth),
+			writable: true,
+		}
+		seg.n = seg.data.Count()
+		r.segments = append(r.segments, seg)
+		rowNum++
+	}
+	return r
+}
+
+// NewRowFromRoaring parses a roaring data file as a row, dividing it into
+// bitmaps and rowSegments based on shard width.
+func NewRowFromRoaring(data []byte) *Row {
+	bitmaps, shards := roaring.RoaringToBitmaps(data, ShardWidth)
+	r := &Row{segments: make([]rowSegment, len(bitmaps))}
+	for i := range bitmaps {
+		segment := rowSegment{
+			shard:    shards[i],
+			data:     bitmaps[i],
+			writable: false,
+			n:        bitmaps[i].Count(),
+		}
+		r.segments[i] = segment
+	}
+	return r
+}
+
+// Roaring returns the row treated as a unified roaring bitmap.
+func (r *Row) Roaring() []byte {
+	bitmaps := make([]*roaring.Bitmap, len(r.segments))
+	for i := range r.segments {
+		bitmaps[i] = r.segments[i].data
+	}
+	return roaring.BitmapsToRoaring(bitmaps)
+}
+
 // IsEmpty returns true if the row doesn't contain any set bits.
 func (r *Row) IsEmpty() bool {
 	if len(r.segments) == 0 {
@@ -194,6 +244,65 @@ func (r *Row) Union(others ...*Row) *Row {
 	return &Row{segments: output}
 }
 
+// GenericBinaryOp returns the output of a generic op on r and other.
+func (r *Row) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *Row, args map[string]interface{}) *Row {
+	var segments []rowSegment
+	itr := newMergeSegmentIterator(r.segments, other.segments)
+	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
+		if s1 == nil {
+			segments = append(segments, *s0)
+			continue
+		} else if s0 == nil {
+			segments = append(segments, *s1)
+			continue
+		}
+		segments = append(segments, *s0.GenericBinaryOp(op, s1, args))
+	}
+
+	return &Row{segments: segments}
+}
+
+// GenericNaryOp returns the output of an nary op on r and others.
+func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args map[string]interface{}) *Row {
+	segments := make([][]rowSegment, 0, len(others)+1)
+	if len(r.segments) > 0 {
+		segments = append(segments, r.segments)
+	}
+	nextSegs := make([][]rowSegment, 0, len(others)+1)
+	toProcess := make([]*rowSegment, 0, len(others)+1)
+	var output []rowSegment
+	for _, other := range others {
+		if len(other.segments) > 0 {
+			segments = append(segments, other.segments)
+		}
+	}
+	for len(segments) > 0 {
+		shard := segments[0][0].shard
+		for _, segs := range segments {
+			if segs[0].shard < shard {
+				shard = segs[0].shard
+			}
+		}
+		nextSegs = nextSegs[:0]
+		toProcess := toProcess[:0]
+		for _, segs := range segments {
+			if segs[0].shard == shard {
+				toProcess = append(toProcess, &segs[0])
+				segs = segs[1:]
+			}
+			if len(segs) > 0 {
+				nextSegs = append(nextSegs, segs)
+			}
+		}
+		// at this point, "toProcess" is a list of all the segments
+		// sharing the lowest ID, and nextSegs is a list of all the others.
+		// Swap the segment lists (so we don't have to reallocate it)
+		segments, nextSegs = nextSegs, segments
+		output = append(output, *toProcess[0].GenericNaryOp(op, toProcess[1:], args))
+	}
+	return &Row{segments: output}
+}
+
 // Difference returns the diff of r and other.
 func (r *Row) Difference(other *Row) *Row {
 	var segments []rowSegment
@@ -212,6 +321,17 @@ func (r *Row) Difference(other *Row) *Row {
 	return &Row{segments: segments}
 }
 
+// GenericUnary returns the results of a generic op on r.
+func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row {
+	work := r
+	var segments []rowSegment
+	for _, segment := range work.segments {
+		opped := segment.GenericUnaryOp(op, args)
+		segments = append(segments, *opped)
+	}
+	return &Row{segments: segments}
+}
+
 // Shift returns the bitwise shift of r by n bits.
 // Currently only positive shift values are supported.
 func (r *Row) Shift(n int64) (*Row, error) {
@@ -299,6 +419,15 @@ func (r *Row) Count() uint64 {
 	return n
 }
 
+// GenericCount applies an op to lots of things.
+func (r *Row) GenericCount(op ext.BitmapOpUnaryCount, args map[string]interface{}) uint64 {
+	var n int64
+	for i := range r.segments {
+		n += op([]ext.Bitmap{WrapBitmap(r.segments[i].data)}, args)
+	}
+	return uint64(n)
+}
+
 // MarshalJSON returns a JSON-encoded byte slice of r.
 func (r *Row) MarshalJSON() ([]byte, error) {
 	var o struct {
@@ -359,7 +488,7 @@ type rowSegment struct {
 }
 
 func (s *rowSegment) Freeze() {
-	s.data.Freeze()
+	s.data = s.data.Freeze()
 }
 
 /*
@@ -392,7 +521,7 @@ func (s *rowSegment) IntersectionCount(other *rowSegment) uint64 {
 // Intersect returns the itersection of s and other.
 func (s *rowSegment) Intersect(other *rowSegment) *rowSegment {
 	data := s.data.Intersect(other.data)
-	data.Freeze()
+	data = data.Freeze()
 
 	return &rowSegment{
 		data:     data,
@@ -419,10 +548,37 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment {
 	}
 }
 
+// GenericOp performs a generic op on s and other
+func (s *rowSegment) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *rowSegment, args map[string]interface{}) *rowSegment {
+	data := op([]ext.Bitmap{WrapBitmap(s.data), WrapBitmap(other.data)}, args)
+
+	return &rowSegment{
+		data:  UnwrapBitmap(data),
+		shard: s.shard,
+		n:     data.Count(),
+	}
+}
+
+// GenericOp performs a generic op on s and others
+func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSegment, args map[string]interface{}) *rowSegment {
+	bitmaps := make([]ext.Bitmap, len(others)+1)
+	bitmaps[0] = WrapBitmap(s.data)
+	for i, seg := range others {
+		bitmaps[i+1] = WrapBitmap(seg.data)
+	}
+	data := op(bitmaps, args)
+
+	return &rowSegment{
+		data:  UnwrapBitmap(data),
+		shard: s.shard,
+		n:     data.Count(),
+	}
+}
+
 // Difference returns the diff of s and other.
 func (s *rowSegment) Difference(other *rowSegment) *rowSegment {
 	data := s.data.Difference(other.data)
-	data.Freeze()
+	data = data.Freeze()
 
 	return &rowSegment{
 		data:     data,
@@ -435,7 +591,7 @@ func (s *rowSegment) Difference(other *rowSegment) *rowSegment {
 // Xor returns the xor of s and other.
 func (s *rowSegment) Xor(other *rowSegment) *rowSegment {
 	data := s.data.Xor(other.data)
-	data.Freeze()
+	data = data.Freeze()
 
 	return &rowSegment{
 		data:     data,
@@ -452,7 +608,7 @@ func (s *rowSegment) Shift() (*rowSegment, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "shifting roaring data")
 	}
-	data.Freeze()
+	data = data.Freeze()
 
 	return &rowSegment{
 		data:     data,
@@ -462,6 +618,18 @@ func (s *rowSegment) Shift() (*rowSegment, error) {
 	}, nil
 }
 
+// GenericUnary returns s subject to op.
+func (s *rowSegment) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *rowSegment {
+	//TODO deal with overflow
+	data := UnwrapBitmap(op([]ext.Bitmap{WrapBitmap(s.data)}, args))
+
+	return &rowSegment{
+		data:  data,
+		shard: s.shard,
+		n:     data.Count(),
+	}
+}
+
 // SetBit sets the i-th column of the row.
 func (s *rowSegment) SetBit(i uint64) (changed bool) {
 	s.ensureWritable()

server.go

@@ -17,17 +17,21 @@ package pilosa
 import (
 	"context"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"plugin"
 	"runtime"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/pilosa/pilosa/v2/ext"
 	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/stats"
 	"github.com/pkg/errors"
@@ -57,6 +61,8 @@ type Server struct { // nolint: maligned
 	hosts            []string
 	clusterDisabled  bool
 	serializer       Serializer
+	extensionPath    string
+	extensions       []*ext.ExtensionInfo
 
 	// External
 	systemInfo SystemInfo
@@ -335,6 +341,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 	if err != nil {
 		return nil, err
 	}
+	s.extensionPath = filepath.Join(path, ".extensions")
 
 	s.holder.Path = path
 	// s.holder.translateFile.Path = filepath.Join(path, ".keys")
@@ -376,6 +383,30 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 	s.cluster.broadcaster = s
 	s.cluster.maxWritesPerRequest = s.maxWritesPerRequest
 	s.holder.broadcaster = s
+	err = s.loadPlugins()
+	if err != nil {
+		s.logger.Printf("not all plugins loaded successfully")
+	}
+	if len(s.extensions) > 0 {
+		s.logger.Printf("loaded extensions:")
+		for _, ext := range s.extensions {
+			if ext == nil {
+				s.logger.Printf("  inexplicably, a nil extension?!?")
+				continue
+			}
+			s.logger.Printf("  %s %s: %s", ext.Name, ext.Version, ext.Description)
+			if ext.License != "" {
+				s.logger.Printf("    License: %s", ext.License)
+			}
+			if len(ext.BitmapOps) > 0 {
+				opList := make([]string, len(ext.BitmapOps))
+				for i := range ext.BitmapOps {
+					opList[i] = ext.BitmapOps[i].Name
+				}
+				s.logger.Printf("    Ops: %s", strings.Join(opList, ", "))
+			}
+		}
+	}
 
 	err = s.cluster.setup()
 	if err != nil {
@@ -389,6 +420,95 @@ func (s *Server) InternalClient() InternalClient {
 	return s.defaultClient
 }
 
+func (s *Server) loadPlugins() error {
+	var anyError error
+	dir, err := os.Open(s.extensionPath)
+	if err != nil {
+		// don't complain about it not existing, that's fine.
+		if os.IsNotExist(err) {
+			s.logger.Printf("extension interface v0: no extensions directory.")
+			return nil
+		}
+		return errors.Wrap(err, "opening extension path:")
+	}
+	defer dir.Close()
+	for files, err := dir.Readdir(64); err != io.EOF; files, err = dir.Readdir(64) {
+		if err != nil {
+			return errors.Wrap(err, "searching extension directory:")
+		}
+		for _, file := range files {
+			name := file.Name()
+			// only .so files are likely plugins.
+			if !strings.HasSuffix(name, ".so") {
+				continue
+			}
+			// only regular files are candidates for loading.
+			mode := file.Mode()
+			if !mode.IsRegular() {
+				s.logger.Printf("extension file '%s' is not a regular file", name)
+				continue
+			}
+			err = s.loadPlugin(name)
+			if err != nil {
+				s.logger.Printf("loading extension %s: %v", name, err)
+				anyError = err
+			}
+		}
+	}
+	return anyError
+}
+
+func (s *Server) loadPlugin(name string) error {
+	path := filepath.Join(s.extensionPath, name)
+	p, err := plugin.Open(path)
+	if err != nil {
+		return err
+	}
+	pluginExtInfo, err := p.Lookup("ExtensionInfo")
+	if err != nil {
+		return fmt.Errorf("%s: no ExtensionInfo found", name)
+	}
+	extInfoFunc, ok := pluginExtInfo.(func(string) (*ext.ExtensionInfo, error))
+	if !ok {
+		return fmt.Errorf("%s: unexpected %T instead of ExtensionInfo object", name, pluginExtInfo)
+	}
+	extInfo, err := extInfoFunc("v0")
+	if err != nil {
+		return errors.Wrap(err, name)
+	}
+	if extInfo == nil {
+		return fmt.Errorf("%s: nil ExtensionInfo", name)
+	}
+	if extInfo.ExtensionAPI != "v0" {
+		return fmt.Errorf("%s: unsupported extension API %s", name, extInfo.ExtensionAPI)
+	}
+	s.extensions = append(s.extensions, extInfo)
+	bitmapOps := extInfo.BitmapOps
+	bmOps, countOps, fieldOps, unknownOps := 0, 0, 0, 0
+	for i := range bitmapOps {
+		// title-case the name
+		bitmapOps[i].Name = strings.Title(bitmapOps[i].Name)
+		typ := bitmapOps[i].Func.BitmapOpType()
+		switch {
+		case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:
+			countOps++
+		case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap:
+			bmOps++
+		case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap:
+			fieldOps++
+		default:
+			unknownOps++
+		}
+	}
+	err = s.executor.registerOps(bitmapOps)
+	if err != nil {
+		s.logger.Printf("warning: extension registration failed: %v", err)
+	} else {
+		pql.RegisterPluginFuncs(bitmapOps)
+	}
+	return nil
+}
+
 // UpAndDown brings the server up minimally and shuts it down
 // again; basically, it exists for testing holder open and close.
 func (s *Server) UpAndDown() error {

snapshotqueue.go

@@ -0,0 +1,405 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pkg/errors"
+)
+
+// snapshotQueue is a thing which can handle enqueuing snapshots. A snapshot
+// queue distinguishes between high-priority requests, which get satisfied
+// by the next available worker, and regular requests, which get enqueued
+// if there's space in the queue, and otherwise dropped. There's also a
+// separate background task to scan a holder for fragments which may need
+// snapshots, but which is processed only when the queue is empty, and only
+// slowly. "Await" awaits an existing snapshot if one is already enqueued.
+// "Immediate" tries to do one right away. (If one's already enqueued, this
+// can leave it in the queue, which will ignore anything that shows up with
+// the request flag cleared.)
+//
+// Await, Enqueue, and Immediate should be called only with the fragment lock
+// held.
+//
+// ScanHolder spawns a new goroutine. You don't need to use `go` on it.
+type snapshotQueue interface {
+	Immediate(*fragment) error
+	Enqueue(*fragment)
+	Await(*fragment) error
+	ScanHolder(*Holder)
+	Stop()
+}
+
+// queuelessSnapshotQueue isn't a snapshot queue, but it satisfies the
+// interface.
+type queuelessSnapshotQueue struct{}
+
+func (q *queuelessSnapshotQueue) Enqueue(f *fragment) {
+	_ = f.snapshot()
+}
+
+func (q *queuelessSnapshotQueue) Await(f *fragment) error {
+	return nil
+}
+
+func (q *queuelessSnapshotQueue) Immediate(f *fragment) error {
+	return f.snapshot()
+}
+
+func (q *queuelessSnapshotQueue) ScanHolder(h *Holder) {
+}
+
+func (q *queuelessSnapshotQueue) Stop() {
+}
+
+// defaultSnapshotQueue is the fallback to use if none is available,
+// and currently uses queueless -- it runs all snapshots immediately.
+var defaultSnapshotQueue *queuelessSnapshotQueue
+
+// newSnapshotQueue makes a new snapshot queue, of depth N, with
+// w worker threads.
+func newSnapshotQueue(n int, w int, l logger.Logger) snapshotQueue {
+	sq := prioritySnapshotQueue{normal: make(chan snapshotRequest, n), urgent: make(chan snapshotRequest), background: make(chan snapshotRequest), done: make(chan struct{}), logger: l}
+	if sq.logger == nil {
+		sq.logger = logger.NewStandardLogger(os.Stderr)
+	}
+	sq.spawnWorkers(w)
+	return &sq
+}
+
+type snapshotRequest struct {
+	frag *fragment
+	when time.Time
+}
+
+// prioritySnapshotQueue gives preference to "immediate" requests, and
+// dispreference to "background" requests from ScanHolder. It timestamps
+// requests, so it can discard a request if the most recent snapshot is
+// newer than the request. The snapshotPending flag in the fragment is
+// used to track that a given fragment thinks it has been successfully
+// enqueued. Background requests are not considered enqueued, since
+// they'll never get processed if there's anything else. In normal workloads,
+// immediate/urgent snapshots should be rare, but we'll happily drop
+// most requests on the floor; the scanner should pick them up once things
+// are quiet.
+type prioritySnapshotQueue struct {
+	logger           logger.Logger
+	urgent           chan snapshotRequest
+	normal           chan snapshotRequest
+	background       chan snapshotRequest
+	done             chan struct{}
+	mu               sync.RWMutex
+	scanWG, workerWG sync.WaitGroup
+	stats            struct {
+		enqueued int64
+		skipped  int64
+	}
+}
+
+func (sq *prioritySnapshotQueue) spawnWorkers(w int) {
+	sq.mu.Lock()
+	defer sq.mu.Unlock()
+	if sq.done == nil {
+		sq.logger.Printf("prioritySnapshotQueue worker: no done channel, already done?")
+		return
+	}
+	sq.workerWG.Add(w)
+	for i := 0; i < w; i++ {
+		go sq.worker(sq.urgent, sq.normal, sq.background, sq.done)
+	}
+}
+
+func (sq *prioritySnapshotQueue) worker(urgent, normal, background chan snapshotRequest, done chan struct{}) {
+	// We don't want a race condition on these. If they're non-nil when
+	// we get them, they should get closed at some point. If done is
+	// already nil, we shouldn't do anything.
+	defer sq.workerWG.Done()
+	ok := true
+	var req snapshotRequest
+	for ok {
+		req.frag = nil
+
+		select {
+		case req, ok = <-urgent:
+		default:
+			select {
+			case req, ok = <-urgent:
+			case req, ok = <-normal:
+			default:
+				select {
+				case req, ok = <-urgent:
+				case req, ok = <-normal:
+				case req, ok = <-background:
+				case _, ok = <-done:
+				}
+			}
+		}
+		if req.frag != nil {
+			sq.process(req)
+		}
+	}
+}
+
+// process actually runs a fragment. it will do this if either the fragment
+// has a pending snapshot, or the force flag is set.
+func (sq *prioritySnapshotQueue) process(req snapshotRequest) {
+	f := req.frag
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.snapshotStamp.Before(req.when) {
+		f.snapshotErr = f.snapshot()
+		if f.snapshotErr != nil {
+			fmt.Printf("snapshot error: %v\n", f.snapshotErr)
+			sq.logger.Printf("snapshot error: %v", f.snapshotErr)
+		}
+		f.snapshotPending = false
+		f.snapshotCond.Broadcast()
+	}
+}
+
+// Stop shuts down the snapshot queue. It first marks it as done, causing
+// the background scanner(s), if any, to shut down, then waits for them, then
+// closes and nils the queues. The background scanner has to get stopped
+// because otherwise it might try to write to those closed queues.
+func (sq *prioritySnapshotQueue) Stop() {
+	sq.mu.Lock()
+	defer sq.mu.Unlock()
+	close(sq.done)
+	// scanners need to be done before we close the other channels.
+	sq.scanWG.Wait()
+	sq.done = nil
+	close(sq.normal)
+	sq.normal = nil
+	close(sq.urgent)
+	sq.urgent = nil
+	close(sq.background)
+	sq.background = nil
+	sq.logger.Printf("snapshot queue: enqueued %d, skipped %d\n", sq.stats.enqueued, sq.stats.skipped)
+}
+
+// Enqueue tries to add a fragment to the queue, if the fragment is not already
+// enqueued. You should hold a lock on the fragment when calling this.
+func (sq *prioritySnapshotQueue) Enqueue(f *fragment) {
+	if f.snapshotPending {
+		return
+	}
+	sq.mu.Lock()
+	defer sq.mu.Unlock()
+	if sq.normal == nil {
+		sq.logger.Printf("requested snapshot after snapshot queue was closed")
+		return
+	}
+	// we have to set this before enqueing, because it's
+	// otherwise possible that we're at the head of the queue,
+	// and the recipient gets the fragment before we execute the
+	// line after the send.
+	f.snapshotPending = true
+	// try to enqueue snapshot
+	select {
+	case sq.normal <- snapshotRequest{frag: f, when: time.Now()}:
+		sq.stats.enqueued++
+		return
+	default:
+		sq.stats.skipped++
+		f.snapshotPending = false
+		return
+	}
+}
+
+// Await returns when f is not pending a snapshot. Call with the fragment lock
+// held. Await waits on a condition variable inside f, associated with the
+// fragment's lock, so this does not conflict with the lock being used for
+// snapshots.
+func (sq *prioritySnapshotQueue) Await(f *fragment) (err error) {
+	for f.snapshotPending {
+		f.snapshotCond.Wait()
+	}
+	err, f.snapshotErr = f.snapshotErr, nil
+	return err
+}
+
+// Immediate forces an immediate snapshot of the given fragment. Call with
+// the fragment locked. If the queue is already closing, the fragment does
+// not get snapshotted.
+func (sq *prioritySnapshotQueue) Immediate(f *fragment) error {
+	sq.mu.RLock()
+	// no deferred unlock, because we want to unlock this before calling Await.
+	// Not because that needs this lock, but because once we're that far, we
+	// *don't* need this lock anymore so someone else should have it.
+	if sq.urgent == nil {
+		sq.mu.RUnlock()
+		sq.logger.Printf("requested immediate snapshot after snapshot queue was closed")
+		return errors.New("requested immediate snapshot after snapshot queue was closed")
+	}
+	f.snapshotPending = true
+	req := snapshotRequest{frag: f, when: time.Now()}
+	// if the fragment was already in the work queue, it's *possible*
+	// that the only available worker just picked it off the queue, and
+	// is now waiting on getting the fragment's lock, so it can run
+	// a snapshot. So we let go of the lock on the fragment, send the
+	// request, then request the fragment lock again, because Await will
+	// be sleeping on the condition variable associated with the lock,
+	// which means it needs to hold the lock so it can let it go during
+	// the wait... No, really, this made sense.
+	f.mu.Unlock()
+	sq.urgent <- req
+	sq.mu.RUnlock()
+	f.mu.Lock()
+	return sq.Await(f)
+}
+
+// needsSnapshot determines whether a fragment probably wants snapshotting.
+// Specifically, it looks for fragments not already marked to receive
+// snapshots, but which have a high enough opN to justify a snapshot. This
+// is only used from the background scan.
+func (sq *prioritySnapshotQueue) needsSnapshot(f *fragment) bool {
+	if f == nil {
+		return false
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.snapshotPending {
+		return false
+	}
+	if f.opN > f.MaxOpN {
+		return true
+	}
+	return false
+}
+
+// ScanHolder spawns a goroutine which iterates through the holder's
+// indexes/fields/views/fragments, looking for fragments which have OpN
+// high enough to justify a snapshot but don't seem to have one pending.
+// It then dumps these in the low priority background queue.
+func (sq *prioritySnapshotQueue) ScanHolder(h *Holder) {
+	sq.mu.Lock()
+	sq.scanWG.Add(1)
+	go sq.scanHolderWorker(h, sq.background, sq.done)
+	sq.mu.Unlock()
+}
+
+// scanHolderWorker is a background task that scans a holder looking for
+// fragments which need snapshots taken. It's the cleanup task for snapshots
+// that would have been requested by Enqueue, but the queue was full.
+func (sq *prioritySnapshotQueue) scanHolderWorker(h *Holder, background chan snapshotRequest, done chan struct{}) {
+	defer sq.scanWG.Done()
+	var indexNames, fieldNames, viewNames []string
+	var fragNums []uint64
+	for {
+		// To avoid abusing things, cap activity rate; every time we finish
+		// the holder, or every couple hundred fragments considered, we
+		// pause for a bit.
+		counter := 0
+		hits := 0
+		h.mu.Lock()
+		indexNames = indexNames[:0]
+		for indexName := range h.indexes {
+			indexNames = append(indexNames, indexName)
+		}
+		h.mu.Unlock()
+		for _, indexName := range indexNames {
+			h.mu.Lock()
+			index := h.indexes[indexName]
+			h.mu.Unlock()
+			if index == nil {
+				continue
+			}
+			fieldNames = fieldNames[:0]
+			index.mu.Lock()
+			for fieldName := range index.fields {
+				fieldNames = append(fieldNames, fieldName)
+			}
+			index.mu.Unlock()
+			for _, fieldName := range fieldNames {
+				index.mu.Lock()
+				field := index.fields[fieldName]
+				index.mu.Unlock()
+				if field == nil {
+					continue
+				}
+				viewNames = viewNames[:0]
+				field.mu.Lock()
+				for viewName := range field.viewMap {
+					viewNames = append(viewNames, viewName)
+				}
+				field.mu.Unlock()
+				for _, viewName := range viewNames {
+					field.mu.Lock()
+					view := field.viewMap[viewName]
+					field.mu.Unlock()
+					if view == nil {
+						continue
+					}
+					fragNums := fragNums[:0]
+					view.mu.Lock()
+					for fragNum := range view.fragments {
+						fragNums = append(fragNums, fragNum)
+					}
+					view.mu.Unlock()
+					for _, fragNum := range fragNums {
+						view.mu.Lock()
+						frag := view.fragments[fragNum]
+						view.mu.Unlock()
+						if sq.needsSnapshot(frag) {
+							hits++
+							select {
+							case background <- snapshotRequest{frag: frag, when: time.Now()}:
+								sq.logger.Debugf("found fragment needing snapshot: %s\n", frag.path)
+							case <-done:
+								return
+							}
+						} else {
+							// Count fragments examined *without* finding anything that
+							// needed a snapshot. When we find things that need snapshots,
+							// the time it takes the workers to respond to us is enough
+							// of a delay to keep us from eating every CPU. So, if a lot
+							// of things need snapshots, and the workers aren't doing
+							// anything else, ScanHolder will mostly keep them saturated.
+							// If they're busy, we'll block forever in the write to the
+							// background queue. If there's nothing that needs snapshots,
+							// we pause frequently for a second or so at a time.
+							counter++
+							if counter == 100 {
+								select {
+								case <-time.After(1 * time.Second):
+								case <-done:
+									return
+								}
+								counter = 0
+							}
+						}
+					}
+				}
+			}
+		}
+		if hits > 0 {
+			sq.logger.Printf("background scan: %d fragments needed snapshots\n", hits)
+			hits = 0
+		} else {
+			sq.logger.Printf("background scan: no fragments needed snapshots, waiting\n")
+			// No reason to be active if we're not finding anything.
+			select {
+			case <-time.After(60 * time.Second):
+			case <-done:
+				return
+			}
+		}
+	}
+}

test/logger.go

@@ -18,12 +18,14 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
+	"sync"
 )
 
 // bufferLogger represents a test Logger that holds log messages
 // in a buffer for review.
 type bufferLogger struct {
 	buf *bytes.Buffer
+	mu  sync.Mutex
 }
 
 // NewBufferLogger returns a new instance of BufferLogger.
@@ -34,6 +36,8 @@ func NewBufferLogger() *bufferLogger {
 }
 
 func (b *bufferLogger) Printf(format string, v ...interface{}) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
 	s := fmt.Sprintf(format, v...)
 	_, err := b.buf.WriteString(s)
 	if err != nil {
@@ -44,5 +48,7 @@ func (b *bufferLogger) Printf(format string, v ...interface{}) {
 func (b *bufferLogger) Debugf(format string, v ...interface{}) {}
 
 func (b *bufferLogger) ReadAll() ([]byte, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
 	return ioutil.ReadAll(b.buf)
 }

utils_internal_test.go

@@ -301,6 +301,9 @@ func (t *ClusterCluster) Close() error {
 		if err != nil {
 			return err
 		}
+		// Make sure open indexes get shut down too. we wouldn't do
+		// this normally for a cluster, but we want to for test cases.
+		c.holder.Close()
 	}
 	return nil
 }

view.go

@@ -59,7 +59,7 @@ type view struct {
 	stats         stats.StatsClient
 	rowAttrStore  AttrStore
 	logger        logger.Logger
-	snapshotQueue chan *fragment
+	snapshotQueue snapshotQueue
 }
 
 // newView returns a new instance of View.
@@ -309,7 +309,9 @@ func (v *view) newFragment(path string, shard uint64) *fragment {
 	frag.CacheSize = v.cacheSize
 	frag.Logger = v.logger
 	frag.stats = v.stats
-	frag.snapshotQueue = v.snapshotQueue
+	if v.snapshotQueue != nil {
+		frag.snapshotQueue = v.snapshotQueue
+	}
 	if v.fieldType == FieldTypeMutex {
 		frag.mutexVector = newRowsVector(frag)
 	} else if v.fieldType == FieldTypeBool {
@@ -483,7 +485,7 @@ func upgradeViewBSIv2(v *view, bitDepth uint) (ok bool, _ error) {
 
 		if tmpPath, err := upgradeRoaringBSIv2(frag, bitDepth); err != nil {
 			return ok, errors.Wrap(err, "upgrading bsi v2")
-		} else if err := frag.closeStorage(true); err != nil {
+		} else if err := frag.closeStorage(); err != nil {
 			return ok, errors.Wrap(err, "closing after bsi v2 upgrade")
 		} else if err := os.Rename(tmpPath, frag.path); err != nil {
 			return ok, errors.Wrap(err, "renaming after bsi v2 upgrade")