diff --git a/api.go b/api.go index 23c36cb0b..2d52676c7 100644 --- a/api.go +++ b/api.go @@ -370,7 +370,8 @@ func importWorker(importWork chan importJob) { var doClear bool switch doAction { case RequestActionOverwrite: - if err := j.field.importRoaringOverwrite(j.ctx, viewData, j.shard, viewName, j.req.Block); err != nil { + tx := &RoaringTx{Field: j.field} + if err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block); err != nil { return errors.Wrap(err, "importing roaring as overwrite") } case RequestActionClear: @@ -581,6 +582,9 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin return ErrFragmentNotFound } + // Obtain transaction + tx := &RoaringTx{Index: index} + // Wrap writer with a CSV writer. cw := csv.NewWriter(w) @@ -616,7 +620,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin } // Iterate over each column. - if err := f.forEachBit(fn); err != nil { + if err := f.forEachBit(tx, fn); err != nil { return errors.Wrap(err, "writing CSV") } @@ -643,7 +647,7 @@ func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) // FragmentBlockData is an endpoint for internal usage. It is not guaranteed to // return anything useful. Currently it returns protobuf encoded row and column // ids from a "block" which is a subdivision of a fragment. -func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { +func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) (_ []byte, err error) { span, _ := tracing.StartSpanFromContext(ctx, "API.FragmentBlockData") defer span.Finish() @@ -667,7 +671,10 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } var resp = BlockDataResponse{} - resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block)) + resp.RowIDs, resp.ColumnIDs, err = f.blockData(int(req.Block)) + if err != nil { + return nil, err + } // Encode response. buf, err := api.Serializer.Marshal(&resp) @@ -694,8 +701,7 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName, fieldName, viewNa } // Retrieve blocks. - blocks := f.Blocks() - return blocks, nil + return f.Blocks() } // FragmentData returns all data in the specified fragment. @@ -1054,6 +1060,9 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp return errors.Wrap(err, "getting index and field") } + // Obtain transaction. + tx := &RoaringTx{Index: index} + if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { return errors.Wrap(err, "validating import value request") } @@ -1146,14 +1155,14 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp // Import columnIDs into existence field. if !options.Clear { - if err := importExistenceColumns(index, req.ColumnIDs); err != nil { + if err := importExistenceColumns(tx, index, req.ColumnIDs); err != nil { api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) return errors.Wrap(err, "importing existence columns") } } // Import into fragment. - err = field.Import(req.RowIDs, req.ColumnIDs, timestamps, opts...) + err = field.Import(tx, req.RowIDs, req.ColumnIDs, timestamps, opts...) if err != nil { api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) } @@ -1178,6 +1187,9 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . return errors.Wrap(err, "validating import value request") } + // Obtain transaction. + tx := &RoaringTx{Index: index} + // Set up import options. options, err := setUpImportOptions(opts...) if err != nil { @@ -1244,7 +1256,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . } // Import columnIDs into existence field. if !options.Clear { - if err := importExistenceColumns(index, req.ColumnIDs); err != nil { + if err := importExistenceColumns(tx, index, req.ColumnIDs); err != nil { api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) return errors.Wrap(err, "importing existence columns") } @@ -1252,12 +1264,12 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . // Import into fragment. if len(req.Values) > 0 { - err = field.importValue(req.ColumnIDs, req.Values, options) + err = field.importValue(tx, req.ColumnIDs, req.Values, options) if err != nil { api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) } } else if len(req.FloatValues) > 0 { - err = field.importFloatValue(req.ColumnIDs, req.FloatValues, options) + err = field.importFloatValue(tx, req.ColumnIDs, req.FloatValues, options) if err != nil { api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) } @@ -1342,14 +1354,14 @@ func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsReq return nil } -func importExistenceColumns(index *Index, columnIDs []uint64) error { +func importExistenceColumns(tx Tx, index *Index, columnIDs []uint64) error { ef := index.existenceField() if ef == nil { return nil } existenceRowIDs := make([]uint64, len(columnIDs)) - return ef.Import(existenceRowIDs, columnIDs, nil) + return ef.Import(tx, existenceRowIDs, columnIDs, nil) } // MaxShards returns the maximum shard number for each index in a map. diff --git a/cluster.go b/cluster.go index 669d2bbce..9695a4dd1 100644 --- a/cluster.go +++ b/cluster.go @@ -755,7 +755,7 @@ func (c *cluster) fragsByHost(idx *Index) fragsByHost { // for the given set of shards with data. func (c *cluster) fragCombos(idx string, availableShards *roaring.Bitmap, fieldViews viewsByField) fragsByHost { t := make(fragsByHost) - availableShards.ForEach(func(i uint64) { + _ = availableShards.ForEach(func(i uint64) error { nodes := c.shardNodes(idx, i) for _, n := range nodes { // for each field/view combination: @@ -765,6 +765,7 @@ func (c *cluster) fragCombos(idx string, availableShards *roaring.Bitmap, fieldV } } } + return nil }) return t } @@ -1060,7 +1061,7 @@ func (c *cluster) unprotectedOwnsPartition(nodeID string, partition int) bool { // containsShards is like OwnsShards, but it includes replicas. func (c *cluster) containsShards(index string, availableShards *roaring.Bitmap, node *Node) []uint64 { var shards []uint64 - availableShards.ForEach(func(i uint64) { + _ = availableShards.ForEach(func(i uint64) error { p := c.shardPartition(index, i) // Determine the nodes for partition. nodes := c.partitionNodes(p) @@ -1069,6 +1070,7 @@ func (c *cluster) containsShards(index string, availableShards *roaring.Bitmap, shards = append(shards, i) } } + return nil }) return shards } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 059053423..00492d068 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -105,7 +105,6 @@ func newIndexWithTempPath(name string) *Index { // Ensure that fragSources creates the correct fragment mapping. func TestFragSources(t *testing.T) { - uri0, err := NewURIFromAddress("host0") if err != nil { t.Fatal(err) @@ -159,23 +158,28 @@ func TestFragSources(t *testing.T) { idx := newIndexWithTempPath("i") defer idx.Close() + + // Obtain transaction. + tx := &RoaringTx{Index: idx} + defer func() { _ = tx.Rollback() }() + field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault()) if err != nil { t.Fatal(err) } - _, err = field.SetBit(1, 101, nil) + _, err = field.SetBit(tx, 1, 101, nil) if err != nil { t.Fatal(err) } - _, err = field.SetBit(1, ShardWidth+1, nil) + _, err = field.SetBit(tx, 1, ShardWidth+1, nil) if err != nil { t.Fatal(err) } - _, err = field.SetBit(1, ShardWidth*2+1, nil) + _, err = field.SetBit(tx, 1, ShardWidth*2+1, nil) if err != nil { t.Fatal(err) } - _, err = field.SetBit(1, ShardWidth*3+1, nil) + _, err = field.SetBit(tx, 1, ShardWidth*3+1, nil) if err != nil { t.Fatal(err) } @@ -795,7 +799,10 @@ func TestCluster_ResizeStates(t *testing.T) { node0Field := node0.holder.Field("i", "f") node0View := node0Field.view("standard") node0Fragment := node0View.Fragment(1) - node0Checksum := node0Fragment.Checksum() + node0Checksum, err := node0Fragment.Checksum() + if err != nil { + t.Fatal(err) + } // addNode needs to block until the resize process has completed. if err := tc.addNode(); err != nil { @@ -828,7 +835,9 @@ func TestCluster_ResizeStates(t *testing.T) { node1Fragment := node1View.Fragment(1) // Ensure checksums are the same. - if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) { + if chksum, err := node1Fragment.Checksum(); err != nil { + t.Fatal(err) + } else if !bytes.Equal(chksum, node0Checksum) { t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum) } diff --git a/cmd/inspect.go b/cmd/convert.go similarity index 100% rename from cmd/inspect.go rename to cmd/convert.go diff --git a/executor.go b/executor.go index 25f5c074e..dcfaf39ed 100644 --- a/executor.go +++ b/executor.go @@ -209,7 +209,15 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar return resp, fmt.Errorf("profiling execution failed: %T is not tracing.Profile", prof) } } - results, err := e.execute(ctx, index, q, shards, opt) + + // TODO: Determine if query is read-only. + tx, err := e.Holder.Begin(true) + if err != nil { + return resp, err + } + defer func() { _ = tx.Rollback() }() + + results, err := e.execute(ctx, tx, index, q, shards, opt) if err != nil { return resp, err } else if err := validateQueryContext(ctx); err != nil { @@ -266,6 +274,11 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar } } + // Commit transaction. + if err := tx.Commit(); err != nil { + return resp, err + } + return resp, nil } @@ -294,7 +307,7 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr // handlePreCalls traverses the call tree looking for calls that need // precomputed values. Right now, that's just Distinct. -func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error { +func (e *executor) handlePreCalls(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) error { if c.Name == "Precomputed" { idx := c.Args["valueidx"].(int64) if idx >= 0 && idx < int64(len(opt.EmbeddedData)) { @@ -329,7 +342,7 @@ func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call // like Distinct, where you can't predict output shard for a result // from the shard being queried. if newIndex != "" && newIndex != index { - if err := e.handlePreCallChildren(ctx, index, c, shards, opt); err != nil { + if err := e.handlePreCallChildren(ctx, tx, index, c, shards, opt); err != nil { return err } @@ -340,7 +353,7 @@ func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call } if c.Type == pql.PrecallNone { // otherwise, handle the children - return e.handlePreCallChildren(ctx, index, c, shards, opt) + return e.handlePreCallChildren(ctx, tx, index, c, shards, opt) } // We don't try to handle sub-calls from here. I'm not 100% // sure that's right, but I think the fact that they're happening @@ -351,7 +364,7 @@ func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call // We set c to look like a normal call, and actually execute it: c.Type = pql.PrecallNone // possibly override call index. - v, err := e.executeCall(ctx, index, c, shards, opt) + v, err := e.executeCall(ctx, tx, index, c, shards, opt) if err != nil { return err } @@ -392,12 +405,12 @@ func (e *executor) dumpPrecomputedCalls(ctx context.Context, c *pql.Call) { } // handlePreCallChildren handles any pre-calls in the children of a given call. -func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error { +func (e *executor) handlePreCallChildren(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) error { for i := range c.Children { if err := ctx.Err(); err != nil { return err } - if err := e.handlePreCalls(ctx, index, c.Children[i], shards, opt); err != nil { + if err := e.handlePreCalls(ctx, tx, index, c.Children[i], shards, opt); err != nil { return err } } @@ -407,7 +420,7 @@ func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *p if err := ctx.Err(); err != nil { return err } - if err := e.handlePreCalls(ctx, index, call, shards, opt); err != nil { + if err := e.handlePreCalls(ctx, tx, index, call, shards, opt); err != nil { return err } } @@ -415,7 +428,7 @@ func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *p return nil } -func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) { +func (e *executor) execute(ctx context.Context, tx Tx, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.execute") defer span.Finish() @@ -438,7 +451,7 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar // Optimize handling for bulk attribute insertion. if hasOnlySetRowAttrs(q.Calls) { - return e.executeBulkSetRowAttrs(ctx, index, q.Calls, opt) + return e.executeBulkSetRowAttrs(ctx, tx, index, q.Calls, opt) } // Execute each call serially. @@ -454,7 +467,7 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar // about the positive values, because only positive values // are valid column IDs. So we don't actually eat top-level // pre calls. - err := e.handlePreCallChildren(ctx, index, call, shards, opt) + err := e.handlePreCallChildren(ctx, tx, index, call, shards, opt) if err != nil { return nil, err } @@ -465,9 +478,9 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar // already precomputed by handlePreCallChildren, though, // we don't need this logic in executeCall. if newIndex := call.CallIndex(); newIndex != "" && newIndex != index { - v, err = e.executeCall(ctx, newIndex, call, nil, opt) + v, err = e.executeCall(ctx, tx, newIndex, call, nil, opt) } else { - v, err = e.executeCall(ctx, index, call, shards, opt) + v, err = e.executeCall(ctx, tx, index, call, shards, opt) } if err != nil { return nil, err @@ -483,7 +496,7 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar } // executeCall executes a call. -func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { +func (e *executor) executeCall(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCall") defer span.Finish() @@ -524,74 +537,74 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s // Special handling for mutation and top-n calls. if op, ok := e.additionalCountOps[c.Name]; ok { statFn() - return e.executeGenericCount(ctx, index, c, op, shards, opt) + return e.executeGenericCount(ctx, tx, index, c, op, shards, opt) } if op, ok := e.additionalFieldOps[c.Name]; ok { statFn() - return e.executeGenericField(ctx, index, c, op, shards, opt) + return e.executeGenericField(ctx, tx, index, c, op, shards, opt) } switch c.Name { case "Sum": statFn() - return e.executeSum(ctx, index, c, shards, opt) + return e.executeSum(ctx, tx, index, c, shards, opt) case "Min": statFn() - return e.executeMin(ctx, index, c, shards, opt) + return e.executeMin(ctx, tx, index, c, shards, opt) case "Max": statFn() - return e.executeMax(ctx, index, c, shards, opt) + return e.executeMax(ctx, tx, index, c, shards, opt) case "MinRow": statFn() - return e.executeMinRow(ctx, index, c, shards, opt) + return e.executeMinRow(ctx, tx, index, c, shards, opt) case "MaxRow": statFn() - return e.executeMaxRow(ctx, index, c, shards, opt) + return e.executeMaxRow(ctx, tx, index, c, shards, opt) case "Clear": statFn() - return e.executeClearBit(ctx, index, c, opt) + return e.executeClearBit(ctx, tx, index, c, opt) case "ClearRow": statFn() - return e.executeClearRow(ctx, index, c, shards, opt) + return e.executeClearRow(ctx, tx, index, c, shards, opt) case "Store": statFn() - return e.executeSetRow(ctx, index, c, shards, opt) + return e.executeSetRow(ctx, tx, index, c, shards, opt) case "Count": statFn() - return e.executeCount(ctx, index, c, shards, opt) + return e.executeCount(ctx, tx, index, c, shards, opt) case "Set": statFn() - return e.executeSet(ctx, index, c, opt) + return e.executeSet(ctx, tx, index, c, opt) case "SetRowAttrs": statFn() - return nil, e.executeSetRowAttrs(ctx, index, c, opt) + return nil, e.executeSetRowAttrs(ctx, tx, index, c, opt) case "SetColumnAttrs": statFn() - return nil, e.executeSetColumnAttrs(ctx, index, c, opt) + return nil, e.executeSetColumnAttrs(ctx, tx, index, c, opt) case "TopN": statFn() - return e.executeTopN(ctx, index, c, shards, opt) + return e.executeTopN(ctx, tx, index, c, shards, opt) case "Rows": statFn() - return e.executeRows(ctx, index, c, shards, opt) + return e.executeRows(ctx, tx, index, c, shards, opt) case "GroupBy": statFn() - return e.executeGroupBy(ctx, index, c, shards, opt) + return e.executeGroupBy(ctx, tx, index, c, shards, opt) case "Options": statFn() - return e.executeOptionsCall(ctx, index, c, shards, opt) + return e.executeOptionsCall(ctx, tx, index, c, shards, opt) case "IncludesColumn": - return e.executeIncludesColumnCall(ctx, index, c, shards, opt) + return e.executeIncludesColumnCall(ctx, tx, index, c, shards, opt) case "FieldValue": statFn() - return e.executeFieldValueCall(ctx, index, c, shards, opt) + return e.executeFieldValueCall(ctx, tx, index, c, shards, opt) case "All": statFn() - return e.executeAllCall(ctx, index, c, shards, opt) + return e.executeAllCall(ctx, tx, index, c, shards, opt) case "Precomputed": - return e.executePrecomputedCall(ctx, index, c, shards, opt) + return e.executePrecomputedCall(ctx, tx, index, c, shards, opt) default: statFn() - return e.executeBitmapCall(ctx, index, c, shards, opt) + return e.executeBitmapCall(ctx, tx, index, c, shards, opt) } } @@ -614,7 +627,7 @@ func (e *executor) validateCallArgs(c *pql.Call) error { return nil } -func (e *executor) executeOptionsCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { +func (e *executor) executeOptionsCall(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeOptionsCall") defer span.Finish() @@ -656,11 +669,11 @@ func (e *executor) executeOptionsCall(ctx context.Context, index string, c *pql. return nil, errors.New("Query(): shards must be a list of unsigned integers") } } - return e.executeCall(ctx, index, c.Children[0], shards, optCopy) + return e.executeCall(ctx, tx, index, c.Children[0], shards, optCopy) } // executeIncludesColumnCall executes an IncludesColumn() call. -func (e *executor) executeIncludesColumnCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { +func (e *executor) executeIncludesColumnCall(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { // Get the shard containing the column, since that's the only // shard that needs to execute this query. var shard uint64 @@ -679,7 +692,7 @@ func (e *executor) executeIncludesColumnCall(ctx context.Context, index string, // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeIncludesColumnCallShard(ctx, index, c, shard, col) + return e.executeIncludesColumnCallShard(ctx, tx, index, c, shard, col) } // Merge returned results at coordinating node. @@ -696,7 +709,7 @@ func (e *executor) executeIncludesColumnCall(ctx context.Context, index string, } // executeFieldValueCall executes a FieldValue() call. -func (e *executor) executeFieldValueCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { +func (e *executor) executeFieldValueCall(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { fieldName, ok := c.Args["field"].(string) if !ok || fieldName == "" { return ValCount{}, ErrFieldRequired @@ -738,7 +751,7 @@ func (e *executor) executeFieldValueCall(ctx context.Context, index string, c *p // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeFieldValueCallShard(ctx, field, colID, shard) + return e.executeFieldValueCallShard(ctx, tx, field, colID, shard) } // Select single returned result at coordinating node. @@ -759,8 +772,8 @@ func (e *executor) executeFieldValueCall(ctx context.Context, index string, c *p return other, nil } -func (e *executor) executeFieldValueCallShard(ctx context.Context, field *Field, col uint64, shard uint64) (ValCount, error) { - value, exists, err := field.Value(col) +func (e *executor) executeFieldValueCallShard(ctx context.Context, tx Tx, field *Field, col uint64, shard uint64) (ValCount, error) { + value, exists, err := field.Value(tx, col) if err != nil { return ValCount{}, errors.Wrap(err, "getting field value") } else if !exists { @@ -785,7 +798,7 @@ func (e *executor) executeFieldValueCallShard(ctx context.Context, field *Field, } // executeAllCall executes an All() call. -func (e *executor) executeAllCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { +func (e *executor) executeAllCall(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { rslt := NewRow() var limit uint64 @@ -814,7 +827,7 @@ func (e *executor) executeAllCall(ctx context.Context, index string, c *pql.Call var got uint64 for _, shard := range shards { - row, err := e.executeAllCallMapReduce(ctx, index, c, shard, opt) + row, err := e.executeAllCallMapReduce(ctx, tx, index, c, shard, opt) if err != nil { return nil, errors.Wrap(err, "executing map reduce on shard") } @@ -864,10 +877,10 @@ func (e *executor) executeAllCall(ctx context.Context, index string, c *pql.Call // executeAllCallMapReduce executes a single shard of the All() call // using the executor.mapReduce() method. -func (e *executor) executeAllCallMapReduce(ctx context.Context, index string, c *pql.Call, shard uint64, opt *execOptions) (*Row, error) { +func (e *executor) executeAllCallMapReduce(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64, opt *execOptions) (*Row, error) { // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeAllCallShard(ctx, index, c, shard) + return e.executeAllCallShard(ctx, tx, index, c, shard) } // Merge returned results at coordinating node. @@ -894,12 +907,12 @@ func (e *executor) executeAllCallMapReduce(ctx context.Context, index string, c } // executeIncludesColumnCallShard -func (e *executor) executeIncludesColumnCallShard(ctx context.Context, index string, c *pql.Call, shard uint64, column uint64) (bool, error) { +func (e *executor) executeIncludesColumnCallShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64, column uint64) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIncludesColumnCallShard") defer span.Finish() if len(c.Children) == 1 { - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return false, errors.Wrap(err, "executing bitmap call") } @@ -910,7 +923,7 @@ func (e *executor) executeIncludesColumnCallShard(ctx context.Context, index str } // executeSum executes a Sum() call. -func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { +func (e *executor) executeSum(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSum") defer span.Finish() @@ -925,7 +938,7 @@ func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, sh // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeSumCountShard(ctx, index, c, nil, shard) + return e.executeSumCountShard(ctx, tx, index, c, nil, shard) } // Merge returned results at coordinating node. @@ -965,7 +978,7 @@ func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, sh // executeGenericField executes a generic call on a field. Note that in this // implementation, the operation is always a BSI op. -func (e *executor) executeGenericField(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shards []uint64, opt *execOptions) (SignedRow, error) { +func (e *executor) executeGenericField(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shards []uint64, opt *execOptions) (SignedRow, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericField") span.LogKV("name", c.Name) defer span.Finish() @@ -977,7 +990,7 @@ func (e *executor) executeGenericField(ctx context.Context, index string, c *pql // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeGenericFieldShard(ctx, index, c, op, shard) + return e.executeGenericFieldShard(ctx, tx, index, c, op, shard) } // Merge returned results at coordinating node. @@ -1000,7 +1013,7 @@ func (e *executor) executeGenericField(ctx context.Context, index string, c *pql } // executeMin executes a Min() call. -func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { +func (e *executor) executeMin(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMin") defer span.Finish() if field := c.Args["field"]; field == "" { @@ -1013,7 +1026,7 @@ func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, sh // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeMinShard(ctx, index, c, shard) + return e.executeMinShard(ctx, tx, index, c, shard) } // Merge returned results at coordinating node. @@ -1035,7 +1048,7 @@ func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, sh } // executeMax executes a Max() call. -func (e *executor) executeMax(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { +func (e *executor) executeMax(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMax") defer span.Finish() @@ -1049,7 +1062,7 @@ func (e *executor) executeMax(ctx context.Context, index string, c *pql.Call, sh // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeMaxShard(ctx, index, c, shard) + return e.executeMaxShard(ctx, tx, index, c, shard) } // Merge returned results at coordinating node. @@ -1071,7 +1084,7 @@ func (e *executor) executeMax(ctx context.Context, index string, c *pql.Call, sh } // executeMinRow executes a MinRow() call. -func (e *executor) executeMinRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { +func (e *executor) executeMinRow(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMinRow") defer span.Finish() @@ -1081,7 +1094,7 @@ func (e *executor) executeMinRow(ctx context.Context, index string, c *pql.Call, // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeMinRowShard(ctx, index, c, shard) + return e.executeMinRowShard(ctx, tx, index, c, shard) } // Merge returned results at coordinating node. @@ -1110,7 +1123,7 @@ func (e *executor) executeMinRow(ctx context.Context, index string, c *pql.Call, } // executeMaxRow executes a MaxRow() call. -func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { +func (e *executor) executeMaxRow(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMaxRow") defer span.Finish() @@ -1120,7 +1133,7 @@ func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call, // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeMaxRowShard(ctx, index, c, shard) + return e.executeMaxRowShard(ctx, tx, index, c, shard) } // Merge returned results at coordinating node. @@ -1149,7 +1162,7 @@ func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call, } // executePrecomputedCall pretends to execute a call that we have a precomputed value for. -func (e *executor) executePrecomputedCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { +func (e *executor) executePrecomputedCall(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.executePrecomputedCall") defer span.Finish() result := NewRow() @@ -1161,7 +1174,7 @@ func (e *executor) executePrecomputedCall(ctx context.Context, index string, c * } // executeBitmapCall executes a call that returns a bitmap. -func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { +func (e *executor) executeBitmapCall(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCall") span.LogKV("pqlCallName", c.Name) defer span.Finish() @@ -1177,7 +1190,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeBitmapCallShard(ctx, index, c, shard) + return e.executeBitmapCallShard(ctx, tx, index, c, shard) } // Merge returned results at coordinating node. @@ -1243,7 +1256,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } // executeBitmapCallShard executes a bitmap call for a single shard. -func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { +func (e *executor) executeBitmapCallShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) { if err := validateQueryContext(ctx); err != nil { return nil, err } @@ -1255,28 +1268,28 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * return nil, fmt.Errorf("count op %s used as bitmap call", c.Name) } if op, ok := e.additionalBitmapOps[c.Name]; ok { - return e.executeGenericBitmapShard(ctx, index, c, op, shard) + return e.executeGenericBitmapShard(ctx, tx, index, c, op, shard) } switch c.Name { case "Row", "Range": - return e.executeRowShard(ctx, index, c, shard) + return e.executeRowShard(ctx, tx, index, c, shard) case "Difference": - return e.executeDifferenceShard(ctx, index, c, shard) + return e.executeDifferenceShard(ctx, tx, index, c, shard) case "Intersect": - return e.executeIntersectShard(ctx, index, c, shard) + return e.executeIntersectShard(ctx, tx, index, c, shard) case "Union": - return e.executeUnionShard(ctx, index, c, shard) + return e.executeUnionShard(ctx, tx, index, c, shard) case "Xor": - return e.executeXorShard(ctx, index, c, shard) + return e.executeXorShard(ctx, tx, index, c, shard) case "Not": - return e.executeNotShard(ctx, index, c, shard) + return e.executeNotShard(ctx, tx, index, c, shard) case "Shift": - return e.executeShiftShard(ctx, index, c, shard) + return e.executeShiftShard(ctx, tx, index, c, shard) case "All": // Allow a shard computation to use All() (note, limit/offset not applied) - return e.executeAllCallShard(ctx, index, c, shard) + return e.executeAllCallShard(ctx, tx, index, c, shard) case "Precomputed": - return e.executePrecomputedCallShard(ctx, index, c, shard) + return e.executePrecomputedCallShard(ctx, tx, index, c, shard) default: return nil, fmt.Errorf("unknown call: %s", c.Name) } @@ -1285,14 +1298,14 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * // executeGenericFieldShard executes a generic/extension command on a // single shard. Note that in this implementation, the op is always // a BSI op. -func (e *executor) executeGenericFieldShard(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shard uint64) (SignedRow, error) { +func (e *executor) executeGenericFieldShard(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shard uint64) (SignedRow, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericShard") defer span.Finish() var filter *Row var filterBitmap *roaring.Bitmap if len(c.Children) == 1 { - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return SignedRow{}, errors.Wrap(err, "executing bitmap call") } @@ -1335,13 +1348,13 @@ func (e *executor) executeGenericFieldShard(ctx context.Context, index string, c } // executeSumCountShard calculates the sum and count for bsiGroups on a shard. -func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pql.Call, filter *Row, shard uint64) (ValCount, error) { +func (e *executor) executeSumCountShard(ctx context.Context, tx Tx, index string, c *pql.Call, filter *Row, shard uint64) (ValCount, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSumCountShard") defer span.Finish() // Only calculate the filter if it doesn't exist and a child call as been passed in. if filter == nil && len(c.Children) == 1 { - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return ValCount{}, errors.Wrap(err, "executing bitmap call") } @@ -1367,7 +1380,7 @@ func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pq sumspan, _ := tracing.StartSpanFromContext(ctx, "Executor.executeSumCountShard_fragment.sum") defer sumspan.Finish() - vsum, vcount, err := fragment.sum(filter, bsig.BitDepth) + vsum, vcount, err := fragment.sum(tx, filter, bsig.BitDepth) if err != nil { return ValCount{}, errors.Wrap(err, "computing sum") } @@ -1378,13 +1391,13 @@ func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pq } // executeMinShard calculates the min for bsiGroups on a shard. -func (e *executor) executeMinShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { +func (e *executor) executeMinShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (ValCount, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMinShard") defer span.Finish() var filter *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return ValCount{}, err } @@ -1398,14 +1411,14 @@ func (e *executor) executeMinShard(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - return field.MinForShard(shard, filter) + return field.MinForShard(tx, shard, filter) } // executeMaxShard calculates the max for bsiGroups on a shard. -func (e *executor) executeMaxShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { +func (e *executor) executeMaxShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return ValCount{}, err } @@ -1419,14 +1432,14 @@ func (e *executor) executeMaxShard(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - return field.MaxForShard(shard, filter) + return field.MaxForShard(tx, shard, filter) } // executeMinRowShard returns the minimum row ID for a shard. -func (e *executor) executeMinRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (PairField, error) { +func (e *executor) executeMinRowShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (PairField, error) { var filter *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return PairField{}, err } @@ -1444,7 +1457,11 @@ func (e *executor) executeMinRowShard(ctx context.Context, index string, c *pql. return PairField{}, nil } - minRowID, count := fragment.minRow(filter) + minRowID, count, err := fragment.minRow(tx, filter) + if err != nil { + return PairField{}, err + } + return PairField{ Pair: Pair{ ID: minRowID, @@ -1455,10 +1472,10 @@ func (e *executor) executeMinRowShard(ctx context.Context, index string, c *pql. } // executeMaxRowShard returns the maximum row ID for a shard. -func (e *executor) executeMaxRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (PairField, error) { +func (e *executor) executeMaxRowShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (PairField, error) { var filter *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return PairField{}, err } @@ -1476,7 +1493,11 @@ func (e *executor) executeMaxRowShard(ctx context.Context, index string, c *pql. return PairField{}, nil } - maxRowID, count := fragment.maxRow(filter) + maxRowID, count, err := fragment.maxRow(tx, filter) + if err != nil { + return PairField{}, nil + } + return PairField{ Pair: Pair{ ID: maxRowID, @@ -1489,7 +1510,7 @@ func (e *executor) executeMaxRowShard(ctx context.Context, index string, c *pql. // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. -func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*PairsField, error) { +func (e *executor) executeTopN(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*PairsField, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopN") defer span.Finish() @@ -1505,7 +1526,7 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s } // Execute original query. - pairs, err := e.executeTopNShards(ctx, index, c, shards, opt) + pairs, err := e.executeTopNShards(ctx, tx, index, c, shards, opt) if err != nil { return nil, errors.Wrap(err, "finding top results") } @@ -1526,7 +1547,7 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s sort.Sort(uint64Slice(ids)) other.Args["ids"] = ids - trimmedList, err := e.executeTopNShards(ctx, index, other, shards, opt) + trimmedList, err := e.executeTopNShards(ctx, tx, index, other, shards, opt) if err != nil { return nil, errors.Wrap(err, "retrieving full counts") } @@ -1541,13 +1562,13 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s }, nil } -func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*PairsField, error) { +func (e *executor) executeTopNShards(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*PairsField, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShards") defer span.Finish() // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeTopNShard(ctx, index, c, shard) + return e.executeTopNShard(ctx, tx, index, c, shard) } // Merge returned results at coordinating node. @@ -1579,7 +1600,7 @@ func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.C } // executeTopNShard executes a TopN call for a single shard. -func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*PairsField, error) { +func (e *executor) executeTopNShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*PairsField, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShard") defer span.Finish() @@ -1609,7 +1630,7 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca // Retrieve bitmap used to intersect. var src *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return nil, err } @@ -1637,7 +1658,7 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca if tanimotoThreshold > 100 { return nil, errors.New("Tanimoto Threshold is from 1 to 100 only") } - pairs, err := f.top(topOptions{ + pairs, err := f.top(tx, topOptions{ N: int(n), Src: src, RowIDs: rowIDs, @@ -1656,7 +1677,7 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca } // executeDifferenceShard executes a difference() call for a local shard. -func (e *executor) executeDifferenceShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { +func (e *executor) executeDifferenceShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDifferenceShard") defer span.Finish() @@ -1665,7 +1686,7 @@ func (e *executor) executeDifferenceShard(ctx context.Context, index string, c * return nil, fmt.Errorf("empty Difference query is currently not supported") } for i, input := range c.Children { - row, err := e.executeBitmapCallShard(ctx, index, input, shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, input, shard) if err != nil { return nil, err } @@ -1771,7 +1792,7 @@ func (r RowIDs) merge(other RowIDs, limit int) RowIDs { return result } -func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]GroupCount, error) { +func (e *executor) executeGroupBy(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]GroupCount, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGroupBy") defer span.Finish() // validate call @@ -1832,7 +1853,7 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call } if hasLimit || hasCol { // we need to perform this query cluster-wide ahead of executeGroupByShard - childRows[i], err = e.executeRows(ctx, index, child, shards, opt) + childRows[i], err = e.executeRows(ctx, tx, index, child, shards, opt) if err != nil { return nil, errors.Wrap(err, "getting rows for ") } @@ -1844,7 +1865,7 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeGroupByShard(ctx, index, c, filter, shard, childRows, bases) + return e.executeGroupByShard(ctx, tx, index, c, filter, shard, childRows, bases) } // Merge returned results at coordinating node. reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { @@ -2195,13 +2216,13 @@ func applyConditionToGroupCounts(gcs []GroupCount, subj string, cond *pql.Condit return gcs[:i] } -func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs, bases map[int]int64) (_ []GroupCount, err error) { +func (e *executor) executeGroupByShard(ctx context.Context, tx Tx, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs, bases map[int]int64) (_ []GroupCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGroupByShard") defer span.Finish() var filterRow *Row if filter != nil { - if filterRow, err = e.executeBitmapCallShard(ctx, index, filter, shard); err != nil { + if filterRow, err = e.executeBitmapCallShard(ctx, tx, index, filter, shard); err != nil { return nil, errors.Wrapf(err, "executing group by filter for shard %d", shard) } } @@ -2212,7 +2233,7 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql } newspan, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGroupByShard_newGroupByIterator") - iter, err := newGroupByIterator(e, childRows, c.Children, aggregate, filterRow, index, shard, e.Holder) + iter, err := newGroupByIterator(e, tx, childRows, c.Children, aggregate, filterRow, index, shard, e.Holder) newspan.Finish() if err != nil { @@ -2253,7 +2274,7 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql return results, nil } -func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { +func (e *executor) executeRows(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { // Fetch field name from argument. // Check "field" first for backwards compatibility. // TODO: remove at Pilosa 2.0 @@ -2273,7 +2294,7 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeRowsShard(ctx, index, fieldName, c, shard) + return e.executeRowsShard(ctx, tx, index, fieldName, c, shard) } // Determine limit so we can use it when reducing. @@ -2301,7 +2322,7 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s return results, nil } -func (e *executor) executeRowsShard(ctx context.Context, index string, fieldName string, c *pql.Call, shard uint64) (RowIDs, error) { +func (e *executor) executeRowsShard(ctx context.Context, tx Tx, index string, fieldName string, c *pql.Call, shard uint64) (RowIDs, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { @@ -2419,20 +2440,23 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, fieldName continue } - viewRows := frag.rows(ctx, start, filters...) + viewRows, err := frag.rows(ctx, tx, start, filters...) + if err != nil { + return nil, err + } rowIDs = rowIDs.merge(viewRows, limit) } return rowIDs, nil } -func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { +func (e *executor) executeRowShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowShard") defer span.Finish() // Handle bsiGroup ranges differently. if c.HasConditionArg() { - return e.executeRowBSIGroupShard(ctx, index, c, shard) + return e.executeRowBSIGroupShard(ctx, tx, index, c, shard) } // Fetch index. @@ -2480,7 +2504,7 @@ func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Cal Value: v, } - return e.executeRowBSIGroupShard(ctx, index, c, shard) + return e.executeRowBSIGroupShard(ctx, tx, index, c, shard) } } } @@ -2498,7 +2522,7 @@ func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Cal if frag == nil { return NewRow(), nil } - return frag.row(rowID), nil + return frag.row(tx, rowID) } // If no quantum exists then return an empty bitmap. @@ -2522,7 +2546,11 @@ func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Cal if f == nil { continue } - rows = append(rows, f.row(rowID)) + row, err := f.row(tx, rowID) + if err != nil { + return nil, err + } + rows = append(rows, row) } if len(rows) == 0 { return &Row{}, nil @@ -2535,7 +2563,7 @@ func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Cal } // executeRowBSIGroupShard executes a range(bsiGroup) call for a local shard. -func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { +func (e *executor) executeRowBSIGroupShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (_ *Row, err error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowBSIGroupShard") defer span.Finish() @@ -2577,7 +2605,7 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c return NewRow(), nil } - return frag.notNull() + return frag.notNull(tx) } else if cond.Op == pql.EQ && cond.Value == nil { // Make sure the index supports existence tracking. @@ -2593,7 +2621,9 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c if existenceFrag == nil { existenceRow = NewRow() } else { - existenceRow = existenceFrag.row(0) + if existenceRow, err = existenceFrag.row(tx, 0); err != nil { + return nil, err + } } var notNull *Row @@ -2601,7 +2631,7 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c // Retrieve notNull from fragment if it exists. if frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard); frag != nil { - if notNull, err = frag.notNull(); err != nil { + if notNull, err = frag.notNull(tx); err != nil { return nil, errors.Wrap(err, "getting fragment not null") } } else { @@ -2646,10 +2676,10 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c // If the query is asking for the entire valid range, just return // the not-null bitmap for the bsiGroup. if predicates[0] <= bsig.Min && predicates[1] >= bsig.Max { - return frag.notNull() + return frag.notNull(tx) } - return frag.rangeBetween(bsig.BitDepth, baseValueMin, baseValueMax) + return frag.rangeBetween(tx, bsig.BitDepth, baseValueMin, baseValueMax) } else { value, err := getScaledInt(f, cond.Value) @@ -2677,20 +2707,20 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range. if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) || (cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) { - return frag.notNull() + return frag.notNull(tx) } // outOfRange for NEQ should return all not-null. if outOfRange && cond.Op == pql.NEQ { - return frag.notNull() + return frag.notNull(tx) } - return frag.rangeOp(cond.Op, bsig.BitDepth, baseValue) + return frag.rangeOp(tx, cond.Op, bsig.BitDepth, baseValue) } } // executeIntersectShard executes a intersect() call for a local shard. -func (e *executor) executeIntersectShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { +func (e *executor) executeIntersectShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIntersectShard") defer span.Finish() @@ -2699,7 +2729,7 @@ func (e *executor) executeIntersectShard(ctx context.Context, index string, c *p return nil, fmt.Errorf("empty Intersect query is currently not supported") } for i, input := range c.Children { - row, err := e.executeBitmapCallShard(ctx, index, input, shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, input, shard) if err != nil { return nil, err } @@ -2715,7 +2745,7 @@ func (e *executor) executeIntersectShard(ctx context.Context, index string, c *p } // executeGenericBitmapShard executes a generic bitmap call for a local shard. -func (e *executor) executeGenericBitmapShard(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBitmap, shard uint64) (*Row, error) { +func (e *executor) executeGenericBitmapShard(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBitmap, shard uint64) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericBitmapShard") defer span.Finish() @@ -2723,7 +2753,7 @@ func (e *executor) executeGenericBitmapShard(ctx context.Context, index string, if len(c.Children) != 1 { return nil, fmt.Errorf("%s needs exactly one row parameter", c.Name) } - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return nil, err } @@ -2733,7 +2763,7 @@ func (e *executor) executeGenericBitmapShard(ctx context.Context, index string, var err error rows := make([]*Row, len(c.Children)) for i, input := range c.Children { - rows[i], err = e.executeBitmapCallShard(ctx, index, input, shard) + rows[i], err = e.executeBitmapCallShard(ctx, tx, index, input, shard) if err != nil { return nil, err } @@ -2753,13 +2783,13 @@ func (e *executor) executeGenericBitmapShard(ctx context.Context, index string, } // executeUnionShard executes a union() call for a local shard. -func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { +func (e *executor) executeUnionShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard") defer span.Finish() other := NewRow() for i, input := range c.Children { - row, err := e.executeBitmapCallShard(ctx, index, input, shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, input, shard) if err != nil { return nil, err } @@ -2775,13 +2805,13 @@ func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.C } // executeXorShard executes a xor() call for a local shard. -func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { +func (e *executor) executeXorShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeXorShard") defer span.Finish() other := NewRow() for i, input := range c.Children { - row, err := e.executeBitmapCallShard(ctx, index, input, shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, input, shard) if err != nil { return nil, err } @@ -2797,7 +2827,7 @@ func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Cal } // executePrecomputedCallShard pretends to execute a precomputed call for a local shard. -func (e *executor) executePrecomputedCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { +func (e *executor) executePrecomputedCallShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) { if c.Precomputed != nil { v := c.Precomputed[shard] if v == nil { @@ -2816,7 +2846,7 @@ func (e *executor) executePrecomputedCallShard(ctx context.Context, index string } // executeNotShard executes a Not() call for a local shard. -func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { +func (e *executor) executeNotShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (_ *Row, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeNotShard") defer span.Finish() @@ -2839,10 +2869,12 @@ func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Cal if existenceFrag == nil { existenceRow = NewRow() } else { - existenceRow = existenceFrag.row(0) + if existenceRow, err = existenceFrag.row(tx, 0); err != nil { + return nil, err + } } - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return nil, err } @@ -2851,7 +2883,7 @@ func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Cal } // executeAllCallShard executes an All() call for a local shard. -func (e *executor) executeAllCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { +func (e *executor) executeAllCallShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (_ *Row, err error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeAllCallShard") defer span.Finish() @@ -2872,14 +2904,16 @@ func (e *executor) executeAllCallShard(ctx context.Context, index string, c *pql if existenceFrag == nil { existenceRow = NewRow() } else { - existenceRow = existenceFrag.row(0) + if existenceRow, err = existenceFrag.row(tx, 0); err != nil { + return nil, err + } } return existenceRow, nil } // executeShiftShard executes a shift() call for a local shard. -func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { +func (e *executor) executeShiftShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) { n, _, err := c.IntArg("n") if err != nil { return nil, fmt.Errorf("executeShiftShard: %v", err) @@ -2891,7 +2925,7 @@ func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.C return nil, errors.New("Shift() only accepts a single row input") } - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return nil, err } @@ -2900,7 +2934,7 @@ func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.C } // executeGeneric executes a provided count-like call. -func (e *executor) executeGenericCount(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpUnaryCount, shards []uint64, opt *execOptions) (uint64, error) { +func (e *executor) executeGenericCount(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpUnaryCount, shards []uint64, opt *execOptions) (uint64, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericCount") defer span.Finish() @@ -2912,7 +2946,7 @@ func (e *executor) executeGenericCount(ctx context.Context, index string, c *pql // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return 0, err } @@ -2935,7 +2969,7 @@ func (e *executor) executeGenericCount(ctx context.Context, index string, c *pql } // executeCount executes a count() call. -func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) { +func (e *executor) executeCount(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount") defer span.Finish() @@ -2947,7 +2981,7 @@ func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return 0, err } @@ -2970,7 +3004,7 @@ func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, } // executeClearBit executes a Clear() call. -func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { +func (e *executor) executeClearBit(ctx context.Context, tx Tx, index string, c *pql.Call, opt *execOptions) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearBit") defer span.Finish() @@ -2999,7 +3033,7 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal // Int field. if f.Type() == FieldTypeInt || f.Type() == FieldTypeDecimal { - return e.executeClearValueField(ctx, index, c, f, colID, opt) + return e.executeClearValueField(ctx, tx, index, c, f, colID, opt) } rowID, ok, err := c.UintArg(fieldName) @@ -3009,11 +3043,11 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return false, fmt.Errorf("row= argument required to Clear() call") } - return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt) + return e.executeClearBitField(ctx, tx, index, c, f, colID, rowID, opt) } // executeClearBitField executes a Clear() call for a field. -func (e *executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *execOptions) (bool, error) { +func (e *executor) executeClearBitField(ctx context.Context, tx Tx, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *execOptions) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearBitField") defer span.Finish() @@ -3022,7 +3056,7 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq for _, node := range e.Cluster.shardNodes(index, shard) { // Update locally if host matches. if node.ID == e.Node.ID { - val, err := f.ClearBit(rowID, colID) + val, err := f.ClearBit(tx, rowID, colID) if err != nil { return false, err } else if val { @@ -3046,7 +3080,7 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq } // executeClearRow executes a ClearRow() call. -func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { +func (e *executor) executeClearRow(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearRow") defer span.Finish() @@ -3069,7 +3103,7 @@ func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Cal // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeClearRowShard(ctx, index, c, shard) + return e.executeClearRowShard(ctx, tx, index, c, shard) } // Merge returned results at coordinating node. @@ -3089,7 +3123,7 @@ func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Cal } // executeClearRowShard executes a ClearRow() call for a single shard. -func (e *executor) executeClearRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (bool, error) { +func (e *executor) executeClearRowShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (bool, error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeClearRowShard") defer span.Finish() @@ -3118,7 +3152,7 @@ func (e *executor) executeClearRowShard(ctx context.Context, index string, c *pq if fragment == nil { continue } - cleared, err := fragment.clearRow(rowID) + cleared, err := fragment.clearRow(tx, rowID) if err != nil { return false, errors.Wrapf(err, "clearing row %d on view %s shard %d", rowID, view.name, shard) } @@ -3129,7 +3163,7 @@ func (e *executor) executeClearRowShard(ctx context.Context, index string, c *pq } // executeSetRow executes a Store() call. -func (e *executor) executeSetRow(ctx context.Context, indexName string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { +func (e *executor) executeSetRow(ctx context.Context, tx Tx, indexName string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { // Ensure the field type supports Store(). fieldName, err := c.FieldArg() if err != nil { @@ -3157,7 +3191,7 @@ func (e *executor) executeSetRow(ctx context.Context, indexName string, c *pql.C // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeSetRowShard(ctx, indexName, c, shard) + return e.executeSetRowShard(ctx, tx, indexName, c, shard) } // Merge returned results at coordinating node. @@ -3190,7 +3224,7 @@ func (e *executor) executeSetRow(ctx context.Context, indexName string, c *pql.C } // executeSetRowShard executes a SetRow() call for a single shard. -func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (bool, error) { +func (e *executor) executeSetRowShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (bool, error) { fieldName, err := c.FieldArg() if err != nil { return false, errors.New("Store() argument required: field") @@ -3212,7 +3246,7 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql. // Retrieve source row. var src *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { return false, errors.Wrap(err, "getting source row") } @@ -3235,7 +3269,7 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql. return false, errors.Wrapf(err, "creating fragment: %d", shard) } } - set, err := fragment.setRow(src, rowID) + set, err := fragment.setRow(tx, src, rowID) if err != nil { return false, errors.Wrapf(err, "storing row %d on view %s shard %d", rowID, viewStandard, shard) } @@ -3245,7 +3279,7 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql. } // executeSet executes a Set() call. -func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { +func (e *executor) executeSet(ctx context.Context, tx Tx, index string, c *pql.Call, opt *execOptions) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSet") defer span.Finish() @@ -3275,7 +3309,7 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op // Set column on existence field. if ef := idx.existenceField(); ef != nil { - if _, err := ef.SetBit(0, colID, nil); err != nil { + if _, err := ef.SetBit(tx, 0, colID, nil); err != nil { return false, errors.Wrap(err, "setting existence column") } } @@ -3302,7 +3336,7 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op if err != nil { return false, fmt.Errorf("reading Set() row (int/decimal): %v", err) } - return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt) + return e.executeSetValueField(ctx, tx, index, c, f, colID, rowVal, opt) default: // Read row ID. @@ -3323,12 +3357,12 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op timestamp = &t } - return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) + return e.executeSetBitField(ctx, tx, index, c, f, colID, rowID, timestamp, opt) } } // executeSetBitField executes a Set() call for a specific field. -func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *execOptions) (bool, error) { +func (e *executor) executeSetBitField(ctx context.Context, tx Tx, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *execOptions) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetBitField") defer span.Finish() @@ -3338,7 +3372,7 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql. for _, node := range e.Cluster.shardNodes(index, shard) { // Update locally if host matches. if node.ID == e.Node.ID { - val, err := f.SetBit(rowID, colID, timestamp) + val, err := f.SetBit(tx, rowID, colID, timestamp) if err != nil { return false, err } else if val { @@ -3363,7 +3397,7 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql. } // executeSetValueField executes a Set() call for a specific int field. -func (e *executor) executeSetValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, value int64, opt *execOptions) (bool, error) { +func (e *executor) executeSetValueField(ctx context.Context, tx Tx, index string, c *pql.Call, f *Field, colID uint64, value int64, opt *execOptions) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetValueField") defer span.Finish() @@ -3373,7 +3407,7 @@ func (e *executor) executeSetValueField(ctx context.Context, index string, c *pq for _, node := range e.Cluster.shardNodes(index, shard) { // Update locally if host matches. if node.ID == e.Node.ID { - val, err := f.SetValue(colID, value) + val, err := f.SetValue(tx, colID, value) if err != nil { return false, err } else if val { @@ -3398,7 +3432,7 @@ func (e *executor) executeSetValueField(ctx context.Context, index string, c *pq } // executeClearValueField removes value for colID if present -func (e *executor) executeClearValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, opt *execOptions) (bool, error) { +func (e *executor) executeClearValueField(ctx context.Context, tx Tx, index string, c *pql.Call, f *Field, colID uint64, opt *execOptions) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearValueField") defer span.Finish() @@ -3408,7 +3442,7 @@ func (e *executor) executeClearValueField(ctx context.Context, index string, c * for _, node := range e.Cluster.shardNodes(index, shard) { // Update locally if host matches. if node.ID == e.Node.ID { - val, err := f.ClearValue(colID) + val, err := f.ClearValue(tx, colID) if err != nil { return false, err } else if val { @@ -3433,7 +3467,7 @@ func (e *executor) executeClearValueField(ctx context.Context, index string, c * } // executeSetRowAttrs executes a SetRowAttrs() call. -func (e *executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *execOptions) error { +func (e *executor) executeSetRowAttrs(ctx context.Context, tx Tx, index string, c *pql.Call, opt *execOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetRowAttrs") defer span.Finish() @@ -3492,7 +3526,7 @@ func (e *executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. } // executeBulkSetRowAttrs executes a set of SetRowAttrs() calls. -func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, calls []*pql.Call, opt *execOptions) ([]interface{}, error) { +func (e *executor) executeBulkSetRowAttrs(ctx context.Context, tx Tx, index string, calls []*pql.Call, opt *execOptions) ([]interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBulkSetRowAttrs") defer span.Finish() @@ -3592,7 +3626,7 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal } // executeSetColumnAttrs executes a SetColumnAttrs() call. -func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *pql.Call, opt *execOptions) error { +func (e *executor) executeSetColumnAttrs(ctx context.Context, tx Tx, index string, c *pql.Call, opt *execOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetColumnAttrs") defer span.Finish() @@ -4802,6 +4836,7 @@ func isValidID(v interface{}) bool { // calls). type groupByIterator struct { executor *executor + tx Tx index string shard uint64 @@ -4833,9 +4868,10 @@ type groupByIterator struct { } // newGroupByIterator initializes a new groupByIterator. -func newGroupByIterator(executor *executor, rowIDs []RowIDs, children []*pql.Call, aggregate *pql.Call, filter *Row, index string, shard uint64, holder *Holder) (*groupByIterator, error) { +func newGroupByIterator(executor *executor, tx Tx, rowIDs []RowIDs, children []*pql.Call, aggregate *pql.Call, filter *Row, index string, shard uint64, holder *Holder) (_ *groupByIterator, err error) { gbi := &groupByIterator{ executor: executor, + tx: tx, index: index, shard: shard, rowIters: make([]rowIterator, len(children)), @@ -4886,7 +4922,10 @@ func newGroupByIterator(executor *executor, rowIDs []RowIDs, children []*pql.Cal if len(rowIDs[i]) > 0 { filters = append(filters, filterWithRows(rowIDs[i])) } - gbi.rowIters[i] = frag.rowIterator(i != 0, filters...) + gbi.rowIters[i], err = frag.rowIterator(tx, i != 0, filters...) + if err != nil { + return nil, err + } prev, hasPrev, err := call.UintArg("previous") if err != nil { @@ -4897,8 +4936,10 @@ func newGroupByIterator(executor *executor, rowIDs []RowIDs, children []*pql.Cal } gbi.rowIters[i].Seek(prev) } - nextRow, rowID, value, wrapped := gbi.rowIters[i].Next() - if nextRow == nil { + nextRow, rowID, value, wrapped, err := gbi.rowIters[i].Next() + if err != nil { + return nil, err + } else if nextRow == nil { gbi.done = true return gbi, nil } @@ -4916,8 +4957,10 @@ func newGroupByIterator(executor *executor, rowIDs []RowIDs, children []*pql.Cal // previous field, and if that one wraps we need to keep going // backward. for j := i - 1; j >= 0; j-- { - nextRow, rowID, value, wrapped := gbi.rowIters[j].Next() - if nextRow == nil { + nextRow, rowID, value, wrapped, err := gbi.rowIters[j].Next() + if err != nil { + return nil, err + } else if nextRow == nil { gbi.done = true return gbi, nil } @@ -4951,8 +4994,10 @@ func (gbi *groupByIterator) nextAtIdx(ctx context.Context, i int) (err error) { if err = ctx.Err(); err != nil { return err } - nr, rowID, value, wrapped := gbi.rowIters[i].Next() - if nr == nil { + nr, rowID, value, wrapped, err := gbi.rowIters[i].Next() + if err != nil { + return err + } else if nr == nil { gbi.done = true return nil } @@ -5005,7 +5050,7 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool switch gbi.aggregate.Name { case "Sum": - result, err := gbi.executor.executeSumCountShard(ctx, gbi.index, gbi.aggregate, filter, gbi.shard) + result, err := gbi.executor.executeSumCountShard(ctx, gbi.tx, gbi.index, gbi.aggregate, filter, gbi.shard) if err != nil { return ret, false, err } diff --git a/executor_internal_test.go b/executor_internal_test.go index 909f48f7a..28dd369ef 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -137,13 +137,18 @@ func TestExecutor_TranslateRowsOnBool(t *testing.T) { holder := NewHolder(DefaultPartitionN) defer holder.Close() + tx, err := holder.Begin(true) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + e := &executor{ Holder: holder, Cluster: NewTestCluster(1), } e.Holder.Path, _ = ioutil.TempDir(*TempDir, "") - err := e.Holder.Open() - if err != nil { + if err := e.Holder.Open(); err != nil { t.Fatalf("opening holder: %v", err) } @@ -158,13 +163,17 @@ func TestExecutor_TranslateRowsOnBool(t *testing.T) { t.Fatalf("creating fields %v, %v", errb, errbk) } - _, err1 := fb.SetBit(1, 1, nil) - _, err2 := fb.SetBit(2, 2, nil) - _, err3 := fb.SetBit(3, 3, nil) + _, err1 := fb.SetBit(tx, 1, 1, nil) + _, err2 := fb.SetBit(tx, 2, 2, nil) + _, err3 := fb.SetBit(tx, 3, 3, nil) if err1 != nil || err2 != nil || err3 != nil { t.Fatalf("setting bit %v, %v, %v", err1, err2, err3) } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + tests := []struct { pql string }{ diff --git a/executor_test.go b/executor_test.go index 0bbf3bde8..d1ae63cb4 100644 --- a/executor_test.go +++ b/executor_test.go @@ -903,8 +903,15 @@ func TestExecutor_Execute_SetValue(t *testing.T) { t.Fatal(err) } + // Obtain transaction. + tx, err := hldr.Begin(false) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + f := hldr.Field("i", "f") - if value, exists, err := f.Value(10); err != nil { + if value, exists, err := f.Value(tx, 10); err != nil { t.Fatal(err) } else if !exists { t.Fatal("expected value to exist") @@ -912,7 +919,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { t.Fatalf("unexpected value: %v", value) } - if value, exists, err := f.Value(100); err != nil { + if value, exists, err := f.Value(tx, 100); err != nil { t.Fatal(err) } else if !exists { t.Fatal("expected value to exist") diff --git a/field.go b/field.go index f82f03543..066e69693 100644 --- a/field.go +++ b/field.go @@ -1088,7 +1088,7 @@ func (f *Field) setTimeQuantum(q TimeQuantum) error { // RowTime gets the row at the particular time with the granularity specified by // the quantum. -func (f *Field) RowTime(rowID uint64, time time.Time, quantum string) (*Row, error) { +func (f *Field) RowTime(tx Tx, rowID uint64, time time.Time, quantum string) (*Row, error) { if !TimeQuantum(quantum).Valid() { return nil, ErrInvalidTimeQuantum } @@ -1097,7 +1097,7 @@ func (f *Field) RowTime(rowID uint64, time time.Time, quantum string) (*Row, err if view == nil { return nil, errors.Errorf("view with quantum %v not found.", quantum) } - return view.row(rowID), nil + return view.row(tx, rowID) } // viewPath returns the path to a view in the field. @@ -1212,21 +1212,21 @@ func (f *Field) deleteView(name string) error { // package, and the fact that it's only allowed on // `set`,`mutex`, and `bool` fields is odd. This may // be considered for deprecation in a future version. -func (f *Field) Row(rowID uint64) (*Row, error) { +func (f *Field) Row(tx Tx, rowID uint64) (*Row, error) { switch f.Type() { case FieldTypeSet, FieldTypeMutex, FieldTypeBool: view := f.view(viewStandard) if view == nil { return nil, ErrInvalidView } - return view.row(rowID), nil + return view.row(tx, rowID) default: return nil, errors.Errorf("row method unsupported for field type: %s", f.Type()) } } // SetBit sets a bit on a view within the field. -func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err error) { +func (f *Field) SetBit(tx Tx, rowID, colID uint64, t *time.Time) (changed bool, err error) { viewName := viewStandard if !f.options.NoStandardView { // Retrieve view. Exit if it doesn't exist. @@ -1236,7 +1236,7 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err } // Set non-time bit. - if v, err := view.setBit(rowID, colID); err != nil { + if v, err := view.setBit(tx, rowID, colID); err != nil { return changed, errors.Wrap(err, "setting on view") } else if v { changed = v @@ -1255,7 +1255,7 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err return changed, errors.Wrapf(err, "creating view %s", subname) } - if c, err := view.setBit(rowID, colID); err != nil { + if c, err := view.setBit(tx, rowID, colID); err != nil { return changed, errors.Wrapf(err, "setting on view %s", subname) } else if c { changed = true @@ -1266,7 +1266,7 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err } // ClearBit clears a bit within the field. -func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) { +func (f *Field) ClearBit(tx Tx, rowID, colID uint64) (changed bool, err error) { viewName := viewStandard // Retrieve view. Exit if it doesn't exist. @@ -1276,7 +1276,7 @@ func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) { } // Clear non-time bit. - if v, err := view.clearBit(rowID, colID); err != nil { + if v, err := view.clearBit(tx, rowID, colID); err != nil { return false, errors.Wrap(err, "clearing on view") } else if v { changed = changed || v @@ -1294,7 +1294,7 @@ func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) { level-- } if level < skipAbove { - cleared, err := view.clearBit(rowID, colID) + cleared, err := view.clearBit(tx, rowID, colID) changed = changed || cleared if err != nil { return changed, errors.Wrapf(err, "clearing on view %s", view.name) @@ -1354,13 +1354,13 @@ func (f *Field) allTimeViewsSortedByQuantum() (me []*view) { // StringValue reads an integer field value for a column, and converts // it to a string based on a foreign index string key. -func (f *Field) StringValue(columnID uint64) (value string, exists bool, err error) { +func (f *Field) StringValue(tx Tx, columnID uint64) (value string, exists bool, err error) { bsig := f.bsiGroup(f.name) if bsig == nil { return value, false, ErrBSIGroupNotFound } - val, exists, err := f.Value(columnID) + val, exists, err := f.Value(tx, columnID) if exists { value, err = f.translateStore.TranslateID(uint64(val)) } @@ -1368,7 +1368,7 @@ func (f *Field) StringValue(columnID uint64) (value string, exists bool, err err } // Value reads a field value for a column. -func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { +func (f *Field) Value(tx Tx, columnID uint64) (value int64, exists bool, err error) { bsig := f.bsiGroup(f.name) if bsig == nil { return 0, false, ErrBSIGroupNotFound @@ -1380,7 +1380,7 @@ func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { return 0, false, nil } - v, exists, err := view.value(columnID, bsig.BitDepth) + v, exists, err := view.value(tx, columnID, bsig.BitDepth) if err != nil { return 0, false, err } else if !exists { @@ -1390,7 +1390,7 @@ func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { } // SetValue sets a field value for a column. -func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) { +func (f *Field) SetValue(tx Tx, columnID uint64, value int64) (changed bool, err error) { // Fetch bsiGroup & validate min/max. bsig := f.bsiGroup(f.name) if bsig == nil { @@ -1430,11 +1430,11 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) if err != nil { return false, errors.Wrap(err, "creating view") } - return view.setValue(columnID, bsig.BitDepth, baseValue) + return view.setValue(tx, columnID, bsig.BitDepth, baseValue) } // ClearValue removes a field value for a column. -func (f *Field) ClearValue(columnID uint64) (changed bool, err error) { +func (f *Field) ClearValue(tx Tx, columnID uint64) (changed bool, err error) { bsig := f.bsiGroup(f.name) if bsig == nil { return false, ErrBSIGroupNotFound @@ -1444,17 +1444,17 @@ func (f *Field) ClearValue(columnID uint64) (changed bool, err error) { if view == nil { return false, nil } - value, exists, err := view.value(columnID, bsig.BitDepth) + value, exists, err := view.value(tx, columnID, bsig.BitDepth) if err != nil { return false, err } if exists { - return view.clearValue(columnID, bsig.BitDepth, value) + return view.clearValue(tx, columnID, bsig.BitDepth, value) } return false, nil } -func (f *Field) MaxForShard(shard uint64, filter *Row) (ValCount, error) { +func (f *Field) MaxForShard(tx Tx, shard uint64, filter *Row) (ValCount, error) { bsig := f.bsiGroup(f.name) if bsig == nil { return ValCount{}, ErrBSIGroupNotFound @@ -1470,7 +1470,7 @@ func (f *Field) MaxForShard(shard uint64, filter *Row) (ValCount, error) { return ValCount{}, nil } - max, cnt, err := fragment.max(filter, bsig.BitDepth) + max, cnt, err := fragment.max(tx, filter, bsig.BitDepth) if err != nil { return ValCount{}, errors.Wrap(err, "calling fragment.max") } @@ -1490,7 +1490,7 @@ func (f *Field) MaxForShard(shard uint64, filter *Row) (ValCount, error) { // MinForShard returns the minimum value which appears in this shard // (this field must be an Int or Decimal field). It also returns the // number of times the minimum value appears. -func (f *Field) MinForShard(shard uint64, filter *Row) (ValCount, error) { +func (f *Field) MinForShard(tx Tx, shard uint64, filter *Row) (ValCount, error) { bsig := f.bsiGroup(f.name) if bsig == nil { return ValCount{}, ErrBSIGroupNotFound @@ -1506,7 +1506,7 @@ func (f *Field) MinForShard(shard uint64, filter *Row) (ValCount, error) { return ValCount{}, nil } - min, cnt, err := fragment.min(filter, bsig.BitDepth) + min, cnt, err := fragment.min(tx, filter, bsig.BitDepth) if err != nil { return ValCount{}, errors.Wrap(err, "calling fragment.min") } @@ -1524,7 +1524,7 @@ func (f *Field) MinForShard(shard uint64, filter *Row) (ValCount, error) { } // Range performs a conditional operation on Field. -func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error) { +func (f *Field) Range(tx Tx, name string, op pql.Token, predicate int64) (*Row, error) { // Retrieve and validate bsiGroup. bsig := f.bsiGroup(name) if bsig == nil { @@ -1544,11 +1544,11 @@ func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error) return NewRow(), nil } - return view.rangeOp(op, bsig.BitDepth, baseValue) + return view.rangeOp(tx, op, bsig.BitDepth, baseValue) } // Import bulk imports data. -func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time, opts ...ImportOption) error { +func (f *Field) Import(tx Tx, rowIDs, columnIDs []uint64, timestamps []*time.Time, opts ...ImportOption) error { // Set up import options. options := &ImportOptions{} @@ -1620,7 +1620,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time, opts return errors.Wrap(err, "creating fragment") } - if err := frag.bulkImport(data.RowIDs, data.ColumnIDs, options); err != nil { + if err := frag.bulkImport(tx, data.RowIDs, data.ColumnIDs, options); err != nil { return err } } @@ -1628,7 +1628,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time, opts return nil } -func (f *Field) importFloatValue(columnIDs []uint64, values []float64, options *ImportOptions) error { +func (f *Field) importFloatValue(tx Tx, columnIDs []uint64, values []float64, options *ImportOptions) error { // convert values to int64 values based on scale ivalues := make([]int64, len(values)) bsig := f.bsiGroup(f.name) @@ -1640,11 +1640,11 @@ func (f *Field) importFloatValue(columnIDs []uint64, values []float64, options * ivalues[i] = int64(fval * mult) } // then call importValue - return f.importValue(columnIDs, ivalues, options) + return f.importValue(tx, columnIDs, ivalues, options) } // importValue bulk imports range-encoded value data. -func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportOptions) error { +func (f *Field) importValue(tx Tx, columnIDs []uint64, values []int64, options *ImportOptions) error { viewName := viewBSIGroupPrefix + f.name // Get the bsiGroup so we know bitDepth. bsig := f.bsiGroup(f.name) @@ -1727,7 +1727,7 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO baseValues[i] = value - bsig.Base } - if err := frag.importValue(data.ColumnIDs, baseValues, requiredDepth, options.Clear); err != nil { + if err := frag.importValue(tx, data.ColumnIDs, baseValues, requiredDepth, options.Clear); err != nil { return err } } @@ -1759,7 +1759,7 @@ func (f *Field) importRoaring(ctx context.Context, data []byte, shard uint64, vi return nil } -func (f *Field) importRoaringOverwrite(ctx context.Context, data []byte, shard uint64, viewName string, block int) error { +func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte, shard uint64, viewName string, block int) error { span, ctx := tracing.StartSpanFromContext(ctx, "Field.importRoaringOverwrite") defer span.Finish() @@ -1776,7 +1776,7 @@ func (f *Field) importRoaringOverwrite(ctx context.Context, data []byte, shard u if err != nil { return errors.Wrap(err, "creating fragment") } - if err := frag.importRoaringOverwrite(ctx, data, block); err != nil { + if err := frag.importRoaringOverwrite(ctx, tx, data, block); err != nil { return err } @@ -1785,9 +1785,14 @@ func (f *Field) importRoaringOverwrite(ctx context.Context, data []byte, shard u switch f.Options().Type { case FieldTypeInt, FieldTypeDecimal: frag.mu.Lock() - frag.calculateMaxRowID() - maxRowID, _ := frag.maxRow(nil) + if err := frag.calculateMaxRowID(); err != nil { + return err + } + maxRowID, _, err := frag.maxRow(tx, nil) frag.mu.Unlock() + if err != nil { + return err + } var bitDepth uint if maxRowID+1 > bsiOffsetBit { diff --git a/field_internal_test.go b/field_internal_test.go index 296e54be0..8a5aaffe4 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -246,15 +246,15 @@ func (f *TestField) Reopen() error { return nil } -func (f *TestField) MustSetBit(row, col uint64, ts ...time.Time) { +func (f *TestField) MustSetBit(tx Tx, row, col uint64, ts ...time.Time) { if len(ts) == 0 { - _, err := f.Field.SetBit(row, col, nil) + _, err := f.Field.SetBit(tx, row, col, nil) if err != nil { panic(err) } } for _, t := range ts { - _, err := f.Field.SetBit(row, col, &t) + _, err := f.Field.SetBit(tx, row, col, &t) if err != nil { panic(err) } @@ -310,41 +310,44 @@ func TestField_RowTime(t *testing.T) { f := OpenField(t, OptFieldTypeTime(TimeQuantum(""))) defer f.Close() + // Obtain transaction. + tx := &RoaringTx{Field: f.Field} + if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil { t.Fatal(err) } - f.MustSetBit(1, 1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC)) - f.MustSetBit(1, 2, time.Date(2011, time.January, 5, 12, 0, 0, 0, time.UTC)) - f.MustSetBit(1, 3, time.Date(2010, time.February, 5, 12, 0, 0, 0, time.UTC)) - f.MustSetBit(1, 4, time.Date(2010, time.January, 6, 12, 0, 0, 0, time.UTC)) - f.MustSetBit(1, 5, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC)) + f.MustSetBit(tx, 1, 1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC)) + f.MustSetBit(tx, 1, 2, time.Date(2011, time.January, 5, 12, 0, 0, 0, time.UTC)) + f.MustSetBit(tx, 1, 3, time.Date(2010, time.February, 5, 12, 0, 0, 0, time.UTC)) + f.MustSetBit(tx, 1, 4, time.Date(2010, time.January, 6, 12, 0, 0, 0, time.UTC)) + f.MustSetBit(tx, 1, 5, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC)) - if r, err := f.RowTime(1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil { + if r, err := f.RowTime(tx, 1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(r.Columns(), []uint64{1, 3, 4, 5}) { t.Fatalf("wrong columns: %#v", r.Columns()) } - if r, err := f.RowTime(1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "YM"); err != nil { + if r, err := f.RowTime(tx, 1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "YM"); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(r.Columns(), []uint64{3}) { t.Fatalf("wrong columns: %#v", r.Columns()) } - if r, err := f.RowTime(1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "M"); err != nil { + if r, err := f.RowTime(tx, 1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "M"); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(r.Columns(), []uint64{3}) { t.Fatalf("wrong columns: %#v", r.Columns()) } - if r, err := f.RowTime(1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC), "MD"); err != nil { + if r, err := f.RowTime(tx, 1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC), "MD"); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(r.Columns(), []uint64{1, 5}) { t.Fatalf("wrong columns: %#v", r.Columns()) } - if r, err := f.RowTime(1, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC), "MDH"); err != nil { + if r, err := f.RowTime(tx, 1, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC), "MDH"); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(r.Columns(), []uint64{5}) { t.Fatalf("wrong columns: %#v", r.Columns()) @@ -578,11 +581,13 @@ func TestBSIGroup_importValue(t *testing.T) { []uint64{100}, }, } { - if err := f.importValue(tt.columnIDs, tt.values, options); err != nil { + tx := &RoaringTx{Field: f.Field} + + if err := f.importValue(tx, tt.columnIDs, tt.values, options); err != nil { t.Fatalf("test %d, importing values: %s", i, err.Error()) } - if row, err := f.Range(f.name, pql.EQ, tt.checkVal); err != nil { + if row, err := f.Range(tx, f.name, pql.EQ, tt.checkVal); err != nil { t.Fatalf("test %d, getting range: %s", i, err.Error()) } else if !reflect.DeepEqual(row.Columns(), tt.expCols) { t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns()) @@ -643,11 +648,13 @@ func TestIntField_MinMaxForShard(t *testing.T) { }, } { t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { - if err := f.importValue(test.columnIDs, test.values, options); err != nil { + tx := &RoaringTx{Field: f.Field} + + if err := f.importValue(tx, test.columnIDs, test.values, options); err != nil { t.Fatalf("test %d, importing values: %s", i, err.Error()) } - maxvc, err := f.MaxForShard(0, nil) + maxvc, err := f.MaxForShard(tx, 0, nil) if err != nil { t.Fatalf("getting max for shard: %v", err) } @@ -655,7 +662,7 @@ func TestIntField_MinMaxForShard(t *testing.T) { t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc) } - minvc, err := f.MinForShard(0, nil) + minvc, err := f.MinForShard(tx, 0, nil) if err != nil { t.Fatalf("getting min for shard: %v", err) } @@ -797,11 +804,13 @@ func TestDecimalField_MinMaxForShard(t *testing.T) { }, } { t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { - if err := f.importFloatValue(test.columnIDs, test.values, options); err != nil { + tx := &RoaringTx{Field: f.Field} + + if err := f.importFloatValue(tx, test.columnIDs, test.values, options); err != nil { t.Fatalf("test %d, importing values: %s", i, err.Error()) } - maxvc, err := f.MaxForShard(0, nil) + maxvc, err := f.MaxForShard(tx, 0, nil) if err != nil { t.Fatalf("getting max for shard: %v", err) } @@ -809,7 +818,7 @@ func TestDecimalField_MinMaxForShard(t *testing.T) { t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc) } - minvc, err := f.MinForShard(0, nil) + minvc, err := f.MinForShard(tx, 0, nil) if err != nil { t.Fatalf("getting min for shard: %v", err) } diff --git a/field_test.go b/field_test.go index ab24b466f..aeade9fcd 100644 --- a/field_test.go +++ b/field_test.go @@ -35,16 +35,17 @@ func TestField_SetValue(t *testing.T) { if err != nil { t.Fatal(err) } + tx := &pilosa.RoaringTx{Field: f.Field} // Set value on field. - if changed, err := f.SetValue(100, 21); err != nil { + if changed, err := f.SetValue(tx, 100, 21); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.Value(100); err != nil { + if value, exists, err := f.Value(tx, 100); err != nil { t.Fatal(err) } else if value != 21 { t.Fatalf("unexpected value: %d", value) @@ -53,7 +54,7 @@ func TestField_SetValue(t *testing.T) { } // Setting value should return no change. - if changed, err := f.SetValue(100, 21); err != nil { + if changed, err := f.SetValue(tx, 100, 21); err != nil { t.Fatal(err) } else if changed { t.Fatal("expected no change") @@ -68,23 +69,24 @@ func TestField_SetValue(t *testing.T) { if err != nil { t.Fatal(err) } + tx := &pilosa.RoaringTx{Field: f.Field} // Set value. - if changed, err := f.SetValue(100, 21); err != nil { + if changed, err := f.SetValue(tx, 100, 21); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Set different value. - if changed, err := f.SetValue(100, 23); err != nil { + if changed, err := f.SetValue(tx, 100, 23); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.Value(100); err != nil { + if value, exists, err := f.Value(tx, 100); err != nil { t.Fatal(err) } else if value != 23 { t.Fatalf("unexpected value: %d", value) @@ -101,9 +103,10 @@ func TestField_SetValue(t *testing.T) { if err != nil { t.Fatal(err) } + tx := &pilosa.RoaringTx{Field: f.Field} // Set value. - if _, err := f.SetValue(100, 21); err != pilosa.ErrBSIGroupNotFound { + if _, err := f.SetValue(tx, 100, 21); err != pilosa.ErrBSIGroupNotFound { t.Fatalf("unexpected error: %s", err) } }) @@ -116,9 +119,10 @@ func TestField_SetValue(t *testing.T) { if err != nil { t.Fatal(err) } + tx := &pilosa.RoaringTx{Field: f.Field} // Set value. - if _, err := f.SetValue(100, 15); err != pilosa.ErrBSIGroupValueTooLow { + if _, err := f.SetValue(tx, 100, 15); err != pilosa.ErrBSIGroupValueTooLow { t.Fatalf("unexpected error: %s", err) } }) @@ -131,9 +135,10 @@ func TestField_SetValue(t *testing.T) { if err != nil { t.Fatal(err) } + tx := &pilosa.RoaringTx{Field: f.Field} // Set value. - if _, err := f.SetValue(100, 31); err != pilosa.ErrBSIGroupValueTooHigh { + if _, err := f.SetValue(tx, 100, 31); err != pilosa.ErrBSIGroupValueTooHigh { t.Fatalf("unexpected error: %s", err) } }) @@ -199,11 +204,12 @@ func TestField_AvailableShards(t *testing.T) { if err != nil { t.Fatal(err) } + tx := &pilosa.RoaringTx{Field: f.Field} // Set values on shards 0 & 2, and verify. - if _, err := f.SetBit(0, 100, nil); err != nil { + if _, err := f.SetBit(tx, 0, 100, nil); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(0, ShardWidth*2, nil); err != nil { + } else if _, err := f.SetBit(tx, 0, ShardWidth*2, nil); err != nil { t.Fatal(err) } else if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" { t.Fatal(diff) @@ -238,16 +244,17 @@ func TestField_ClearValue(t *testing.T) { if err != nil { t.Fatal(err) } + tx := &pilosa.RoaringTx{Field: f.Field} // Set value on field. - if changed, err := f.SetValue(100, 21); err != nil { + if changed, err := f.SetValue(tx, 100, 21); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.Value(100); err != nil { + if value, exists, err := f.Value(tx, 100); err != nil { t.Fatal(err) } else if value != 21 { t.Fatalf("unexpected value: %d", value) @@ -255,14 +262,14 @@ func TestField_ClearValue(t *testing.T) { t.Fatal("expected value to exist") } - if changed, err := f.ClearValue(100); err != nil { + if changed, err := f.ClearValue(tx, 100); err != nil { t.Fatal(err) } else if !changed { t.Fatal(err) } // Read value. - if _, exists, err := f.Value(100); err != nil { + if _, exists, err := f.Value(tx, 100); err != nil { t.Fatal(err) } else if exists { t.Fatal("expected value to not exist") diff --git a/fragment.go b/fragment.go index 61ac78313..b31e8d8ff 100644 --- a/fragment.go +++ b/fragment.go @@ -164,12 +164,13 @@ type fragment struct { // newFragment returns a new instance of Fragment. func newFragment(holder *Holder, path, index, field, view string, shard uint64, flags byte) *fragment { f := &fragment{ - path: path, - index: index, - field: field, - view: view, - shard: shard, - flags: flags, + path: path, + index: index, + field: field, + view: view, + shard: shard, + flags: flags, + CacheType: DefaultCacheType, CacheSize: DefaultCacheSize, @@ -197,7 +198,7 @@ func (f *fragment) inspect(params InspectRequestParams) (fi FragmentInfo) { fi.BitmapInfo = *f.bitmapInfo } if params.Checksum { - fi.BlockChecksums = f.Blocks() + fi.BlockChecksums, _ = f.Blocks() } return fi } @@ -228,8 +229,7 @@ func (f *fragment) Open() error { f.checksums = make(map[int][]byte) // Read last bit to determine max row. - f.maxRowID = f.storage.Max() / ShardWidth - return nil + return f.calculateMaxRowID() }(); err != nil { f.close() return err @@ -510,28 +510,40 @@ func (f *fragment) closeStorage() error { } // row returns a row by ID. -func (f *fragment) row(rowID uint64) *Row { +func (f *fragment) row(tx Tx, rowID uint64) (*Row, error) { f.mu.Lock() defer f.mu.Unlock() - return f.unprotectedRow(rowID) + return f.unprotectedRow(tx, rowID) +} + +// mustRow returns a row by ID. Panic on error. Only used for testing. +func (f *fragment) mustRow(tx Tx, rowID uint64) *Row { + row, err := f.row(tx, rowID) + if err != nil { + panic(err) + } + return row } // unprotectedRow returns a row from the row cache if available or from storage // (updating the cache). -func (f *fragment) unprotectedRow(rowID uint64) *Row { +func (f *fragment) unprotectedRow(tx Tx, rowID uint64) (*Row, error) { r, ok := f.rowCache.Fetch(rowID) if ok && r != nil { - return r + return r, nil } - row := f.rowFromStorage(rowID) + row, err := f.rowFromStorage(tx, rowID) + if err != nil { + return nil, err + } f.rowCache.Add(rowID, row) - return row + return row, nil } // rowFromStorage clones a row data out of fragment storage and returns it as a // Row object. -func (f *fragment) rowFromStorage(rowID uint64) *Row { +func (f *fragment) rowFromStorage(tx Tx, rowID uint64) (*Row, error) { // Only use a subset of the containers. // NOTE: The start & end ranges must be divisible by container width. // @@ -539,7 +551,10 @@ func (f *fragment) rowFromStorage(rowID uint64) *Row { // containers which will use copy-on-write semantics. The actual bitmap // and Containers object are new and not shared, but the containers are // shared. - data := f.storage.OffsetRange(f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) + data, err := tx.OffsetRange(f.index, f.field, f.view, f.shard, f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) + if err != nil { + return nil, err + } row := &Row{ segments: []rowSegment{{ @@ -550,22 +565,22 @@ func (f *fragment) rowFromStorage(rowID uint64) *Row { } row.invalidateCount() - return row + return row, nil } // setBit sets a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *fragment) setBit(rowID, columnID uint64) (changed bool, err error) { +func (f *fragment) setBit(tx Tx, rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() err = f.gen.Transaction(&f.storage.OpWriter, func() error { // handle mutux field type if f.mutexVector != nil { - if err := f.handleMutex(rowID, columnID); err != nil { + if err := f.handleMutex(tx, rowID, columnID); err != nil { return errors.Wrap(err, "handling mutex") } } - changed, err = f.unprotectedSetBit(rowID, columnID) + changed, err = f.unprotectedSetBit(tx, rowID, columnID) return err }) return changed, err @@ -573,11 +588,11 @@ func (f *fragment) setBit(rowID, columnID uint64) (changed bool, err error) { // handleMutex will clear an existing row and store the new row // in the vector. -func (f *fragment) handleMutex(rowID, columnID uint64) error { - if existingRowID, found, err := f.mutexVector.Get(columnID); err != nil { +func (f *fragment) handleMutex(tx Tx, rowID, columnID uint64) error { + if existingRowID, found, err := f.mutexVector.Get(tx, columnID); err != nil { return errors.Wrap(err, "getting mutex vector data") } else if found && existingRowID != rowID { - if _, err := f.unprotectedClearBit(existingRowID, columnID); err != nil { + if _, err := f.unprotectedClearBit(tx, existingRowID, columnID); err != nil { return errors.Wrap(err, "clearing mutex value") } } @@ -585,7 +600,7 @@ func (f *fragment) handleMutex(rowID, columnID uint64) error { } // unprotectedSetBit TODO should be replaced by an invocation of importPositions with a single bit to set. -func (f *fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err error) { +func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) @@ -594,7 +609,7 @@ func (f *fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err } // Write to storage. - if changed, err = f.storage.Add(pos); err != nil { + if changed, err = tx.Add(f.index, f.field, f.view, f.shard, pos); err != nil { return false, errors.Wrap(err, "writing") } @@ -612,7 +627,10 @@ func (f *fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err // If we're using a cache, update it. Otherwise skip the // possibly-expensive count operation. if f.CacheType != CacheTypeNone { - n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) + n, err := tx.CountRange(f.index, f.field, f.view, f.shard, rowID*ShardWidth, (rowID+1)*ShardWidth) + if err != nil { + return false, err + } f.cache.Add(rowID, n) } // Drop the rowCache entry; it's wrong, and we don't want to force @@ -631,11 +649,11 @@ func (f *fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err // clearBit clears a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { +func (f *fragment) clearBit(tx Tx, rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() err = f.gen.Transaction(&f.storage.OpWriter, func() error { - changed, err = f.unprotectedClearBit(rowID, columnID) + changed, err = f.unprotectedClearBit(tx, rowID, columnID) return err }) return changed, err @@ -643,7 +661,7 @@ func (f *fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { // unprotectedClearBit TODO should be replaced by an invocation of // importPositions with a single bit to clear. -func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, err error) { +func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) @@ -652,7 +670,7 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er } // Write to storage. - if changed, err = f.storage.Remove(pos); err != nil { + if changed, err = tx.Remove(f.index, f.field, f.view, f.shard, pos); err != nil { return false, errors.Wrap(err, "writing") } @@ -670,7 +688,10 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er // If we're using a cache, update it. Otherwise skip the // possibly-expensive count operation. if f.CacheType != CacheTypeNone { - n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) + n, err := tx.CountRange(f.index, f.field, f.view, f.shard, rowID*ShardWidth, (rowID+1)*ShardWidth) + if err != nil { + return changed, err + } f.cache.Add(rowID, n) } // Drop the rowCache entry; it's wrong, and we don't want to force @@ -684,17 +705,17 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er // setRow replaces an existing row (specified by rowID) with the given // Row. This updates both the on-disk storage and the in-cache bitmap. -func (f *fragment) setRow(row *Row, rowID uint64) (changed bool, err error) { +func (f *fragment) setRow(tx Tx, row *Row, rowID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() err = f.gen.Transaction(&f.storage.OpWriter, func() error { - changed, err = f.unprotectedSetRow(row, rowID) + changed, err = f.unprotectedSetRow(tx, row, rowID) return err }) return changed, err } -func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err error) { +func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed bool, err error) { // TODO: In order to return `changed`, we need to first compare // the existing row with the given row. Determine if the overhead // of this is worth having `changed`. @@ -706,7 +727,9 @@ func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err // Remove every existing container in the row. for i := uint64(0); i < (1 << shardVsContainerExponent); i++ { - f.storage.Containers.Remove(headContainerKey + i) + if err := tx.RemoveContainer(f.index, f.field, f.view, f.shard, headContainerKey+i); err != nil { + return changed, err + } } // From the given row, get the rowSegment for this shard. @@ -719,12 +742,17 @@ func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err citer, _ := seg.data.Containers.Iterator(f.shard << shardVsContainerExponent) for citer.Next() { k, c := citer.Value() - f.storage.Containers.Put(headContainerKey+(k%(1<= 0 || clear { - if c, err := f.unprotectedClearBit(uint64(bsiSignBit), columnID); err != nil { + if c, err := f.unprotectedClearBit(tx, uint64(bsiSignBit), columnID); err != nil { return errors.Wrap(err, "clearing sign") } else if c { changed = true } } else { - if c, err := f.unprotectedSetBit(uint64(bsiSignBit), columnID); err != nil { + if c, err := f.unprotectedSetBit(tx, uint64(bsiSignBit), columnID); err != nil { return errors.Wrap(err, "marking sign") } else if c { changed = true @@ -949,7 +982,7 @@ func (f *fragment) setValueBase(columnID uint64, bitDepth uint, value int64, cle } // importSetValue is a more efficient SetValue just for imports. -func (f *fragment) importSetValue(columnID uint64, bitDepth uint, value int64, clear bool) (changed int, err error) { // nolint: unparam +func (f *fragment) importSetValue(tx Tx, columnID uint64, bitDepth uint, value int64, clear bool) (changed int, err error) { // nolint: unparam // Convert value to an unsigned representation. uvalue := uint64(value) if value < 0 { @@ -963,13 +996,13 @@ func (f *fragment) importSetValue(columnID uint64, bitDepth uint, value int64, c } if uvalue&(1<= 0 || clear { - if c, err := f.storage.Remove(p); err != nil { + if c, err := tx.Remove(f.index, f.field, f.view, f.shard, p); err != nil { return changed, errors.Wrap(err, "removing sign from storage") } else if c { changed++ } } else { - if c, err := f.storage.Add(p); err != nil { + if c, err := tx.Add(f.index, f.field, f.view, f.shard, p); err != nil { return changed, errors.Wrap(err, "adding sign to storage") } else if c { changed++ @@ -1016,16 +1049,21 @@ func (f *fragment) importSetValue(columnID uint64, bitDepth uint, value int64, c // sum returns the sum of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *fragment) sum(filter *Row, bitDepth uint) (sum int64, count uint64, err error) { +func (f *fragment) sum(tx Tx, filter *Row, bitDepth uint) (sum int64, count uint64, err error) { // Compute count based on the existence row. - consider := f.row(bsiExistsBit) - if filter != nil { + consider, err := f.row(tx, bsiExistsBit) + if err != nil { + return sum, count, err + } else if filter != nil { consider = consider.Intersect(filter) } count = consider.Count() // Get negative set - nrow := f.row(bsiSignBit) + nrow, err := f.row(tx, bsiSignBit) + if err != nil { + return sum, count, err + } // Filter negative set nrow = consider.Intersect(nrow) @@ -1043,7 +1081,10 @@ func (f *fragment) sum(filter *Row, bitDepth uint) (sum int64, count uint64, err // Execute once for positive numbers and once for negative. Subtract the // negative sum from the positive sum. for i := uint(0); i < bitDepth; i++ { - row := f.row(uint64(bsiOffsetBit + i)) + row, err := f.row(tx, uint64(bsiOffsetBit+i)) + if err != nil { + return sum, count, err + } psum := int64((1 << i) * row.intersectionCount(prow)) nsum := int64((1 << i) * row.intersectionCount(nrow)) @@ -1057,9 +1098,11 @@ func (f *fragment) sum(filter *Row, bitDepth uint) (sum int64, count uint64, err // min returns the min of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *fragment) min(filter *Row, bitDepth uint) (min int64, count uint64, err error) { - consider := f.row(bsiExistsBit) - if filter != nil { +func (f *fragment) min(tx Tx, filter *Row, bitDepth uint) (min int64, count uint64, err error) { + consider, err := f.row(tx, bsiExistsBit) + if err != nil { + return min, count, err + } else if filter != nil { consider = consider.Intersect(filter) } @@ -1072,20 +1115,25 @@ func (f *fragment) min(filter *Row, bitDepth uint) (min int64, count uint64, err // from that set, then negate it, and return it. For example, if values // (-1, -2) exist, they are stored unsigned (1,2) with a negative sign bit // set. We take the highest of that set (2) and negate it and return it. - if row := f.row(bsiSignBit).Intersect(consider); row.Any() { - min, count := f.maxUnsigned(row, bitDepth) - return -min, count, nil + if row, err := f.row(tx, bsiSignBit); err != nil { + return min, count, err + } else if row = row.Intersect(consider); row.Any() { + min, count, err := f.maxUnsigned(tx, row, bitDepth) + return -min, count, err } // Otherwise find lowest positive number. - min, count = f.minUnsigned(consider, bitDepth) - return min, count, nil + return f.minUnsigned(tx, consider, bitDepth) } // minUnsigned the lowest value without considering the sign bit. Filter is required. -func (f *fragment) minUnsigned(filter *Row, bitDepth uint) (min int64, count uint64) { +func (f *fragment) minUnsigned(tx Tx, filter *Row, bitDepth uint) (min int64, count uint64, err error) { for i := int(bitDepth - 1); i >= 0; i-- { - row := filter.Difference(f.row(uint64(bsiOffsetBit + i))) + row, err := f.row(tx, uint64(bsiOffsetBit+i)) + if err != nil { + return min, count, err + } + row = filter.Difference(row) count = row.Count() if count > 0 { filter = row @@ -1096,14 +1144,16 @@ func (f *fragment) minUnsigned(filter *Row, bitDepth uint) (min int64, count uin } } } - return min, count + return min, count, nil } // max returns the max of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *fragment) max(filter *Row, bitDepth uint) (max int64, count uint64, err error) { - consider := f.row(bsiExistsBit) - if filter != nil { +func (f *fragment) max(tx Tx, filter *Row, bitDepth uint) (max int64, count uint64, err error) { + consider, err := f.row(tx, bsiExistsBit) + if err != nil { + return max, count, err + } else if filter != nil { consider = consider.Intersect(filter) } @@ -1113,21 +1163,29 @@ func (f *fragment) max(filter *Row, bitDepth uint) (max int64, count uint64, err } // Find lowest negative number w/o sign and negate, if no positives are available. - pos := consider.Difference(f.row(bsiSignBit)) + row, err := f.row(tx, bsiSignBit) + if err != nil { + return max, count, err + } + pos := consider.Difference(row) if !pos.Any() { - max, count = f.minUnsigned(consider, bitDepth) - return -max, count, nil + max, count, err = f.minUnsigned(tx, consider, bitDepth) + return -max, count, err } // Otherwise find highest positive number. - max, count = f.maxUnsigned(pos, bitDepth) - return max, count, nil + return f.maxUnsigned(tx, pos, bitDepth) } // maxUnsigned the highest value without considering the sign bit. Filter is required. -func (f *fragment) maxUnsigned(filter *Row, bitDepth uint) (max int64, count uint64) { +func (f *fragment) maxUnsigned(tx Tx, filter *Row, bitDepth uint) (max int64, count uint64, err error) { for i := int(bitDepth - 1); i >= 0; i-- { - row := f.row(uint64(bsiOffsetBit + i)).Intersect(filter) + row, err := f.row(tx, uint64(bsiOffsetBit+i)) + if err != nil { + return max, count, err + } + row = row.Intersect(filter) + count = row.Count() if count > 0 { max += (1 << uint(i)) @@ -1136,69 +1194,86 @@ func (f *fragment) maxUnsigned(filter *Row, bitDepth uint) (max int64, count uin count = filter.Count() } } - return max, count + return max, count, nil } // minRow returns minRowID of the rows in the filter and its count. // if filter is nil, it returns fragment.minRowID, 1 // if fragment has no rows, it returns 0, 0 -func (f *fragment) minRow(filter *Row) (uint64, uint64) { - minRowID, hasRowID := f.minRowID() +func (f *fragment) minRow(tx Tx, filter *Row) (uint64, uint64, error) { + minRowID, hasRowID, err := f.minRowID(tx) + if err != nil { + return 0, 0, err + } if hasRowID { if filter == nil { - return minRowID, 1 + return minRowID, 1, nil } // iterate from min row ID and return the first that intersects with filter. for i := minRowID; i <= f.maxRowID; i++ { - row := f.row(i).Intersect(filter) + row, err := f.row(tx, i) + if err != nil { + return 0, 0, err + } + row = row.Intersect(filter) + count := row.Count() if count > 0 { - return i, count + return i, count, nil } } } - return 0, 0 + return 0, 0, nil } // maxRow returns maxRowID of the rows in the filter and its count. // if filter is nil, it returns fragment.maxRowID, 1 // if fragment has no rows, it returns 0, 0 -func (f *fragment) maxRow(filter *Row) (uint64, uint64) { - minRowID, hasRowID := f.minRowID() +func (f *fragment) maxRow(tx Tx, filter *Row) (uint64, uint64, error) { + minRowID, hasRowID, err := f.minRowID(tx) + if err != nil { + return 0, 0, err + } if hasRowID { if filter == nil { - return f.maxRowID, 1 + return f.maxRowID, 1, nil } // iterate back from max row ID and return the first that intersects with filter. // TODO: implement reverse container iteration to improve performance here for sparse data. --Jaffee for i := f.maxRowID; i >= minRowID; i-- { - row := f.row(i).Intersect(filter) + row, err := f.row(tx, i) + if err != nil { + return 0, 0, err + } + row = row.Intersect(filter) + count := row.Count() if count > 0 { - return i, count + return i, count, nil } } } - return 0, 0 + return 0, 0, nil } // calculateMaxRowID determines the field's maxRowID value based // on the contents of its storage, and sets the struct argument. -func (f *fragment) calculateMaxRowID() { +func (f *fragment) calculateMaxRowID() (err error) { f.maxRowID = f.storage.Max() / ShardWidth + return nil } // rangeOp returns bitmaps with a bsiGroup value encoding matching the predicate. -func (f *fragment) rangeOp(op pql.Token, bitDepth uint, predicate int64) (*Row, error) { +func (f *fragment) rangeOp(tx Tx, op pql.Token, bitDepth uint, predicate int64) (*Row, error) { switch op { case pql.EQ: - return f.rangeEQ(bitDepth, predicate) + return f.rangeEQ(tx, bitDepth, predicate) case pql.NEQ: - return f.rangeNEQ(bitDepth, predicate) + return f.rangeNEQ(tx, bitDepth, predicate) case pql.LT, pql.LTE: - return f.rangeLT(bitDepth, predicate, op == pql.LTE) + return f.rangeLT(tx, bitDepth, predicate, op == pql.LTE) case pql.GT, pql.GTE: - return f.rangeGT(bitDepth, predicate, op == pql.GTE) + return f.rangeGT(tx, bitDepth, predicate, op == pql.GTE) default: return nil, ErrInvalidRangeOperation } @@ -1215,21 +1290,35 @@ func absInt64(v int64) uint64 { } } -func (f *fragment) rangeEQ(bitDepth uint, predicate int64) (*Row, error) { +func (f *fragment) rangeEQ(tx Tx, bitDepth uint, predicate int64) (*Row, error) { // Start with set of columns with values set. - b := f.row(bsiExistsBit) + b, err := f.row(tx, bsiExistsBit) + if err != nil { + return nil, err + } // Filter to only positive/negative numbers. upredicate := absInt64(predicate) if predicate < 0 { - b = b.Intersect(f.row(bsiSignBit)) // only negatives + r, err := f.row(tx, bsiSignBit) + if err != nil { + return nil, err + } + b = b.Intersect(r) // only negatives } else { - b = b.Difference(f.row(bsiSignBit)) // only positives + r, err := f.row(tx, bsiSignBit) + if err != nil { + return nil, err + } + b = b.Difference(r) // only positives } // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.row(uint64(bsiOffsetBit + i)) + row, err := f.row(tx, uint64(bsiOffsetBit+i)) + if err != nil { + return nil, err + } bit := (upredicate >> uint(i)) & 1 if bit == 1 { @@ -1242,12 +1331,15 @@ func (f *fragment) rangeEQ(bitDepth uint, predicate int64) (*Row, error) { return b, nil } -func (f *fragment) rangeNEQ(bitDepth uint, predicate int64) (*Row, error) { +func (f *fragment) rangeNEQ(tx Tx, bitDepth uint, predicate int64) (*Row, error) { // Start with set of columns with values set. - b := f.row(bsiExistsBit) + b, err := f.row(tx, bsiExistsBit) + if err != nil { + return nil, err + } // Get the equal bitmap. - eq, err := f.rangeEQ(bitDepth, predicate) + eq, err := f.rangeEQ(tx, bitDepth, predicate) if err != nil { return nil, err } @@ -1258,16 +1350,22 @@ func (f *fragment) rangeNEQ(bitDepth uint, predicate int64) (*Row, error) { return b, nil } -func (f *fragment) rangeLT(bitDepth uint, predicate int64, allowEquality bool) (*Row, error) { +func (f *fragment) rangeLT(tx Tx, bitDepth uint, predicate int64, allowEquality bool) (*Row, error) { if predicate == 1 && !allowEquality { predicate, allowEquality = 0, true } // Start with set of columns with values set. - b := f.row(bsiExistsBit) + b, err := f.row(tx, bsiExistsBit) + if err != nil { + return nil, err + } // Get the sign bit row. - sign := f.row(bsiSignBit) + sign, err := f.row(tx, bsiSignBit) + if err != nil { + return nil, err + } // Create predicate without sign bit. upredicate := absInt64(predicate) @@ -1278,17 +1376,17 @@ func (f *fragment) rangeLT(bitDepth uint, predicate int64, allowEquality bool) ( return b.Intersect(sign), nil case predicate == 0 && allowEquality: // Match all integers that are either negative or 0. - zeroes, err := f.rangeEQ(bitDepth, 0) + zeroes, err := f.rangeEQ(tx, bitDepth, 0) if err != nil { return nil, err } return b.Intersect(sign).Union(zeroes), nil case predicate < 0: // Match all every negative number beyond the predicate. - return f.rangeGTUnsigned(b.Intersect(sign), bitDepth, upredicate, allowEquality) + return f.rangeGTUnsigned(tx, b.Intersect(sign), bitDepth, upredicate, allowEquality) default: // Match positive numbers less than the predicate, and all negatives. - pos, err := f.rangeLTUnsigned(b.Difference(sign), bitDepth, upredicate, allowEquality) + pos, err := f.rangeLTUnsigned(tx, b.Difference(sign), bitDepth, upredicate, allowEquality) if err != nil { return nil, err } @@ -1306,7 +1404,7 @@ func msb(x uint64) uint { } // rangeLTUnsigned returns all bits LT/LTE the predicate without considering the sign bit. -func (f *fragment) rangeLTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { +func (f *fragment) rangeLTUnsigned(tx Tx, filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { switch { case msb(predicate) > bitDepth: fallthrough @@ -1317,7 +1415,10 @@ func (f *fragment) rangeLTUnsigned(filter *Row, bitDepth uint, predicate uint64, // This query matches everything that is not (1<= 0 && predicate > 0 && remaining.Any(); i-- { - row := f.row(uint64(bsiOffsetBit + i)) + row, err := f.row(tx, uint64(bsiOffsetBit+i)) + if err != nil { + return nil, err + } zeroes := remaining.Difference(row) switch (predicate >> uint(i)) & 1 { case 1: @@ -1345,22 +1449,28 @@ func (f *fragment) rangeLTUnsigned(filter *Row, bitDepth uint, predicate uint64, return matched, nil } -func (f *fragment) rangeGT(bitDepth uint, predicate int64, allowEquality bool) (*Row, error) { +func (f *fragment) rangeGT(tx Tx, bitDepth uint, predicate int64, allowEquality bool) (*Row, error) { if predicate == -1 && !allowEquality { predicate, allowEquality = 0, true } - b := f.row(bsiExistsBit) + b, err := f.row(tx, bsiExistsBit) + if err != nil { + return nil, err + } // Create predicate without sign bit. upredicate := absInt64(predicate) - sign := f.row(bsiSignBit) + sign, err := f.row(tx, bsiSignBit) + if err != nil { + return nil, err + } switch { case predicate == 0 && !allowEquality: // Match all positive numbers except zero. - nonzero, err := f.rangeNEQ(bitDepth, 0) + nonzero, err := f.rangeNEQ(tx, bitDepth, 0) if err != nil { return nil, err } @@ -1371,10 +1481,10 @@ func (f *fragment) rangeGT(bitDepth uint, predicate int64, allowEquality bool) ( return b.Difference(sign), nil case predicate >= 0: // Match all positive numbers greater than the predicate. - return f.rangeGTUnsigned(b.Difference(sign), bitDepth, upredicate, allowEquality) + return f.rangeGTUnsigned(tx, b.Difference(sign), bitDepth, upredicate, allowEquality) default: // Match all positives and greater negatives. - neg, err := f.rangeLTUnsigned(b.Intersect(sign), bitDepth, upredicate, allowEquality) + neg, err := f.rangeLTUnsigned(tx, b.Intersect(sign), bitDepth, upredicate, allowEquality) if err != nil { return nil, err } @@ -1383,7 +1493,7 @@ func (f *fragment) rangeGT(bitDepth uint, predicate int64, allowEquality bool) ( } } -func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { +func (f *fragment) rangeGTUnsigned(tx Tx, filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { switch { case predicate == 0 && allowEquality: // This query matches all possible values. @@ -1392,7 +1502,10 @@ func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, // This query matches everything that is not 0. matches := NewRow() for i := uint(0); i < bitDepth; i++ { - row := f.row(uint64(bsiOffsetBit + i)) + row, err := f.row(tx, uint64(bsiOffsetBit+i)) + if err != nil { + return nil, err + } matches = matches.Union(filter.Intersect(row)) } return matches, nil @@ -1405,7 +1518,10 @@ func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, remaining := filter predicate |= (^uint64(0)) << bitDepth for i := int(bitDepth - 1); i >= 0 && predicate < ^uint64(0) && remaining.Any(); i-- { - row := f.row(uint64(bsiOffsetBit + i)) + row, err := f.row(tx, uint64(bsiOffsetBit+i)) + if err != nil { + return nil, err + } ones := remaining.Intersect(row) switch (predicate >> uint(i)) & 1 { case 1: @@ -1422,33 +1538,52 @@ func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, } // notNull returns the exists row. -func (f *fragment) notNull() (*Row, error) { - return f.row(bsiExistsBit), nil +func (f *fragment) notNull(tx Tx) (*Row, error) { + return f.row(tx, bsiExistsBit) } // rangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax. -func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax int64) (*Row, error) { - b := f.row(bsiExistsBit) +func (f *fragment) rangeBetween(tx Tx, bitDepth uint, predicateMin, predicateMax int64) (*Row, error) { + b, err := f.row(tx, bsiExistsBit) + if err != nil { + return nil, err + } // Convert predicates to unsigned values. upredicateMin, upredicateMax := absInt64(predicateMin), absInt64(predicateMax) switch { case predicateMin == predicateMax: - return f.rangeEQ(bitDepth, predicateMin) + return f.rangeEQ(tx, bitDepth, predicateMin) case predicateMin >= 0: // Handle positive-only values. - return f.rangeBetweenUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMin, upredicateMax) - case predicateMax < 0: - // Handle negative-only values. Swap unsigned min/max predicates. - return f.rangeBetweenUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMax, upredicateMin) - default: - // If predicate crosses positive/negative boundary then handle separately and union. - pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMax, true) + r, err := f.row(tx, bsiSignBit) if err != nil { return nil, err } - neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMin, true) + return f.rangeBetweenUnsigned(tx, b.Difference(r), bitDepth, upredicateMin, upredicateMax) + case predicateMax < 0: + // Handle negative-only values. Swap unsigned min/max predicates. + r, err := f.row(tx, bsiSignBit) + if err != nil { + return nil, err + } + return f.rangeBetweenUnsigned(tx, b.Intersect(r), bitDepth, upredicateMax, upredicateMin) + default: + // If predicate crosses positive/negative boundary then handle separately and union. + r0, err := f.row(tx, bsiSignBit) + if err != nil { + return nil, err + } + pos, err := f.rangeLTUnsigned(tx, b.Difference(r0), bitDepth, upredicateMax, true) + if err != nil { + return nil, err + } + r1, err := f.row(tx, bsiSignBit) + if err != nil { + return nil, err + } + neg, err := f.rangeLTUnsigned(tx, b.Intersect(r1), bitDepth, upredicateMin, true) if err != nil { return nil, err } @@ -1457,21 +1592,24 @@ func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax int64) } // rangeBetweenUnsigned returns BSI columns for a range of values. Disregards the sign bit. -func (f *fragment) rangeBetweenUnsigned(filter *Row, bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { +func (f *fragment) rangeBetweenUnsigned(tx Tx, filter *Row, bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { switch { case predicateMax > (1< firstDiff; i-- { - row := f.row(uint64(bsiOffsetBit + i)) + row, err := f.row(tx, uint64(bsiOffsetBit+i)) + if err != nil { + return nil, err + } switch (predicateMin >> uint(i)) & 1 { case 1: remaining = remaining.Intersect(row) @@ -1481,11 +1619,11 @@ func (f *fragment) rangeBetweenUnsigned(filter *Row, bitDepth uint, predicateMin } var err error - remaining, err = f.rangeGTUnsigned(remaining, uint(firstDiff+1), predicateMin, true) + remaining, err = f.rangeGTUnsigned(tx, remaining, uint(firstDiff+1), predicateMin, true) if err != nil { return nil, err } - remaining, err = f.rangeLTUnsigned(remaining, uint(firstDiff+1), predicateMax, true) + remaining, err = f.rangeLTUnsigned(tx, remaining, uint(firstDiff+1), predicateMax, true) if err != nil { return nil, err } @@ -1504,29 +1642,23 @@ func (f *fragment) pos(rowID, columnID uint64) (uint64, error) { // forEachBit executes fn for every bit set in the fragment. // Errors returned from fn are passed through. -func (f *fragment) forEachBit(fn func(rowID, columnID uint64) error) error { +func (f *fragment) forEachBit(tx Tx, fn func(rowID, columnID uint64) error) error { f.mu.Lock() defer f.mu.Unlock() - - var err error - f.storage.ForEach(func(i uint64) { - // Skip if an error has already occurred. - if err != nil { - return - } - - // Invoke caller's function. - err = fn(i/ShardWidth, (f.shard*ShardWidth)+(i%ShardWidth)) + return tx.ForEach(f.index, f.field, f.view, f.shard, func(i uint64) error { + return fn(i/ShardWidth, (f.shard*ShardWidth)+(i%ShardWidth)) }) - return err } // top returns the top rows from the fragment. // If opt.Src is specified then only rows which intersect src are returned. // If opt.FilterValues exist then the row attribute specified by field is matched. -func (f *fragment) top(opt topOptions) ([]Pair, error) { +func (f *fragment) top(tx Tx, opt topOptions) ([]Pair, error) { // Retrieve pairs. If no row ids specified then return from cache. - pairs := f.topBitmapPairs(opt.RowIDs) + pairs, err := f.topBitmapPairs(tx, opt.RowIDs) + if err != nil { + return nil, err + } // If row ids are provided, we don't want to truncate the result set if len(opt.RowIDs) > 0 { @@ -1595,7 +1727,11 @@ func (f *fragment) top(opt topOptions) ([]Pair, error) { // Calculate count and append. count := cnt if opt.Src != nil { - count = opt.Src.intersectionCount(f.row(rowID)) + r, err := f.row(tx, rowID) + if err != nil { + return nil, err + } + count = opt.Src.intersectionCount(r) } if count == 0 { continue @@ -1639,7 +1775,11 @@ func (f *fragment) top(opt topOptions) ([]Pair, error) { // Calculate the intersecting column count and skip if it's below our // last row in our current result set. - count := opt.Src.intersectionCount(f.row(rowID)) + r, err := f.row(tx, rowID) + if err != nil { + return nil, err + } + count := opt.Src.intersectionCount(r) if count < threshold { continue } @@ -1658,17 +1798,17 @@ func (f *fragment) top(opt topOptions) ([]Pair, error) { return r, nil } -func (f *fragment) topBitmapPairs(rowIDs []uint64) []bitmapPair { +func (f *fragment) topBitmapPairs(tx Tx, rowIDs []uint64) ([]bitmapPair, error) { // Don't retrieve from storage if CacheTypeNone. if f.CacheType == CacheTypeNone { - return f.cache.Top() + return f.cache.Top(), nil } // If no specific rows are requested, retrieve top rows. if len(rowIDs) == 0 { f.mu.Lock() defer f.mu.Unlock() f.cache.Invalidate() - return f.cache.Top() + return f.cache.Top(), nil } // Otherwise retrieve specific rows. @@ -1683,7 +1823,10 @@ func (f *fragment) topBitmapPairs(rowIDs []uint64) []bitmapPair { continue } - row := f.row(rowID) + row, err := f.row(tx, rowID) + if err != nil { + return nil, err + } if row.Count() > 0 { // Otherwise load from storage. pairs = append(pairs, bitmapPair{ @@ -1694,7 +1837,7 @@ func (f *fragment) topBitmapPairs(rowIDs []uint64) []bitmapPair { } sortPairs := bitmapPairs(pairs) sort.Sort(&sortPairs) - return pairs + return pairs, nil } // topOptions represents options passed into the Top() function. @@ -1717,12 +1860,18 @@ type topOptions struct { // Checksum returns a checksum for the entire fragment. // If two fragments have the same checksum then they have the same data. -func (f *fragment) Checksum() []byte { +func (f *fragment) Checksum() ([]byte, error) { h := xxhash.New() - for _, block := range f.Blocks() { + + blocks, err := f.Blocks() + if err != nil { + return nil, err + } + + for _, block := range blocks { _, _ = h.Write(block.Checksum) } - return h.Sum(nil) + return h.Sum(nil), nil } // InvalidateChecksums clears all cached block checksums. @@ -1733,7 +1882,7 @@ func (f *fragment) InvalidateChecksums() { } // Blocks returns info for all blocks containing data. -func (f *fragment) Blocks() []FragmentBlock { +func (f *fragment) Blocks() ([]FragmentBlock, error) { f.mu.Lock() defer f.mu.Unlock() @@ -1749,7 +1898,7 @@ func (f *fragment) Blocks() []FragmentBlock { // Iterate over each value in the fragment. v, eof := itr.Next() if eof { - return nil + return nil, nil } blockID := int(v / (HashBlockSize * ShardWidth)) for { @@ -1795,7 +1944,7 @@ func (f *fragment) Blocks() []FragmentBlock { } } - return a + return a, nil } // readContiguousChecksums appends multiple checksums in a row and returns the count added. @@ -1814,14 +1963,17 @@ func (f *fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i } // blockData returns bits in a block as row & column ID pairs. -func (f *fragment) blockData(id int) (rowIDs, columnIDs []uint64) { +func (f *fragment) blockData(id int) (rowIDs, columnIDs []uint64, err error) { f.mu.Lock() defer f.mu.Unlock() - f.storage.ForEachRange(uint64(id)*HashBlockSize*ShardWidth, (uint64(id)+1)*HashBlockSize*ShardWidth, func(i uint64) { + if err := f.storage.ForEachRange(uint64(id)*HashBlockSize*ShardWidth, (uint64(id)+1)*HashBlockSize*ShardWidth, func(i uint64) error { rowIDs = append(rowIDs, i/ShardWidth) columnIDs = append(columnIDs, i%ShardWidth) - }) - return rowIDs, columnIDs + return nil + }); err != nil { + return nil, nil, err + } + return rowIDs, columnIDs, nil } // mergeBlock compares the block's bits and computes a diff with another set of block bits. @@ -1830,7 +1982,7 @@ func (f *fragment) blockData(id int) (rowIDs, columnIDs []uint64) { // For example, if 3 blocks are compared and two have a set bit and one has a // cleared bit then the bit is considered set. The function returns the // diff per incoming block so that all can be in sync. -func (f *fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, err error) { +func (f *fragment) mergeBlock(tx Tx, id int, data []pairSet) (sets, clears []pairSet, err error) { // Ensure that all pair sets are of equal length. for i := range data { if len(data[i].rowIDs) != len(data[i].columnIDs) { @@ -1850,10 +2002,14 @@ func (f *fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e maxColumnID := uint64(ShardWidth) - 1 // Create buffered iterator for local block. + bm, err := tx.RoaringBitmap(f.index, f.field, f.view, f.shard) + if err != nil { + return nil, nil, err + } itrs := make([]*bufIterator, 1, len(data)+1) itrs[0] = newBufIterator( newLimitIterator( - newRoaringIterator(f.storage.Iterator()), maxRowID, maxColumnID, + newRoaringIterator(bm.Iterator()), maxRowID, maxColumnID, ), ) @@ -1951,21 +2107,21 @@ func (f *fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e // bulkImport bulk imports a set of bits and then snapshots the storage. // The cache is updated to reflect the new data. -func (f *fragment) bulkImport(rowIDs, columnIDs []uint64, options *ImportOptions) error { +func (f *fragment) bulkImport(tx Tx, rowIDs, columnIDs []uint64, options *ImportOptions) error { // Verify that there are an equal number of row ids and column ids. if len(rowIDs) != len(columnIDs) { return fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs)) } if f.mutexVector != nil && !options.Clear { - return f.bulkImportMutex(rowIDs, columnIDs) + return f.bulkImportMutex(tx, rowIDs, columnIDs) } - return f.bulkImportStandard(rowIDs, columnIDs, options) + return f.bulkImportStandard(tx, rowIDs, columnIDs, options) } // bulkImportStandard performs a bulk import on a standard fragment. May mutate // its rowIDs and columnIDs arguments. -func (f *fragment) bulkImportStandard(rowIDs, columnIDs []uint64, options *ImportOptions) (err error) { +func (f *fragment) bulkImportStandard(tx Tx, rowIDs, columnIDs []uint64, options *ImportOptions) (err error) { // rowSet maintains the set of rowIDs present in this import. It allows the // cache to be updated once per row, instead of once per bit. TODO: consider // sorting by rowID/columnID first and avoiding the map allocation here. (we @@ -2067,7 +2223,7 @@ func (f *fragment) importPositions(set, clear []uint64, rowSet map[uint64]struct // mutex restrictions. Because the mutex requirements must be checked // against storage, this method must acquire a write lock on the fragment // during the entire process, and it handles every bit independently. -func (f *fragment) bulkImportMutex(rowIDs, columnIDs []uint64) error { +func (f *fragment) bulkImportMutex(tx Tx, rowIDs, columnIDs []uint64) error { f.mu.Lock() defer f.mu.Unlock() @@ -2086,7 +2242,7 @@ func (f *fragment) bulkImportMutex(rowIDs, columnIDs []uint64) error { clearIdx := 0 for i := range rowIDs { rowID, columnID := rowIDs[i], columnIDs[i] - if existingRowID, found, err := f.mutexVector.Get(columnID); err != nil { + if existingRowID, found, err := f.mutexVector.Get(tx, columnID); err != nil { return errors.Wrap(err, "getting mutex vector data") } else if found && existingRowID != rowID { // Determine the position of the bit in the storage. @@ -2165,7 +2321,7 @@ func (f *fragment) importValueSmallWrite(columnIDs []uint64, values []int64, bit } // importValue bulk imports a set of range-encoded values. -func (f *fragment) importValue(columnIDs []uint64, values []int64, bitDepth uint, clear bool) error { +func (f *fragment) importValue(tx Tx, columnIDs []uint64, values []int64, bitDepth uint, clear bool) error { f.mu.Lock() defer f.mu.Unlock() @@ -2180,12 +2336,14 @@ func (f *fragment) importValue(columnIDs []uint64, values []int64, bitDepth uint // Process every value. // If an error occurs then reopen the storage. - f.storage.OpWriter = nil + if f.storage != nil { + f.storage.OpWriter = nil + } totalChanges := 0 if err := func() (err error) { for i := range columnIDs { columnID, value := columnIDs[i], values[i] - changed, err := f.importSetValue(columnID, bitDepth, value, clear) + changed, err := f.importSetValue(tx, columnID, bitDepth, value, clear) if err != nil { return errors.Wrapf(err, "importSetValue") } @@ -2275,12 +2433,12 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, data []byte, cl } // importRoaringOverwrite overwrites the specified block with the provided data. -func (f *fragment) importRoaringOverwrite(ctx context.Context, data []byte, block int) error { +func (f *fragment) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte, block int) error { f.mu.Lock() defer f.mu.Unlock() // Clear the existing data from fragment block. - if _, err := f.unprotectedClearBlock(block); err != nil { + if _, err := f.unprotectedClearBlock(tx, block); err != nil { return errors.Wrapf(err, "clearing block: %d", block) } @@ -2609,9 +2767,9 @@ func (f *fragment) readCacheFromArchive(r io.Reader) error { return nil } -func (f *fragment) minRowID() (uint64, bool) { - min, ok := f.storage.Min() - return min / ShardWidth, ok +func (f *fragment) minRowID(tx Tx) (uint64, bool, error) { + min, ok, err := tx.Min(f.index, f.field, f.view, f.shard) + return min / ShardWidth, ok, err } // rowFilter is a function signature for controlling iteration over containers @@ -2681,24 +2839,27 @@ func filterWithRows(rows []uint64) rowFilter { // returning done == true will cause processing to stop after all filters for // this container have been processed. The rows accumulated up to this point // (including this row if all filters passed) will be returned. -func (f *fragment) rows(ctx context.Context, start uint64, filters ...rowFilter) []uint64 { +func (f *fragment) rows(ctx context.Context, tx Tx, start uint64, filters ...rowFilter) ([]uint64, error) { f.mu.RLock() defer f.mu.RUnlock() - return f.unprotectedRows(ctx, start, filters...) + return f.unprotectedRows(ctx, tx, start, filters...) } // unprotectedRows calls rows without grabbing the mutex. -func (f *fragment) unprotectedRows(ctx context.Context, start uint64, filters ...rowFilter) []uint64 { +func (f *fragment) unprotectedRows(ctx context.Context, tx Tx, start uint64, filters ...rowFilter) ([]uint64, error) { startKey := rowToKey(start) - i, _ := f.storage.Containers.Iterator(startKey) + i, _, err := tx.ContainerIterator(f.index, f.field, f.view, f.shard, startKey) + if err != nil { + return nil, err + } rows := make([]uint64, 0) var lastRow uint64 = math.MaxUint64 // Loop over the existing containers. for i.Next() { // caller doesn't need a result anymore. - if ctx.Err() != nil { - return nil + if err := ctx.Err(); err != nil { + return nil, err } key, c := i.Value() @@ -2725,10 +2886,10 @@ func (f *fragment) unprotectedRows(ctx context.Context, start uint64, filters .. rows = append(rows, vRow) } if done { - return rows + return rows, nil } } - return rows + return rows, nil } // blockToRoaringData converts a fragment block into a roaring.Bitmap @@ -2737,7 +2898,10 @@ func (f *fragment) unprotectedRows(ctx context.Context, start uint64, filters .. // block data as roaring without having to go through // this rows/columns step. func (f *fragment) blockToRoaringData(block int) ([]byte, error) { - rowIDs, columnIDs := f.blockData(block) + rowIDs, columnIDs, err := f.blockData(block) + if err != nil { + return nil, err + } return bitsToRoaringData(pairSet{ columnIDs: columnIDs, rowIDs: rowIDs, @@ -2759,13 +2923,14 @@ func upgradeRoaringBSIv2(f *fragment, bitDepth uint) (string, error) { f.mu.Lock() defer f.mu.Unlock() - f.storage.ForEach(func(i uint64) { + _ = f.storage.ForEach(func(i uint64) error { rowID, columnID := i/ShardWidth, (f.shard*ShardWidth)+(i%ShardWidth) if rowID == uint64(bitDepth) { _, _ = other.Add(pos(bsiExistsBit, columnID)) // move exists bit to beginning } else { _, _ = other.Add(pos(rowID+bsiOffsetBit, columnID)) // move other bits up } + return nil }) }() @@ -2794,17 +2959,17 @@ type rowIterator interface { // Seek(offset int64, whence int) (int64, error) Seek(uint64) - Next() (*Row, uint64, *int64, bool) + Next() (*Row, uint64, *int64, bool, error) } -func (f *fragment) rowIterator(wrap bool, filters ...rowFilter) rowIterator { +func (f *fragment) rowIterator(tx Tx, wrap bool, filters ...rowFilter) (rowIterator, error) { if strings.HasPrefix(f.view, viewBSIGroupPrefix) { - return f.intRowIterator(wrap, filters...) + return f.intRowIterator(tx, wrap, filters...) } // viewStandard // TODO(kuba) - IMHO we should check if f.view is viewStandard, // but because of testing the function returns set iterator as default one. - return f.setRowIterator(wrap, filters...) + return f.setRowIterator(tx, wrap, filters...) } type intRowIterator struct { @@ -2815,7 +2980,7 @@ type intRowIterator struct { wrap bool } -func (f *fragment) intRowIterator(wrap bool, filters ...rowFilter) rowIterator { +func (f *fragment) intRowIterator(tx Tx, wrap bool, filters ...rowFilter) (rowIterator, error) { it := intRowIterator{ f: f, colIDs: make(map[int64][]uint64), @@ -2828,21 +2993,37 @@ func (f *fragment) intRowIterator(wrap bool, filters ...rowFilter) rowIterator { f.mu.RLock() defer f.mu.RUnlock() - f.foreachRow(filters, func(rid uint64) { + if err := f.foreachRow(tx, filters, func(rid uint64) error { // skip exist(0) and sign(1) rows if rid == bsiExistsBit || rid == bsiSignBit { - return + return nil } val := int64(1 << (rid - bsiOffsetBit)) - for _, cid := range f.unprotectedRow(rid).Columns() { + r, err := f.unprotectedRow(tx, rid) + if err != nil { + return err + } + for _, cid := range r.Columns() { acc[cid] |= val } - }) + return nil + }); err != nil { + return nil, err + } // apply exist and sign bits - allCols := f.unprotectedRow(0).Columns() - signCols := f.unprotectedRow(1).Columns() + r0, err := f.unprotectedRow(tx, 0) + if err != nil { + return nil, err + } + allCols := r0.Columns() + + r1, err := f.unprotectedRow(tx, 1) + if err != nil { + return nil, err + } + signCols := r1.Columns() signIdx, signLen := 0, len(signCols) // all distinct values @@ -2867,12 +3048,15 @@ func (f *fragment) intRowIterator(wrap bool, filters ...rowFilter) rowIterator { } sort.Sort(it.values) - return &it + return &it, nil } -func (f *fragment) foreachRow(filters []rowFilter, fn func(rid uint64)) { +func (f *fragment) foreachRow(tx Tx, filters []rowFilter, fn func(rid uint64) error) error { var lastRow uint64 = math.MaxUint64 - i, _ := f.storage.Containers.Iterator(rowToKey(0)) + i, _, err := tx.ContainerIterator(f.index, f.field, f.view, f.shard, rowToKey(0)) + if err != nil { + return err + } // Loop over the existing containers. for i.Next() { key, c := i.Value() @@ -2896,13 +3080,16 @@ func (f *fragment) foreachRow(filters []rowFilter, fn func(rid uint64)) { if addRow { lastRow = vRow if fn != nil { - fn(vRow) + if err := fn(vRow); err != nil { + return err + } } } if done { break } } + return nil } func (it *intRowIterator) Seek(rowID uint64) { @@ -2912,10 +3099,10 @@ func (it *intRowIterator) Seek(rowID uint64) { it.cur = idx } -func (it *intRowIterator) Next() (r *Row, rowID uint64, value *int64, wrapped bool) { +func (it *intRowIterator) Next() (r *Row, rowID uint64, value *int64, wrapped bool, err error) { if it.cur >= len(it.values) { if !it.wrap || len(it.values) == 0 { - return nil, 0, nil, true + return nil, 0, nil, true, nil } wrapped = true it.cur = 0 @@ -2926,22 +3113,28 @@ func (it *intRowIterator) Next() (r *Row, rowID uint64, value *int64, wrapped bo r = NewRow(it.colIDs[*value]...) } it.cur++ - return r, rowID, value, wrapped + return r, rowID, value, wrapped, nil } type setRowIterator struct { + tx Tx f *fragment rowIDs []uint64 cur int wrap bool } -func (f *fragment) setRowIterator(wrap bool, filters ...rowFilter) rowIterator { - return &setRowIterator{ - f: f, - rowIDs: f.rows(context.Background(), 0, filters...), // TODO: this may be memory intensive in high cardinality cases - wrap: wrap, +func (f *fragment) setRowIterator(tx Tx, wrap bool, filters ...rowFilter) (rowIterator, error) { + rows, err := f.rows(context.Background(), tx, 0, filters...) + if err != nil { + return nil, err } + return &setRowIterator{ + tx: tx, + f: f, + rowIDs: rows, // TODO: this may be memory intensive in high cardinality cases + wrap: wrap, + }, nil } func (it *setRowIterator) Seek(rowID uint64) { @@ -2951,21 +3144,24 @@ func (it *setRowIterator) Seek(rowID uint64) { it.cur = idx } -func (it *setRowIterator) Next() (r *Row, rowID uint64, _ *int64, wrapped bool) { +func (it *setRowIterator) Next() (r *Row, rowID uint64, _ *int64, wrapped bool, err error) { if it.cur >= len(it.rowIDs) { if !it.wrap || len(it.rowIDs) == 0 { - return nil, 0, nil, true + return nil, 0, nil, true, nil } it.Seek(0) wrapped = true } id := it.rowIDs[it.cur] - r = it.f.row(id) + r, err = it.f.row(it.tx, id) + if err != nil { + return r, rowID, nil, wrapped, err + } rowID = id it.cur++ - return r, rowID, nil, wrapped + return r, rowID, nil, wrapped, nil } // FragmentBlock represents info about a subsection of the rows in a block. @@ -3050,7 +3246,10 @@ func (s *fragmentSyncer) syncFragment() error { for _, node := range nodes { // Read local blocks. if node.ID == s.Node.ID { - b := s.Fragment.Blocks() + b, err := s.Fragment.Blocks() + if err != nil { + return err + } blockSets = append(blockSets, b) continue } @@ -3188,6 +3387,7 @@ func (s *fragmentSyncer) syncBlock(id int) error { defer span.Finish() f := s.Fragment + tx := &RoaringTx{fragment: f} // Read pairs from each remote block. var uris []*URI @@ -3223,7 +3423,7 @@ func (s *fragmentSyncer) syncBlock(id int) error { } // Merge blocks together. - sets, clears, err := f.mergeBlock(id, pairSets) + sets, clears, err := f.mergeBlock(tx, id, pairSets) if err != nil { return errors.Wrap(err, "merging") } @@ -3339,7 +3539,7 @@ func pos(rowID, columnID uint64) uint64 { // vector stores the mapping of colID to rowID. // It's used for a mutex field type. type vector interface { - Get(colID uint64) (uint64, bool, error) + Get(tx Tx, colID uint64) (uint64, bool, error) } // rowsVector implements the vector interface by looking @@ -3359,9 +3559,11 @@ func newRowsVector(f *fragment) *rowsVector { // Additionally, it returns true if a value was found, // otherwise it returns false. Ensure that you already // have the mutex before calling this. -func (v *rowsVector) Get(colID uint64) (uint64, bool, error) { - rows := v.f.unprotectedRows(context.Background(), 0, filterColumn(colID)) - if len(rows) > 1 { +func (v *rowsVector) Get(tx Tx, colID uint64) (uint64, bool, error) { + rows, err := v.f.unprotectedRows(context.Background(), tx, 0, filterColumn(colID)) + if err != nil { + return 0, false, err + } else if len(rows) > 1 { return 0, false, errors.New("found multiple row values for column") } else if len(rows) == 1 { return rows[0], true, nil @@ -3393,9 +3595,11 @@ func newBoolVector(f *fragment) *boolVector { // Additionally, it returns true if a value was found, // otherwise it returns false. Ensure that you already // have the fragment mutex before calling this. -func (v *boolVector) Get(colID uint64) (uint64, bool, error) { - rows := v.f.unprotectedRows(context.Background(), 0, filterColumn(colID)) - if len(rows) > 1 { +func (v *boolVector) Get(tx Tx, colID uint64) (uint64, bool, error) { + rows, err := v.f.unprotectedRows(context.Background(), tx, 0, filterColumn(colID)) + if err != nil { + return 0, false, err + } else if len(rows) > 1 { return 0, false, errors.New("found multiple row values for column") } else if len(rows) == 1 { switch rows[0] { diff --git a/fragment_internal_test.go b/fragment_internal_test.go index fa1ffbeec..661f8430a 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -54,28 +54,31 @@ func TestFragment_SetBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set bits on the fragment. - if _, err := f.setBit(120, 1); err != nil { + if _, err := f.setBit(tx, 120, 1); err != nil { t.Fatal(err) - } else if _, err := f.setBit(120, 6); err != nil { + } else if _, err := f.setBit(tx, 120, 6); err != nil { t.Fatal(err) - } else if _, err := f.setBit(121, 0); err != nil { + } else if _, err := f.setBit(tx, 121, 0); err != nil { t.Fatal(err) } // Verify counts on rows. - if n := f.row(120).Count(); n != 2 { + if n := f.mustRow(tx, 120).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) - } else if n := f.row(121).Count(); n != 1 { + } else if n := f.mustRow(tx, 121).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if n := f.row(120).Count(); n != 2 { + } else if n := f.mustRow(tx, 120).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) - } else if n := f.row(121).Count(); n != 1 { + } else if n := f.mustRow(tx, 121).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -85,24 +88,27 @@ func TestFragment_ClearBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set and then clear bits on the fragment. - if _, err := f.setBit(1000, 1); err != nil { + if _, err := f.setBit(tx, 1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.setBit(1000, 2); err != nil { + } else if _, err := f.setBit(tx, 1000, 2); err != nil { t.Fatal(err) - } else if _, err := f.clearBit(1000, 1); err != nil { + } else if _, err := f.clearBit(tx, 1000, 1); err != nil { t.Fatal(err) } // Verify count on row. - if n := f.row(1000).Count(); n != 1 { + if n := f.mustRow(tx, 1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if n := f.row(1000).Count(); n != 1 { + } else if n := f.mustRow(tx, 1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -111,6 +117,10 @@ func TestFragment_ClearBit(t *testing.T) { func TestFragment_RowcacheMap(t *testing.T) { var done int64 f := mustOpenFragment("i", "f", viewStandard, 0, "") + + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Under -race, this test turns out to take a fairly long time // to run with larger OpN, because we write 50,000 bits to // the bitmap, and everything is being race-detected, and we don't @@ -121,11 +131,11 @@ func TestFragment_RowcacheMap(t *testing.T) { ch := make(chan struct{}) for i := 0; i < f.MaxOpN; i++ { - _, _ = f.setBit(0, uint64(i*32)) + _, _ = f.setBit(tx, 0, uint64(i*32)) } // force snapshot so we get a mmapped row... _ = f.Snapshot() - row := f.row(0) + row := f.mustRow(tx, 0) segment := row.Segments()[0] bitmap := segment.data @@ -143,7 +153,7 @@ func TestFragment_RowcacheMap(t *testing.T) { // then invalidates the other map... for j := 0; j < 5; j++ { for i := 0; i < f.MaxOpN; i++ { - _, _ = f.setBit(0, uint64(i*32+j+1)) + _, _ = f.setBit(tx, 0, uint64(i*32+j+1)) } } atomic.StoreInt64(&done, 1) @@ -155,24 +165,27 @@ func TestFragment_ClearRow(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set and then clear bits on the fragment. - if _, err := f.setBit(1000, 1); err != nil { + if _, err := f.setBit(tx, 1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.setBit(1000, 65536); err != nil { + } else if _, err := f.setBit(tx, 1000, 65536); err != nil { t.Fatal(err) - } else if _, err := f.unprotectedClearRow(1000); err != nil { + } else if _, err := f.unprotectedClearRow(tx, 1000); err != nil { t.Fatal(err) } // Verify count on row. - if n := f.row(1000).Count(); n != 0 { + if n := f.mustRow(tx, 1000).Count(); n != 0 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if n := f.row(1000).Count(); n != 0 { + } else if n := f.mustRow(tx, 1000).Count(); n != 0 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -182,45 +195,48 @@ func TestFragment_SetRow(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 7, "") defer f.Clean(t) + // Obtain transction. + tx := &RoaringTx{fragment: f} + rowID := uint64(1000) // Set bits on the fragment. - if _, err := f.setBit(rowID, 7*ShardWidth+1); err != nil { + if _, err := f.setBit(tx, rowID, 7*ShardWidth+1); err != nil { t.Fatal(err) - } else if _, err := f.setBit(rowID, 7*ShardWidth+65536); err != nil { + } else if _, err := f.setBit(tx, rowID, 7*ShardWidth+65536); err != nil { t.Fatal(err) } // Verify data on row. - if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65536}) { + if cols := f.mustRow(tx, rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65536}) { t.Fatalf("unexpected columns: %+v", cols) } // Verify count on row. - if n := f.row(rowID).Count(); n != 2 { + if n := f.mustRow(tx, rowID).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) } // Set row (overwrite existing data). row := NewRow(7*ShardWidth+1, 7*ShardWidth+65537, 7*ShardWidth+140000) - if changed, err := f.unprotectedSetRow(row, rowID); err != nil { + if changed, err := f.unprotectedSetRow(tx, row, rowID); err != nil { t.Fatal(err) } else if !changed { t.Fatalf("expected changed value: %v", changed) } // Verify data on row. - if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65537, 7*ShardWidth + 140000}) { + if cols := f.mustRow(tx, rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65537, 7*ShardWidth + 140000}) { t.Fatalf("unexpected columns after set row: %+v", cols) } // Verify count on row. - if n := f.row(rowID).Count(); n != 3 { + if n := f.mustRow(tx, rowID).Count(); n != 3 { t.Fatalf("unexpected count after set row: %d", n) } // Close and reopen the fragment & verify the data. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if n := f.row(rowID).Count(); n != 3 { + } else if n := f.mustRow(tx, rowID).Count(); n != 3 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -231,15 +247,18 @@ func TestFragment_SetValue(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set value. - if changed, err := f.setValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(tx, 100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.value(100, 16); err != nil { + if value, exists, err := f.value(tx, 100, 16); err != nil { t.Fatal(err) } else if value != 3829 { t.Fatalf("unexpected value: %d", value) @@ -248,7 +267,7 @@ func TestFragment_SetValue(t *testing.T) { } // Setting value should return no change. - if changed, err := f.setValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(tx, 100, 16, 3829); err != nil { t.Fatal(err) } else if changed { t.Fatal("expected no change") @@ -259,22 +278,25 @@ func TestFragment_SetValue(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set value. - if changed, err := f.setValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(tx, 100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Overwriting value should overwrite all bits. - if changed, err := f.setValue(100, 16, 2028); err != nil { + if changed, err := f.setValue(tx, 100, 16, 2028); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.value(100, 16); err != nil { + if value, exists, err := f.value(tx, 100, 16); err != nil { t.Fatal(err) } else if value != 2028 { t.Fatalf("unexpected value: %d", value) @@ -287,22 +309,25 @@ func TestFragment_SetValue(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set value. - if changed, err := f.setValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(tx, 100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Clear value should overwrite all bits, and set not-null to 0. - if changed, err := f.clearValue(100, 16, 2028); err != nil { + if changed, err := f.clearValue(tx, 100, 16, 2028); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.value(100, 16); err != nil { + if value, exists, err := f.value(tx, 100, 16); err != nil { t.Fatal(err) } else if value != 0 { t.Fatalf("unexpected value: %d", value) @@ -315,15 +340,18 @@ func TestFragment_SetValue(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set value. - if changed, err := f.setValue(100, 10, 20); err != nil { + if changed, err := f.setValue(tx, 100, 10, 20); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Non-existent value. - if value, exists, err := f.value(101, 11); err != nil { + if value, exists, err := f.value(tx, 101, 11); err != nil { t.Fatal(err) } else if value != 0 { t.Fatalf("unexpected value: %d", value) @@ -345,6 +373,9 @@ func TestFragment_SetValue(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set values. m := make(map[uint64]int64) for _, value := range values { @@ -352,14 +383,14 @@ func TestFragment_SetValue(t *testing.T) { m[columnID] = int64(value) - if _, err := f.setValue(columnID, bitDepth, int64(value)); err != nil { + if _, err := f.setValue(tx, columnID, bitDepth, int64(value)); err != nil { t.Fatal(err) } } // Ensure values are set. for columnID, value := range m { - v, exists, err := f.value(columnID, bitDepth) + v, exists, err := f.value(tx, columnID, bitDepth) if err != nil { t.Fatal(err) } else if value != int64(v) { @@ -383,6 +414,9 @@ func TestFragment_Sum(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set values. vals := []struct { cid uint64 @@ -395,13 +429,13 @@ func TestFragment_Sum(t *testing.T) { {4000, 300}, } for _, v := range vals { - if _, err := f.setValue(v.cid, bitDepth, v.val); err != nil { + if _, err := f.setValue(tx, v.cid, bitDepth, v.val); err != nil { t.Fatal(err) } } t.Run("NoFilter", func(t *testing.T) { - if sum, n, err := f.sum(nil, bitDepth); err != nil { + if sum, n, err := f.sum(tx, nil, bitDepth); err != nil { t.Fatal(err) } else if n != 5 { t.Fatalf("unexpected count: %d", n) @@ -411,7 +445,7 @@ func TestFragment_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if sum, n, err := f.sum(NewRow(2000, 4000, 5000), bitDepth); err != nil { + if sum, n, err := f.sum(tx, NewRow(2000, 4000, 5000), bitDepth); err != nil { t.Fatal(err) } else if n != 2 { t.Fatalf("unexpected count: %d", n) @@ -421,11 +455,11 @@ func TestFragment_Sum(t *testing.T) { }) // verify that clearValue clears values - if _, err := f.clearValue(1000, bitDepth, 23); err != nil { + if _, err := f.clearValue(tx, 1000, bitDepth, 23); err != nil { t.Fatal(err) } t.Run("ClearValue", func(t *testing.T) { - if sum, n, err := f.sum(nil, bitDepth); err != nil { + if sum, n, err := f.sum(tx, nil, bitDepth); err != nil { t.Fatal(err) } else if n != 4 { t.Fatalf("unexpected count: %d", n) @@ -442,20 +476,23 @@ func TestFragment_MinMax(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set values. - if _, err := f.setValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(tx, 2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(tx, 3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(tx, 4000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(5000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(tx, 5000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.setValue(6000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(tx, 6000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.setValue(7000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(tx, 7000, bitDepth, 0); err != nil { t.Fatal(err) } @@ -473,7 +510,7 @@ func TestFragment_MinMax(t *testing.T) { {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if min, cnt, err := f.min(test.filter, bitDepth); err != nil { + if min, cnt, err := f.min(tx, test.filter, bitDepth); err != nil { t.Fatal(err) } else if min != test.exp { t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min) @@ -502,7 +539,7 @@ func TestFragment_MinMax(t *testing.T) { columns = test.filter.Columns() } - if max, cnt, err := f.max(test.filter, bitDepth); err != nil { + if max, cnt, err := f.max(tx, test.filter, bitDepth); err != nil { t.Fatal(err) } else if max != test.exp || cnt != test.cnt { t.Errorf("%d. max(%v, %v)=(%v, %v), expected (%v, %v)", i, columns, bitDepth, max, cnt, test.exp, test.cnt) @@ -519,19 +556,22 @@ func TestFragment_Range(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set values. - if _, err := f.setValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(tx, 2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(tx, 3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(tx, 4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for equality. - if b, err := f.rangeOp(pql.EQ, bitDepth, 300); err != nil { + if b, err := f.rangeOp(tx, pql.EQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -542,19 +582,22 @@ func TestFragment_Range(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set values. - if _, err := f.setValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(tx, 2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(tx, 3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(tx, 4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for inequality. - if b, err := f.rangeOp(pql.NEQ, bitDepth, 300); err != nil { + if b, err := f.rangeOp(tx, pql.NEQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -565,44 +608,47 @@ func TestFragment_Range(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set values. - if _, err := f.setValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(tx, 2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(tx, 3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(tx, 4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(tx, 5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(tx, 6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for values less than (ending with set column). - if b, err := f.rangeOp(pql.LT, bitDepth, 301); err != nil { + if b, err := f.rangeOp(tx, pql.LT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values less than (ending with unset column). - if b, err := f.rangeOp(pql.LT, bitDepth, 300); err != nil { + if b, err := f.rangeOp(tx, pql.LT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values less than or equal to (ending with set column). - if b, err := f.rangeOp(pql.LTE, bitDepth, 301); err != nil { + if b, err := f.rangeOp(tx, pql.LTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values less than or equal to (ending with unset column). - if b, err := f.rangeOp(pql.LTE, bitDepth, 300); err != nil { + if b, err := f.rangeOp(tx, pql.LTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -613,11 +659,14 @@ func TestFragment_Range(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) - if _, err := f.setValue(1, 1, 1); err != nil { + // Obtain transaction. + tx := &RoaringTx{fragment: f} + + if _, err := f.setValue(tx, 1, 1, 1); err != nil { t.Fatal(err) } - if b, err := f.rangeOp(pql.LT, 1, 2); err != nil { + if b, err := f.rangeOp(tx, pql.LT, 1, 2); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1}) { t.Fatalf("unepxected coulmns: %+v", b.Columns()) @@ -628,13 +677,16 @@ func TestFragment_Range(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) - if _, err := f.setValue(1, 2, 3); err != nil { + // Obtain transaction. + tx := &RoaringTx{fragment: f} + + if _, err := f.setValue(tx, 1, 2, 3); err != nil { t.Fatal(err) - } else if _, err := f.setValue(2, 2, 0); err != nil { + } else if _, err := f.setValue(tx, 2, 2, 0); err != nil { t.Fatal(err) } - if b, err := f.rangeLTUnsigned(NewRow(1, 2), 2, 3, false); err != nil { + if b, err := f.rangeLTUnsigned(tx, NewRow(1, 2), 2, 3, false); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2}) { t.Fatalf("unepxected coulmns: %+v", b.Columns()) @@ -645,44 +697,47 @@ func TestFragment_Range(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set values. - if _, err := f.setValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(tx, 2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(tx, 3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(tx, 4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(tx, 5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(tx, 6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for values greater than (ending with unset bit). - if b, err := f.rangeOp(pql.GT, bitDepth, 300); err != nil { + if b, err := f.rangeOp(tx, pql.GT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than (ending with set bit). - if b, err := f.rangeOp(pql.GT, bitDepth, 301); err != nil { + if b, err := f.rangeOp(tx, pql.GT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with unset bit). - if b, err := f.rangeOp(pql.GTE, bitDepth, 300); err != nil { + if b, err := f.rangeOp(tx, pql.GTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with set bit). - if b, err := f.rangeOp(pql.GTE, bitDepth, 301); err != nil { + if b, err := f.rangeOp(tx, pql.GTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -693,13 +748,16 @@ func TestFragment_Range(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) - if _, err := f.setValue(1, 2, 0); err != nil { + // Obtain transaction. + tx := &RoaringTx{fragment: f} + + if _, err := f.setValue(tx, 1, 2, 0); err != nil { t.Fatal(err) - } else if _, err := f.setValue(2, 2, 1); err != nil { + } else if _, err := f.setValue(tx, 2, 2, 1); err != nil { t.Fatal(err) } - if b, err := f.rangeGTUnsigned(NewRow(1, 2), 2, 0, false); err != nil { + if b, err := f.rangeGTUnsigned(tx, NewRow(1, 2), 2, 0, false); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2}) { t.Fatalf("unepxected coulmns: %+v", b.Columns()) @@ -710,44 +768,47 @@ func TestFragment_Range(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set values. - if _, err := f.setValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(tx, 2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(tx, 3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(tx, 4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(tx, 5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(tx, 6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for values greater than (ending with unset column). - if b, err := f.rangeBetween(bitDepth, 300, 2817); err != nil { + if b, err := f.rangeBetween(tx, bitDepth, 300, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than (ending with set column). - if b, err := f.rangeBetween(bitDepth, 301, 2817); err != nil { + if b, err := f.rangeBetween(tx, bitDepth, 301, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with unset column). - if b, err := f.rangeBetween(bitDepth, 301, 2816); err != nil { + if b, err := f.rangeBetween(tx, bitDepth, 301, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with set column). - if b, err := f.rangeBetween(bitDepth, 300, 2816); err != nil { + if b, err := f.rangeBetween(tx, bitDepth, 300, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -758,11 +819,14 @@ func TestFragment_Range(t *testing.T) { // benchmarkSetValues is a helper function to explore, very roughly, the cost // of setting values. func benchmarkSetValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { + // Obtain transaction. + tx := &RoaringTx{fragment: f} + column := uint64(0) for i := 0; i < b.N; i++ { // We're not checking the error because this is a benchmark. // That does mean the result could be completely wrong... - _, _ = f.setValue(column, bitDepth, int64(i)) + _, _ = f.setValue(tx, column, bitDepth, int64(i)) column = cfunc(column) } } @@ -773,6 +837,7 @@ func BenchmarkFragment_SetValue(b *testing.B) { for _, bitDepth := range depths { name := fmt.Sprintf("Depth%d", bitDepth) f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + b.Run(name+"_Sparse", func(b *testing.B) { benchmarkSetValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) }) }) @@ -788,6 +853,9 @@ func BenchmarkFragment_SetValue(b *testing.B) { // benchmarkImportValues is a helper function to explore, very roughly, the cost // of setting values using the special setter used for imports. func benchmarkImportValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { + // Obtain transaction. + tx := &RoaringTx{fragment: f} + column := uint64(0) b.StopTimer() columns := make([]uint64, b.N) @@ -798,7 +866,7 @@ func benchmarkImportValues(b *testing.B, bitDepth uint, f *fragment, cfunc func( column = cfunc(column) } b.StartTimer() - err := f.importValue(columns, values, bitDepth, false) + err := f.importValue(tx, columns, values, bitDepth, false) if err != nil { b.Fatalf("error importing values: %s", err) } @@ -851,13 +919,17 @@ func BenchmarkFragment_RepeatedSmallImports(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") f.MaxOpN = opN defer f.Clean(b) + + // Obtain transaction. + tx := &RoaringTx{fragment: f} + err := f.importRoaringT(getZipfRowsSliceRoaring(uint64(numRows), 1, 0, ShardWidth), false) if err != nil { b.Fatalf("importing base data for benchmark: %v", err) } b.StartTimer() for i := 0; i < numUpdates; i++ { - err := f.bulkImportStandard( + err := f.bulkImportStandard(tx, updateRows[bitsPerUpdate*i:bitsPerUpdate*(i+1)], updateRows[bitsPerUpdate*i:bitsPerUpdate*(i+1)], &ImportOptions{}, @@ -887,6 +959,7 @@ func BenchmarkFragment_RepeatedSmallImportsRoaring(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") f.MaxOpN = opN defer f.Clean(b) + err := f.importRoaringT(getZipfRowsSliceRoaring(numRows, 1, 0, ShardWidth), false) if err != nil { b.Fatalf("importing base data for benchmark: %v", err) @@ -935,13 +1008,17 @@ func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) { b.StopTimer() f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) f.MaxOpN = opN - err := f.importValue(initialCols, initialVals, 21, false) + + // Obtain transaction. + tx := &RoaringTx{fragment: f} + + err := f.importValue(tx, initialCols, initialVals, 21, false) if err != nil { b.Fatalf("initial value import: %v", err) } b.StartTimer() for j := 0; j < numUpdates; j++ { - err := f.importValue( + err := f.importValue(tx, updateCols[valsPerUpdate*j:valsPerUpdate*(j+1)], updateVals[valsPerUpdate*j:valsPerUpdate*(j+1)], 21, @@ -964,26 +1041,29 @@ func TestFragment_Snapshot(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set and then clear bits on the fragment. - if _, err := f.setBit(1000, 1); err != nil { + if _, err := f.setBit(tx, 1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.setBit(1000, 2); err != nil { + } else if _, err := f.setBit(tx, 1000, 2); err != nil { t.Fatal(err) - } else if _, err := f.clearBit(1000, 1); err != nil { + } else if _, err := f.clearBit(tx, 1000, 1); err != nil { t.Fatal(err) } // Snapshot bitmap and verify data. if err := f.Snapshot(); err != nil { t.Fatal(err) - } else if n := f.row(1000).Count(); n != 1 { + } else if n := f.mustRow(tx, 1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if n := f.row(1000).Count(); n != 1 { + } else if n := f.mustRow(tx, 1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -993,18 +1073,21 @@ func TestFragment_ForEachBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set bits on the fragment. - if _, err := f.setBit(100, 20); err != nil { + if _, err := f.setBit(tx, 100, 20); err != nil { t.Fatal(err) - } else if _, err := f.setBit(2, 38); err != nil { + } else if _, err := f.setBit(tx, 2, 38); err != nil { t.Fatal(err) - } else if _, err := f.setBit(2, 37); err != nil { + } else if _, err := f.setBit(tx, 2, 37); err != nil { t.Fatal(err) } // Iterate over bits. var result [][2]uint64 - if err := f.forEachBit(func(rowID, columnID uint64) error { + if err := f.forEachBit(tx, func(rowID, columnID uint64) error { result = append(result, [2]uint64{rowID, columnID}) return nil }); err != nil { @@ -1021,14 +1104,18 @@ func TestFragment_ForEachBit(t *testing.T) { func TestFragment_Top(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Clean(t) + + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set bits on the rows 100, 101, & 102. - f.mustSetBits(100, 1, 3, 200) - f.mustSetBits(101, 1) - f.mustSetBits(102, 1, 2) + f.mustSetBits(tx, 100, 1, 3, 200) + f.mustSetBits(tx, 101, 1) + f.mustSetBits(tx, 102, 1, 2) f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.top(topOptions{N: 2}); err != nil { + if pairs, err := f.top(tx, topOptions{N: 2}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -1044,10 +1131,13 @@ func TestFragment_Top_Filter(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set bits on the rows 100, 101, & 102. - f.mustSetBits(100, 1, 3, 200) - f.mustSetBits(101, 1) - f.mustSetBits(102, 1, 2) + f.mustSetBits(tx, 100, 1, 3, 200) + f.mustSetBits(tx, 101, 1) + f.mustSetBits(tx, 102, 1, 2) f.RecalculateCache() // Assign attributes. err := f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": int64(10)}) @@ -1060,7 +1150,7 @@ func TestFragment_Top_Filter(t *testing.T) { } // Retrieve top rows. - if pairs, err := f.top(topOptions{ + if pairs, err := f.top(tx, topOptions{ N: 2, FilterName: "x", FilterValues: []interface{}{int64(10), int64(15), int64(20)}, @@ -1080,18 +1170,21 @@ func TestFragment_TopN_Intersect(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Create an intersecting input row. src := NewRow(1, 2, 3) // Set bits on various rows. - f.mustSetBits(100, 1, 10, 11, 12) // one intersection - f.mustSetBits(101, 1, 2, 3, 4) // three intersections - f.mustSetBits(102, 1, 2, 4, 5, 6) // two intersections - f.mustSetBits(103, 1000, 1001, 1002) // no intersection + f.mustSetBits(tx, 100, 1, 10, 11, 12) // one intersection + f.mustSetBits(tx, 101, 1, 2, 3, 4) // three intersections + f.mustSetBits(tx, 102, 1, 2, 4, 5, 6) // two intersections + f.mustSetBits(tx, 103, 1000, 1001, 1002) // no intersection f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.top(topOptions{N: 3, Src: src}); err != nil { + if pairs, err := f.top(tx, topOptions{N: 3, Src: src}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 3}, @@ -1111,6 +1204,9 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Create an intersecting input row. src := NewRow( 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, @@ -1136,7 +1232,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.top(topOptions{N: 10, Src: src}); err != nil { + if pairs, err := f.top(tx, topOptions{N: 10, Src: src}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 999, Count: 19}, @@ -1159,13 +1255,16 @@ func TestFragment_TopN_IDs(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set bits on various rows. - f.mustSetBits(100, 1, 2, 3) - f.mustSetBits(101, 4, 5, 6, 7) - f.mustSetBits(102, 8, 9, 10, 11, 12) + f.mustSetBits(tx, 100, 1, 2, 3) + f.mustSetBits(tx, 101, 4, 5, 6, 7) + f.mustSetBits(tx, 102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.top(topOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.top(tx, topOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 4}, @@ -1180,13 +1279,16 @@ func TestFragment_TopN_NopCache(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeNone) defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set bits on various rows. - f.mustSetBits(100, 1, 2, 3) - f.mustSetBits(101, 4, 5, 6, 7) - f.mustSetBits(102, 8, 9, 10, 11, 12) + f.mustSetBits(tx, 100, 1, 2, 3) + f.mustSetBits(tx, 101, 4, 5, 6, 7) + f.mustSetBits(tx, 102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.top(topOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.top(tx, topOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{}) { t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs)) @@ -1228,13 +1330,16 @@ func TestFragment_TopN_CacheSize(t *testing.T) { } defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set bits on various rows. - f.mustSetBits(100, 1, 2, 3) - f.mustSetBits(101, 4, 5, 6, 7) - f.mustSetBits(102, 8, 9, 10, 11, 12) - f.mustSetBits(103, 8, 9, 10, 11, 12, 13) - f.mustSetBits(104, 8, 9, 10, 11, 12, 13, 14) - f.mustSetBits(105, 10, 11) + f.mustSetBits(tx, 100, 1, 2, 3) + f.mustSetBits(tx, 101, 4, 5, 6, 7) + f.mustSetBits(tx, 102, 8, 9, 10, 11, 12) + f.mustSetBits(tx, 103, 8, 9, 10, 11, 12, 13) + f.mustSetBits(tx, 104, 8, 9, 10, 11, 12, 13, 14) + f.mustSetBits(tx, 105, 10, 11) f.RecalculateCache() @@ -1245,7 +1350,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { } // Retrieve top rows. - if pairs, err := f.top(topOptions{N: 5}); err != nil { + if pairs, err := f.top(tx, topOptions{N: 5}); err != nil { t.Fatal(err) } else if len(pairs) > int(cacheSize) { t.Fatalf("TopN count cannot exceed cache size: %d", cacheSize) @@ -1261,16 +1366,24 @@ func TestFragment_Checksum(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Retrieve checksum and set bits. - orig := f.Checksum() - if _, err := f.setBit(1, 200); err != nil { + orig, err := f.Checksum() + if err != nil { t.Fatal(err) - } else if _, err := f.setBit(HashBlockSize*2, 200); err != nil { + } + if _, err := f.setBit(tx, 1, 200); err != nil { + t.Fatal(err) + } else if _, err := f.setBit(tx, HashBlockSize*2, 200); err != nil { t.Fatal(err) } // Ensure new checksum is different. - if chksum := f.Checksum(); bytes.Equal(chksum, orig) { + if chksum, err := f.Checksum(); err != nil { + t.Fatal(err) + } else if bytes.Equal(chksum, orig) { t.Fatalf("expected checksum to change: %x - %x", chksum, orig) } } @@ -1280,35 +1393,44 @@ func TestFragment_Blocks(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Retrieve initial checksum. var prev []FragmentBlock // Set first bit. - if _, err := f.setBit(0, 0); err != nil { + if _, err := f.setBit(tx, 0, 0); err != nil { t.Fatal(err) } - blocks := f.Blocks() - if blocks[0].Checksum == nil { + blocks, err := f.Blocks() + if err != nil { + t.Fatal(err) + } else if blocks[0].Checksum == nil { t.Fatalf("expected checksum: %x", blocks[0].Checksum) } prev = blocks // Set bit on different row. - if _, err := f.setBit(20, 0); err != nil { + if _, err := f.setBit(tx, 20, 0); err != nil { t.Fatal(err) } - blocks = f.Blocks() - if bytes.Equal(blocks[0].Checksum, prev[0].Checksum) { + blocks, err = f.Blocks() + if err != nil { + t.Fatal(err) + } else if bytes.Equal(blocks[0].Checksum, prev[0].Checksum) { t.Fatalf("expected checksum to change: %x", blocks[0].Checksum) } prev = blocks // Set bit on different column. - if _, err := f.setBit(20, 100); err != nil { + if _, err := f.setBit(tx, 20, 100); err != nil { t.Fatal(err) } - blocks = f.Blocks() - if bytes.Equal(blocks[0].Checksum, prev[0].Checksum) { + blocks, err = f.Blocks() + if err != nil { + t.Fatal(err) + } else if bytes.Equal(blocks[0].Checksum, prev[0].Checksum) { t.Fatalf("expected checksum to change: %x", blocks[0].Checksum) } } @@ -1318,13 +1440,18 @@ func TestFragment_Blocks_Empty(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set bits on a different block. - if _, err := f.setBit(100, 1); err != nil { + if _, err := f.setBit(tx, 100, 1); err != nil { t.Fatal(err) } // Ensure checksum for block 1 is blank. - if blocks := f.Blocks(); len(blocks) != 1 { + if blocks, err := f.Blocks(); err != nil { + t.Fatal(err) + } else if len(blocks) != 1 { t.Fatalf("unexpected block count: %d", len(blocks)) } else if blocks[0].ID != 1 { t.Fatalf("unexpected block id: %d", blocks[0].ID) @@ -1336,9 +1463,12 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeLRU) defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.setBit(i, 0); err != nil { + if _, err := f.setBit(tx, i, 0); err != nil { t.Fatal(err) } } @@ -1386,9 +1516,12 @@ func TestFragment_RankCache_Persistence(t *testing.T) { t.Fatal(err) } + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.setBit(i, 0); err != nil { + if _, err := f.setBit(tx, i, 0); err != nil { t.Fatal(err) } } @@ -1421,12 +1554,15 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { f0 := mustOpenFragment("i", "f", viewStandard, 0, "") defer f0.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f0} + // Set and then clear bits on the fragment. - if _, err := f0.setBit(1000, 1); err != nil { + if _, err := f0.setBit(tx, 1000, 1); err != nil { t.Fatal(err) - } else if _, err := f0.setBit(1000, 2); err != nil { + } else if _, err := f0.setBit(tx, 1000, 2); err != nil { t.Fatal(err) - } else if _, err := f0.clearBit(1000, 1); err != nil { + } else if _, err := f0.clearBit(tx, 1000, 1); err != nil { t.Fatal(err) } @@ -1445,6 +1581,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { // Read into another fragment. f1 := mustOpenFragment("i", "f", viewStandard, 0, "") defer f1.Clean(t) + if rn, err := f1.ReadFrom(&buf); err != nil { t.Fatal(err) } else if wn != rn { @@ -1457,7 +1594,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Verify data in other fragment. - if a := f1.row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + if a := f1.mustRow(tx, 1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected columns: %+v", a) } @@ -1466,7 +1603,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { t.Fatal(err) } else if n := f1.cache.Len(); n != 1 { t.Fatalf("unexpected cache size (reopen): %d", n) - } else if a := f1.row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + } else if a := f1.mustRow(tx, 1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected columns (reopen): %+v", a) } } @@ -1486,7 +1623,9 @@ func BenchmarkFragment_Blocks(b *testing.B) { // Reset timer and execute benchmark. b.ResetTimer() for i := 0; i < b.N; i++ { - if a := f.Blocks(); len(a) == 0 { + if a, err := f.Blocks(); err != nil { + b.Fatal(err) + } else if len(a) == 0 { b.Fatal("no blocks in fragment") } } @@ -1497,14 +1636,17 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { defer f.Clean(b) f.MaxOpN = math.MaxInt32 + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Generate some intersecting data. for i := 0; i < 10000; i += 2 { - if _, err := f.setBit(1, uint64(i)); err != nil { + if _, err := f.setBit(tx, 1, uint64(i)); err != nil { b.Fatal(err) } } for i := 0; i < 10000; i += 3 { - if _, err := f.setBit(2, uint64(i)); err != nil { + if _, err := f.setBit(tx, 2, uint64(i)); err != nil { b.Fatal(err) } } @@ -1517,7 +1659,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Start benchmark b.ResetTimer() for i := 0; i < b.N; i++ { - if n := f.row(1).intersectionCount(f.row(2)); n == 0 { + if n := f.mustRow(tx, 1).intersectionCount(f.mustRow(tx, 2)); n == 0 { b.Fatalf("unexpected count: %d", n) } } @@ -1527,15 +1669,18 @@ func TestFragment_Tanimoto(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + src := NewRow(1, 2, 3) // Set bits on the rows 100, 101, & 102. - f.mustSetBits(100, 1, 3, 2, 200) - f.mustSetBits(101, 1, 3) - f.mustSetBits(102, 1, 2, 10, 12) + f.mustSetBits(tx, 100, 1, 3, 2, 200) + f.mustSetBits(tx, 101, 1, 3) + f.mustSetBits(tx, 102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.top(topOptions{TanimotoThreshold: 50, Src: src}); err != nil { + if pairs, err := f.top(tx, topOptions{TanimotoThreshold: 50, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -1550,15 +1695,18 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + src := NewRow(1, 2, 3) // Set bits on the rows 100, 101, & 102. - f.mustSetBits(100, 1, 3, 2, 200) - f.mustSetBits(101, 1, 3) - f.mustSetBits(102, 1, 2, 10, 12) + f.mustSetBits(tx, 100, 1, 3, 2, 200) + f.mustSetBits(tx, 101, 1, 3) + f.mustSetBits(tx, 102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.top(topOptions{TanimotoThreshold: 0, Src: src}); err != nil { + if pairs, err := f.top(tx, topOptions{TanimotoThreshold: 0, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 3 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -1575,9 +1723,12 @@ func TestFragment_Snapshot_Run(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set bits on the fragment. for i := uint64(1); i < 3; i++ { - if _, err := f.setBit(1000, i); err != nil { + if _, err := f.setBit(tx, 1000, i); err != nil { t.Fatal(err) } } @@ -1585,14 +1736,14 @@ func TestFragment_Snapshot_Run(t *testing.T) { // Snapshot bitmap and verify data. if err := f.Snapshot(); err != nil { t.Fatal(err) - } else if n := f.row(1000).Count(); n != 2 { + } else if n := f.mustRow(tx, 1000).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if n := f.row(1000).Count(); n != 2 { + } else if n := f.mustRow(tx, 1000).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -1602,28 +1753,31 @@ func TestFragment_SetMutex(t *testing.T) { f := mustOpenMutexFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + var cols []uint64 // Set a value on column 100. - if _, err := f.setBit(1, 100); err != nil { + if _, err := f.setBit(tx, 1, 100); err != nil { t.Fatal(err) } // Verify the value was set. - cols = f.row(1).Columns() + cols = f.mustRow(tx, 1).Columns() if !reflect.DeepEqual(cols, []uint64{100}) { t.Fatalf("mutex unexpected columns: %v", cols) } // Set a different value on column 100. - if _, err := f.setBit(2, 100); err != nil { + if _, err := f.setBit(tx, 2, 100); err != nil { t.Fatal(err) } // Verify that value (row 1) was replaced (by row 2). - cols = f.row(1).Columns() + cols = f.mustRow(tx, 1).Columns() if !reflect.DeepEqual(cols, []uint64{}) { t.Fatalf("mutex unexpected columns: %v", cols) } - cols = f.row(2).Columns() + cols = f.mustRow(tx, 2).Columns() if !reflect.DeepEqual(cols, []uint64{100}) { t.Fatalf("mutex unexpected columns: %v", cols) } @@ -1716,29 +1870,32 @@ func TestFragment_ImportSet(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set import. - err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) + err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { t.Fatalf("bulk importing ids: %v", err) } // Check for expected results. for k, v := range test.setExp { - cols := f.row(k).Columns() + cols := f.mustRow(tx, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("expected: %v, but got: %v", v, cols) } } // Clear import. - err = f.bulkImport(test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) + err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) if err != nil { t.Fatalf("bulk clearing ids: %v", err) } // Check for expected results. for k, v := range test.clearExp { - cols := f.row(k).Columns() + cols := f.mustRow(tx, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("expected: %v, but got: %v", v, cols) } @@ -1752,9 +1909,12 @@ func TestFragment_ConcurrentImport(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + eg := errgroup.Group{} - eg.Go(func() error { return f.bulkImportStandard([]uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) }) - eg.Go(func() error { return f.bulkImportStandard([]uint64{3, 4}, []uint64{3, 4}, &ImportOptions{}) }) + eg.Go(func() error { return f.bulkImportStandard(tx, []uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) }) + eg.Go(func() error { return f.bulkImportStandard(tx, []uint64{3, 4}, []uint64{3, 4}, &ImportOptions{}) }) err := eg.Wait() if err != nil { t.Fatalf("importing data to fragment: %v", err) @@ -1849,29 +2009,32 @@ func TestFragment_ImportMutex(t *testing.T) { f := mustOpenMutexFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set import. - err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) + err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { t.Fatalf("bulk importing ids: %v", err) } // Check for expected results. for k, v := range test.setExp { - cols := f.row(k).Columns() + cols := f.mustRow(tx, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("row: %d, expected: %v, but got: %v", k, v, cols) } } // Clear import. - err = f.bulkImport(test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) + err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) if err != nil { t.Fatalf("bulk clearing ids: %v", err) } // Check for expected results. for k, v := range test.clearExp { - cols := f.row(k).Columns() + cols := f.mustRow(tx, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("row: %d expected: %v, but got: %v", k, v, cols) } @@ -1968,29 +2131,32 @@ func TestFragment_ImportBool(t *testing.T) { f := mustOpenBoolFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Set import. - err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) + err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { t.Fatalf("bulk importing ids: %v", err) } // Check for expected results. for k, v := range test.setExp { - cols := f.row(k).Columns() + cols := f.mustRow(tx, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("expected: %v, but got: %v", v, cols) } } // Clear import. - err = f.bulkImport(test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) + err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) if err != nil { t.Fatalf("bulk importing ids: %v", err) } // Check for expected results. for k, v := range test.clearExp { - cols := f.row(k).Columns() + cols := f.mustRow(tx, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("expected: %v, but got: %v", v, cols) } @@ -2027,6 +2193,10 @@ func BenchmarkFragment_Snapshot(b *testing.B) { func BenchmarkFragment_FullSnapshot(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(b) + + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // Generate some intersecting data. maxX := ShardWidth / 2 sz := maxX @@ -2044,7 +2214,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { val += 2 i++ } - if err := f.bulkImport(rows, cols, options); err != nil { + if err := f.bulkImport(tx, rows, cols, options); err != nil { b.Fatalf("Error Building Sample: %s", err) } if row > max { @@ -2089,8 +2259,10 @@ func BenchmarkFragment_Import(b *testing.B) { copy(rowsUse, rows) copy(colsUse, cols) f := mustOpenFragment("i", "f", viewStandard, 0, "") + // Obtain transaction. + tx := &RoaringTx{fragment: f} b.StartTimer() - if err := f.bulkImport(rowsUse, colsUse, options); err != nil { + if err := f.bulkImport(tx, rowsUse, colsUse, options); err != nil { b.Errorf("Error Building Sample: %s", err) } b.StopTimer() @@ -2196,6 +2368,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { for i := 0; i < b.N; i++ { for j := 0; j < concurrency; j++ { frags[j] = mustOpenFragment("i", "f", viewStandard, uint64(j), cacheType) + // the cost of actually doing the op log for the large initial data set // is excessive. force storage into snapshotted state, then use import // to generate an op log and/or snapshot. @@ -2248,8 +2421,12 @@ func BenchmarkImportStandard(b *testing.B) { copy(rowIDs, rowIDsOrig) copy(columnIDs, columnIDsOrig) f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) + + // Obtain transaction. + tx := &RoaringTx{fragment: f} + b.StartTimer() - err := f.bulkImport(rowIDs, columnIDs, &ImportOptions{}) + err := f.bulkImport(tx, rowIDs, columnIDs, &ImportOptions{}) if err != nil { b.Errorf("import error: %v", err) } @@ -2275,6 +2452,7 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { b.StopTimer() for i := 0; i < b.N; i++ { f := mustOpenFragment("i", fmt.Sprintf("r%dc%dcache_%s", numRows, numCols, cacheType), viewStandard, 0, cacheType) + // the cost of actually doing the op log for the large initial data set // is excessive. force storage into snapshotted state, then use import // to generate an op log and/or snapshot. @@ -2396,10 +2574,14 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) { if err != nil { b.Fatalf("opening fragment: %v", err) } + + // Obtain transaction. + tx := &RoaringTx{fragment: nf} + copy(rows, rowsOrig) copy(cols, colsOrig) b.StartTimer() - err = nf.bulkImport(rows, cols, opts) + err = nf.bulkImport(tx, rows, cols, opts) b.StopTimer() if err != nil { b.Fatalf("bulkImport: %v", err) @@ -2446,17 +2628,23 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { func TestGetZipfRowsSliceRoaring(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, DefaultCacheType) + + // Obtain transaction. + tx := &RoaringTx{fragment: f} + data := getZipfRowsSliceRoaring(10, 1, 0, ShardWidth) err := f.importRoaringT(data, false) if err != nil { t.Fatalf("importing roaring: %v", err) } - rows := f.rows(context.Background(), 0) - if !reflect.DeepEqual(rows, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) { + rows, err := f.rows(context.Background(), tx, 0) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(rows, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) { t.Fatalf("unexpected rows: %v", rows) } for i := uint64(1); i < 10; i++ { - if f.row(i).Count() >= f.row(i-1).Count() { + if f.mustRow(tx, i).Count() >= f.mustRow(tx, i-1).Count() { t.Fatalf("suspect distribution from getZipfRowsSliceRoaring") } } @@ -2600,6 +2788,7 @@ func (f *fragment) sanityCheck(t testing.TB) { if err != nil { t.Fatalf("sanityCheck couldn't unmarshal fragment %s: %v", f.path, err) } + // Refactor fragment.storage if equal, reason := newBM.BitwiseEqual(f.storage); !equal { t.Fatalf("fragment %s: unmarshalled bitmap different: %v", f.path, reason) } @@ -2724,9 +2913,9 @@ func (f *fragment) Reopen() error { // mustSetBits sets columns on a row. Panic on error. // This function does not accept a timestamp or quantum. -func (f *fragment) mustSetBits(rowID uint64, columnIDs ...uint64) { +func (f *fragment) mustSetBits(tx Tx, rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { - if _, err := f.setBit(rowID, columnID); err != nil { + if _, err := f.setBit(tx, rowID, columnID); err != nil { panic(err) } } @@ -2745,11 +2934,12 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("firstContainer", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + tx := &RoaringTx{fragment: f} expectedAll := make([]uint64, 0) expectedOdd := make([]uint64, 0) for i := uint64(100); i < uint64(200); i++ { - if _, err := f.setBit(i, i%2); err != nil { + if _, err := f.setBit(tx, i, i%2); err != nil { t.Fatal(err) } expectedAll = append(expectedAll, i) @@ -2758,13 +2948,17 @@ func TestFragment_RowsIteration(t *testing.T) { } } - ids := f.rows(context.Background(), 0) - if !reflect.DeepEqual(expectedAll, ids) { + ids, err := f.rows(context.Background(), tx, 0) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(expectedAll, ids) { t.Fatalf("Do not match %v %v", expectedAll, ids) } - ids = f.rows(context.Background(), 0, filterColumn(1)) - if !reflect.DeepEqual(expectedOdd, ids) { + ids, err = f.rows(context.Background(), tx, 0, filterColumn(1)) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(expectedOdd, ids) { t.Fatalf("Do not match %v %v", expectedOdd, ids) } }) @@ -2772,23 +2966,28 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("secondRow", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + tx := &RoaringTx{fragment: f} expected := []uint64{1, 2} - if _, err := f.setBit(1, 66000); err != nil { + if _, err := f.setBit(tx, 1, 66000); err != nil { t.Fatal(err) - } else if _, err := f.setBit(2, 66000); err != nil { + } else if _, err := f.setBit(tx, 2, 66000); err != nil { t.Fatal(err) - } else if _, err := f.setBit(2, 166000); err != nil { + } else if _, err := f.setBit(tx, 2, 166000); err != nil { t.Fatal(err) } - ids := f.rows(context.Background(), 0) - if !reflect.DeepEqual(expected, ids) { + ids, err := f.rows(context.Background(), tx, 0) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(expected, ids) { t.Fatalf("Do not match %v %v", expected, ids) } - ids = f.rows(context.Background(), 0, filterColumn(66000)) - if !reflect.DeepEqual(expected, ids) { + ids, err = f.rows(context.Background(), tx, 0, filterColumn(66000)) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(expected, ids) { t.Fatalf("Do not match %v %v", expected, ids) } }) @@ -2796,21 +2995,26 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("combinations", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + tx := &RoaringTx{fragment: f} expectedRows := make([]uint64, 0) for r := uint64(1); r < uint64(10000); r += 250 { expectedRows = append(expectedRows, r) for c := uint64(1); c < uint64(ShardWidth-1); c += (ShardWidth >> 5) { - if _, err := f.setBit(r, c); err != nil { + if _, err := f.setBit(tx, r, c); err != nil { t.Fatal(err) } - ids := f.rows(context.Background(), 0) - if !reflect.DeepEqual(expectedRows, ids) { + ids, err := f.rows(context.Background(), tx, 0) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(expectedRows, ids) { t.Fatalf("Do not match %v %v", expectedRows, ids) } - ids = f.rows(context.Background(), 0, filterColumn(c)) - if !reflect.DeepEqual(expectedRows, ids) { + ids, err = f.rows(context.Background(), tx, 0, filterColumn(c)) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(expectedRows, ids) { t.Fatalf("Do not match %v %v", expectedRows, ids) } } @@ -2843,6 +3047,8 @@ func TestFragment_RoaringImport(t *testing.T) { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) + tx := &RoaringTx{fragment: f} + for num, input := range test { buf := &bytes.Buffer{} bm := roaring.NewBitmap(input...) @@ -2856,7 +3062,7 @@ func TestFragment_RoaringImport(t *testing.T) { } exp := calcExpected(test[:num+1]...) for row, expCols := range exp { - cols := f.row(uint64(row)).Columns() + cols := f.mustRow(tx, uint64(row)).Columns() t.Logf("\nrow: %d\n exp:%v\n got:%v", row, expCols, cols) if !reflect.DeepEqual(cols, expCols) { t.Fatalf("input%d, row %d\n exp:%v\n got:%v", num, row, expCols, cols) @@ -2889,14 +3095,15 @@ func TestFragment_RoaringImportTopN(t *testing.T) { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Clean(t) + tx := &RoaringTx{fragment: f} options := &ImportOptions{} - err := f.bulkImport(test.rowIDs, test.colIDs, options) + err := f.bulkImport(tx, test.rowIDs, test.colIDs, options) if err != nil { t.Fatalf("bulk importing ids: %v", err) } expPairs := calcTop(test.rowIDs, test.colIDs) - pairs, err := f.top(topOptions{}) + pairs, err := f.top(tx, topOptions{}) if err != nil { t.Fatalf("executing top after bulk import: %v", err) } @@ -2904,14 +3111,14 @@ func TestFragment_RoaringImportTopN(t *testing.T) { t.Fatalf("post bulk import:\n exp: %v\n got: %v\n", expPairs, pairs) } - err = f.bulkImport(test.rowIDs2, test.colIDs2, options) + err = f.bulkImport(tx, test.rowIDs2, test.colIDs2, options) if err != nil { t.Fatalf("bulk importing ids: %v", err) } test.rowIDs = append(test.rowIDs, test.rowIDs2...) test.colIDs = append(test.colIDs, test.colIDs2...) expPairs = calcTop(test.rowIDs, test.colIDs) - pairs, err = f.top(topOptions{}) + pairs, err = f.top(tx, topOptions{}) if err != nil { t.Fatalf("executing top after bulk import: %v", err) } @@ -2931,7 +3138,7 @@ func TestFragment_RoaringImportTopN(t *testing.T) { } rows, cols := toRowsCols(test.roaring) expPairs = calcTop(append(test.rowIDs, rows...), append(test.colIDs, cols...)) - pairs, err = f.top(topOptions{}) + pairs, err = f.top(tx, topOptions{}) if err != nil { t.Fatalf("executing top after roaring import: %v", err) } @@ -3026,14 +3233,22 @@ func TestFragmentRowIterator(t *testing.T) { t.Run("basic", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) defer f.Clean(t) - f.mustSetBits(0, 0) - f.mustSetBits(1, 0) - f.mustSetBits(2, 0) - f.mustSetBits(3, 0) + tx := &RoaringTx{fragment: f} - iter := f.rowIterator(false) + f.mustSetBits(tx, 0, 0) + f.mustSetBits(tx, 1, 0) + f.mustSetBits(tx, 2, 0) + f.mustSetBits(tx, 3, 0) + + iter, err := f.rowIterator(tx, false) + if err != nil { + t.Fatal(err) + } for i := uint64(0); i < 4; i++ { - row, id, _, wrapped := iter.Next() + row, id, _, wrapped, err := iter.Next() + if err != nil { + t.Fatal(err) + } if id != i { t.Fatalf("expected row %d but got %d", i, id) } @@ -3044,7 +3259,10 @@ func TestFragmentRowIterator(t *testing.T) { t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) } } - row, id, _, wrapped := iter.Next() + row, id, _, wrapped, err := iter.Next() + if err != nil { + t.Fatal(err) + } if row != nil { t.Fatalf("row should be nil after iterator is exhausted, got %v", row.Columns()) } @@ -3059,14 +3277,22 @@ func TestFragmentRowIterator(t *testing.T) { t.Run("skipped rows", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) defer f.Clean(t) - f.mustSetBits(1, 0) - f.mustSetBits(3, 0) - f.mustSetBits(5, 0) - f.mustSetBits(7, 0) + tx := &RoaringTx{fragment: f} - iter := f.rowIterator(false) + f.mustSetBits(tx, 1, 0) + f.mustSetBits(tx, 3, 0) + f.mustSetBits(tx, 5, 0) + f.mustSetBits(tx, 7, 0) + + iter, err := f.rowIterator(tx, false) + if err != nil { + t.Fatal(err) + } for i := uint64(1); i < 8; i += 2 { - row, id, _, wrapped := iter.Next() + row, id, _, wrapped, err := iter.Next() + if err != nil { + t.Fatal(err) + } if id != i { t.Fatalf("expected row %d but got %d", i, id) } @@ -3077,7 +3303,10 @@ func TestFragmentRowIterator(t *testing.T) { t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) } } - row, id, _, wrapped := iter.Next() + row, id, _, wrapped, err := iter.Next() + if err != nil { + t.Fatal(err) + } if row != nil { t.Fatalf("row should be nil after iterator is exhausted, got %v", row.Columns()) } @@ -3092,14 +3321,22 @@ func TestFragmentRowIterator(t *testing.T) { t.Run("basic wrapped", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) defer f.Clean(t) - f.mustSetBits(0, 0) - f.mustSetBits(1, 0) - f.mustSetBits(2, 0) - f.mustSetBits(3, 0) + tx := &RoaringTx{fragment: f} - iter := f.rowIterator(true) + f.mustSetBits(tx, 0, 0) + f.mustSetBits(tx, 1, 0) + f.mustSetBits(tx, 2, 0) + f.mustSetBits(tx, 3, 0) + + iter, err := f.rowIterator(tx, true) + if err != nil { + t.Fatal(err) + } for i := uint64(0); i < 5; i++ { - row, id, _, wrapped := iter.Next() + row, id, _, wrapped, err := iter.Next() + if err != nil { + t.Fatal(err) + } if id != i%4 { t.Fatalf("expected row %d but got %d", i%4, id) } @@ -3117,14 +3354,22 @@ func TestFragmentRowIterator(t *testing.T) { t.Run("skipped rows wrapped", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) defer f.Clean(t) - f.mustSetBits(1, 0) - f.mustSetBits(3, 0) - f.mustSetBits(5, 0) - f.mustSetBits(7, 0) + tx := &RoaringTx{fragment: f} - iter := f.rowIterator(true) + f.mustSetBits(tx, 1, 0) + f.mustSetBits(tx, 3, 0) + f.mustSetBits(tx, 5, 0) + f.mustSetBits(tx, 7, 0) + + iter, err := f.rowIterator(tx, true) + if err != nil { + t.Fatal(err) + } for i := uint64(1); i < 10; i += 2 { - row, id, _, wrapped := iter.Next() + row, id, _, wrapped, err := iter.Next() + if err != nil { + t.Fatal(err) + } if id != i%8 { t.Errorf("expected row %d but got %d", i%8, id) } @@ -3146,6 +3391,7 @@ func TestUnionInPlaceMapped(t *testing.T) { // the lock *not* held, because it is sometimes so it has to grab the // lock... defer f.Clean(t) + f.mu.Lock() defer f.mu.Unlock() r0 := rand.New(rand.NewSource(2)) @@ -3177,6 +3423,7 @@ func TestUnionInPlaceMapped(t *testing.T) { f.storage.UnionInPlace(setBM1) countUnion := f.storage.Count() + // UnionInPlace produces no ops log, we have to make it snapshot, to // ensure that the on-disk representation is correct. Note, UIP is // not used for things that are modifying real fragments, usually; @@ -3299,12 +3546,15 @@ func TestIntLTRegression(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeNone) defer f.Clean(t) - _, err := f.setValue(1, 6, 33) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + + _, err := f.setValue(tx, 1, 6, 33) if err != nil { t.Fatalf("setting value: %v", err) } - row, err := f.rangeOp(pql.LT, 6, 33) + row, err := f.rangeOp(tx, pql.LT, 6, 33) if err != nil { t.Fatalf("doing range of: %v", err) } @@ -3363,7 +3613,11 @@ func TestImportClearRestart(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") f.MaxOpN = maxOpN - err := f.bulkImport(testrows, testcols, &ImportOptions{}) + + // Obtain transaction. + tx := &RoaringTx{fragment: f} + + err := f.bulkImport(tx, testrows, testcols, &ImportOptions{}) if err != nil { t.Fatalf("initial small import: %v", err) } @@ -3410,7 +3664,7 @@ func TestImportClearRestart(t *testing.T) { copy(testrows, test.rows) copy(testcols, test.cols) - err = f2.bulkImport(testrows, testcols, &ImportOptions{Clear: true}) + err = f2.bulkImport(tx, testrows, testcols, &ImportOptions{Clear: true}) if err != nil { t.Fatalf("clearing imported data: %v", err) } @@ -3446,8 +3700,10 @@ func TestImportClearRestart(t *testing.T) { } func check(t *testing.T, f *fragment, exp map[uint64]map[uint64]struct{}) { + tx := &RoaringTx{fragment: f} + for rowID, colsExp := range exp { - colsAct := f.row(rowID).Columns() + colsAct := f.mustRow(tx, rowID).Columns() if len(colsAct) != len(colsExp) { t.Errorf("row %d len mismatch got: %d exp:%d", rowID, len(colsAct), len(colsExp)) } @@ -3479,8 +3735,9 @@ func TestImportValueConcurrent(t *testing.T) { for i := 0; i < 4; i++ { i := i eg.Go(func() error { + tx := &RoaringTx{fragment: f} for j := uint64(0); j < 10; j++ { - err := f.importValue([]uint64{j}, []int64{int64(rand.Int63n(1000))}, 10, i%2 == 0) + err := f.importValue(tx, []uint64{j}, []int64{int64(rand.Int63n(1000))}, 10, i%2 == 0) if err != nil { return err } @@ -3517,14 +3774,18 @@ func TestImportMultipleValues(t *testing.T) { f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) f.MaxOpN = maxOpN defer f.Clean(t) - err := f.importValue(test.cols, test.vals, test.depth, false) + + // Obtain transaction. + tx := &RoaringTx{fragment: f} + + err := f.importValue(tx, test.cols, test.vals, test.depth, false) if err != nil { t.Fatalf("importing values: %v", err) } for i := range test.checkCols { cc, cv := test.checkCols[i], test.checkVals[i] - n, exists, err := f.value(cc, test.depth) + n, exists, err := f.value(tx, cc, test.depth) if err != nil { t.Fatalf("getting value: %v", err) } @@ -3575,23 +3836,26 @@ func TestImportValueRowCache(t *testing.T) { f.MaxOpN = maxOpN defer f.Clean(t) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // First import (tc1) - if err := f.importValue(test.tc1.cols, test.tc1.vals, test.tc1.depth, false); err != nil { + if err := f.importValue(tx, test.tc1.cols, test.tc1.vals, test.tc1.depth, false); err != nil { t.Fatalf("importing values: %v", err) } - if r, err := f.rangeOp(pql.GT, test.tc1.depth, 0); err != nil { + if r, err := f.rangeOp(tx, pql.GT, test.tc1.depth, 0); err != nil { t.Error("getting range of values") } else if !reflect.DeepEqual(r.Columns(), test.tc1.checkCols) { t.Errorf("wrong column values. expected: %v, but got: %v", test.tc1.checkCols, r.Columns()) } // Second import (tc2) - if err := f.importValue(test.tc2.cols, test.tc2.vals, test.tc2.depth, false); err != nil { + if err := f.importValue(tx, test.tc2.cols, test.tc2.vals, test.tc2.depth, false); err != nil { t.Fatalf("importing values: %v", err) } - if r, err := f.rangeOp(pql.GT, test.tc2.depth, 0); err != nil { + if r, err := f.rangeOp(tx, pql.GT, test.tc2.depth, 0); err != nil { t.Error("getting range of values") } else if !reflect.DeepEqual(r.Columns(), test.tc2.checkCols) { t.Errorf("wrong column values. expected: %v, but got: %v", test.tc2.checkCols, r.Columns()) @@ -3607,8 +3871,11 @@ func TestFragmentConcurrentReadWrite(t *testing.T) { eg := &errgroup.Group{} eg.Go(func() error { + // Obtain transaction. + tx := &RoaringTx{fragment: f} + for i := uint64(0); i < 1000; i++ { - _, err := f.setBit(i%4, i) + _, err := f.setBit(tx, i%4, i) if err != nil { return errors.Wrap(err, "setting bit") } @@ -3616,9 +3883,12 @@ func TestFragmentConcurrentReadWrite(t *testing.T) { return nil }) + // Obtain transaction. + tx := &RoaringTx{fragment: f} + acc := uint64(0) for i := uint64(0); i < 100; i++ { - r := f.row(i % 4) + r := f.mustRow(tx, i%4) acc += r.Count() } if err := eg.Wait(); err != nil { @@ -3645,6 +3915,10 @@ func TestRemapCache(t *testing.T) { t.Fatalf("unexpected panic: %v", r) } }() + + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // create a container _, err := f.storage.Add(65537) if err != nil { @@ -3656,7 +3930,7 @@ func TestRemapCache(t *testing.T) { t.Fatalf("storage snapshot: %v", err) } // freeze the row - _ = f.row(0) + _ = f.mustRow(tx, 0) // add a bit that isn't in that container, so that container doesn't // change _, err = f.storage.Add(2) @@ -3664,7 +3938,7 @@ func TestRemapCache(t *testing.T) { t.Fatalf("storage add: %v", err) } // make the original container be the most recent, thus cached, container - _, err = f.bit(0, 65537) + _, err = f.bit(tx, 0, 65537) if err != nil { t.Fatalf("storage bit check: %v", err) } @@ -3676,7 +3950,7 @@ func TestRemapCache(t *testing.T) { // get rid of the old mapping runtime.GC() // try to read that container again - _, err = f.bit(0, 65537) + _, err = f.bit(tx, 0, 65537) if err != nil { t.Fatalf("storage bit check: %v", err) } @@ -3685,6 +3959,9 @@ func TestRemapCache(t *testing.T) { func TestFragment_Bug_Q2DoubleDelete(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") + // Obtain transaction. + tx := &RoaringTx{fragment: f} + // byShardWidth is a map of the same roaring (fragment) data generated // with different shard widths. // TODO: a better approach may be to generate this in the test based @@ -3705,17 +3982,17 @@ func TestFragment_Bug_Q2DoubleDelete(t *testing.T) { t.Fatalf("importing roaring: %v", err) } //check the bit - res := f.row(1).Columns() - if len(res) < 1 || f.row(1).Columns()[0] != 1 { + res := f.mustRow(tx, 1).Columns() + if len(res) < 1 || f.mustRow(tx, 1).Columns()[0] != 1 { t.Fatalf("expecting 1 got: %v", res) } //clear the bit - changed, _ := f.clearBit(1, 1) + changed, _ := f.clearBit(tx, 1, 1) if !changed { t.Fatalf("expected change got %v", changed) } //check missing - res = f.row(1).Columns() + res = f.mustRow(tx, 1).Columns() if len(res) != 0 { t.Fatalf("expected nothing got %v", res) } @@ -3725,16 +4002,16 @@ func TestFragment_Bug_Q2DoubleDelete(t *testing.T) { t.Fatalf("importing roaring: %v", err) } //check - res = f.row(1).Columns() - if len(res) < 1 || f.row(1).Columns()[0] != 1 { + res = f.mustRow(tx, 1).Columns() + if len(res) < 1 || f.mustRow(tx, 1).Columns()[0] != 1 { t.Fatalf("again expecting 1 got: %v", res) } - changed, _ = f.clearBit(1, 1) + changed, _ = f.clearBit(tx, 1, 1) if !changed { t.Fatalf("again expected change got %v", changed) } //check missing - res = f.row(1).Columns() + res = f.mustRow(tx, 1).Columns() if len(res) != 0 { t.Fatalf("expected nothing got %v", res) } diff --git a/go.mod b/go.mod index 9d9ecd366..485e2a5be 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,10 @@ require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/benbjohnson/immutable v0.2.0 github.com/boltdb/bolt v1.3.1 github.com/cespare/xxhash v1.1.0 + github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/davecgh/go-spew v1.1.1 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 9da5feace..997603c23 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/benbjohnson/immutable v0.2.0 h1:t0rW3lNFwfQ85IDO1mhMbumxdVSti4nnVaal4r45Oio= +github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -22,6 +24,8 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= @@ -112,6 +116,7 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -242,3 +247,4 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130= diff --git a/holder.go b/holder.go index d1f457371..30fb7ede0 100644 --- a/holder.go +++ b/holder.go @@ -613,6 +613,11 @@ func (h *Holder) Close() error { return nil } +// Begin starts a transaction on the holder. +func (h *Holder) Begin(writable bool) (Tx, error) { + return NewMultiTx(writable, h), nil +} + // HasData returns true if Holder contains at least one index. // This is used to determine if the rebalancing of data is necessary // when a node joins the cluster. diff --git a/holder_internal_test.go b/holder_internal_test.go index 5ae365362..1a229858a 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -83,6 +83,12 @@ func makeHolder() (*Holder, string, error) { } func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) { + tx, err := h.Begin(true) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + idx, err := h.CreateIndexIfNotExists(index, IndexOptions{}) if err != nil { t.Fatalf("creating index: %v", err) @@ -91,10 +97,13 @@ func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID ui if err != nil { t.Fatalf("setting bit: %v", err) } - _, err = f.SetBit(rowID, columnID, nil) + _, err = f.SetBit(tx, rowID, columnID, nil) if err != nil { t.Fatalf("setting bit: %v", err) } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } } func TestHolderOperatorProcess(t *testing.T) { diff --git a/holder_test.go b/holder_test.go index 42c010079..45df98852 100644 --- a/holder_test.go +++ b/holder_test.go @@ -162,11 +162,19 @@ func TestHolder_Open(t *testing.T) { h := test.MustOpenHolder() defer h.Close() + tx, err := h.Begin(true) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) } else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := field.SetBit(0, 0, nil); err != nil { + } else if _, err := field.SetBit(tx, 0, 0, nil); err != nil { + t.Fatal(err) + } else if err := tx.Commit(); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -184,11 +192,19 @@ func TestHolder_Open(t *testing.T) { h := test.MustOpenHolder() defer h.Close() + tx, err := h.Begin(true) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) } else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := field.SetBit(0, 0, nil); err != nil { + } else if _, err := field.SetBit(tx, 0, 0, nil); err != nil { + t.Fatal(err) + } else if err := tx.Commit(); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -204,11 +220,19 @@ func TestHolder_Open(t *testing.T) { h := test.MustOpenHolder() defer h.Close() + tx, err := h.Begin(true) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) } else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := field.SetBit(0, 0, nil); err != nil { + } else if _, err := field.SetBit(tx, 0, 0, nil); err != nil { + t.Fatal(err) + } else if err := tx.Commit(); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) diff --git a/index.go b/index.go index 4a7ee3f5f..86c0ca94f 100644 --- a/index.go +++ b/index.go @@ -362,6 +362,12 @@ func (i *Index) AvailableShards() *roaring.Bitmap { return b } +// Begin starts a transaction on a shard of the index. +func (i *Index) Begin(writable bool, shard uint64) (Tx, error) { + // TODO(bbj): Check for underlying storage as RBF or roaring. + return &RoaringTx{Index: i}, nil +} + // fieldPath returns the path to a field in the index. func (i *Index) fieldPath(name string) string { return filepath.Join(i.path, name) } diff --git a/mmap_test.go b/mmap_test.go index 5ce157bc4..a41830fc5 100644 --- a/mmap_test.go +++ b/mmap_test.go @@ -35,8 +35,10 @@ func forceSnapshotsCheckMapping(t *testing.T) { f.Logger = logger.NewLogfLogger(t) defer f.Clean(t) + tx := &RoaringTx{fragment: f} + for i := 0; i < f.MaxOpN; i++ { - _, _ = f.setBit(0, uint64(i*32)) + _, _ = f.setBit(tx, 0, uint64(32*i)) } // force snapshot so we get a mmapped row... err := f.Snapshot() @@ -67,7 +69,7 @@ func forceSnapshotsCheckMapping(t *testing.T) { if i%5 == 0 { runtime.GC() } - err := f.importValue(cv.cols, cv.vals, depth, (i%3 == 1)) + err := f.importValue(tx, cv.cols, cv.vals, depth, (i%3 == 1)) if err != nil { t.Fatalf("importValue[%d]: %v", i, err) } diff --git a/roaring/btree_test.go b/roaring/btree_test.go index f0f528a7b..65f05d0c4 100644 --- a/roaring/btree_test.go +++ b/roaring/btree_test.go @@ -399,6 +399,7 @@ func BenchmarkBtreeSetSeq1e6(b *testing.B) { func benchmarkSetSeq(b *testing.B, n int) { b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { b.StopTimer() r := treeNew() @@ -436,6 +437,7 @@ func benchmarkGetSeq(b *testing.B, n int) { } debug.FreeOSMemory() b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { for j := 0; j < n; j++ { r.Get(uint64(j)) @@ -468,6 +470,7 @@ func benchmarkSetRnd(b *testing.B, n int) { a[i] = rng.Next() } b.ResetTimer() + b.ReportAllocs() c := getDummyC(1) for i := 0; i < b.N; i++ { b.StopTimer() @@ -512,6 +515,7 @@ func benchmarkGetRnd(b *testing.B, n int) { } debug.FreeOSMemory() b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { for _, v := range a { r.Get(uint64(v)) diff --git a/roaring/container_stash.go b/roaring/container_stash.go index 0f616f0f2..3531a0317 100644 --- a/roaring/container_stash.go +++ b/roaring/container_stash.go @@ -165,29 +165,29 @@ func NewContainerArrayN(set []uint16, n int32) *Container { // NewContainerRun creates a new run container using a provided (possibly nil) // slice of intervals. -func NewContainerRun(set []interval16) *Container { +func NewContainerRun(set []Interval16) *Container { c := &Container{typeID: containerRun} c.setRuns(set) for _, run := range set { - c.n += int32(run.last-run.start) + 1 + c.n += int32(run.Last-run.Start) + 1 } return c } // NewContainerRunCopy creates a new run container using a provided (possibly nil) // slice of intervals. It copies the provided slice to new storage. -func NewContainerRunCopy(set []interval16) *Container { +func NewContainerRunCopy(set []Interval16) *Container { c := &Container{typeID: containerRun} c.setRunsMaybeCopy(set, true) for _, run := range set { - c.n += int32(run.last-run.start) + 1 + c.n += int32(run.Last-run.Start) + 1 } return c } // NewContainerRunN creates a new run array using a provided (possibly nil) // slice of intervals. It overrides n using the provided value. -func NewContainerRunN(set []interval16, n int32) *Container { +func NewContainerRunN(set []Interval16, n int32) *Container { c := &Container{typeID: containerRun, n: n} c.setRuns(set) return c @@ -426,27 +426,33 @@ var fillerBitmap = func() (a [1024]uint64) { return a }() -func splatRun(into *[1024]uint64, from interval16) { +func splatRun(into *[1024]uint64, from Interval16) { + // TODO this can be ~64x faster for long runs by setting maxBitmap instead of single bits + // note v must be int or will overflow + // for v := int(from.Start); v <= int(from.Last); v++ { + // into[v/64] |= (uint64(1) << uint(v%64)) + // } + // Handle the case where the start and end fall within the same word. - if from.start/64 == from.last/64 { - highMask := ^uint64(0) >> (63 - (from.last % 64)) - lowMask := ^uint64(0) << (from.start % 64) - into[from.start/64] |= highMask & lowMask + if from.Start/64 == from.Last/64 { + highMask := ^uint64(0) >> (63 - (from.Last % 64)) + lowMask := ^uint64(0) << (from.Start % 64) + into[from.Start/64] |= highMask & lowMask return } // Calculate preliminary bulk fill bounds. - fillStart, fillEnd := from.start/64, from.last/64 + fillStart, fillEnd := from.Start/64, from.Last/64 // Handle run start. - if from.start%64 != 0 { - into[from.start/64] |= ^uint64(0) << (from.start % 64) + if from.Start%64 != 0 { + into[from.Start/64] |= ^uint64(0) << (from.Start % 64) fillStart++ } // Handle run end. - if from.last%64 != 63 { - into[from.last/64] |= ^uint64(0) >> (63 - (from.last % 64)) + if from.Last%64 != 63 { + into[from.Last/64] |= ^uint64(0) >> (63 - (from.Last % 64)) fillEnd-- } @@ -480,7 +486,7 @@ func (c *Container) setBitmap(bitmap []uint64) { } // runs yields the data viewed as a slice of intervals. -func (c *Container) runs() []interval16 { +func (c *Container) runs() []Interval16 { if c == nil { panic("attempt to read nil container's runs") } @@ -489,17 +495,17 @@ func (c *Container) runs() []interval16 { panic("attempt to read non-run's runs") } } - return (*[1 << 15]interval16)(unsafe.Pointer(c.pointer))[:c.len:c.cap] + return (*[1 << 15]Interval16)(unsafe.Pointer(c.pointer))[:c.len:c.cap] } // setRuns stores a set of intervals as data. c must not be frozen. -func (c *Container) setRuns(runs []interval16) { +func (c *Container) setRuns(runs []Interval16) { c.setRunsMaybeCopy(runs, false) } // setRunsMaybeCopy stores a set of intervals as data. c must not be frozen. // If doCopy is set, the values will be copied to different storage. -func (c *Container) setRunsMaybeCopy(runs []interval16, doCopy bool) { +func (c *Container) setRunsMaybeCopy(runs []Interval16, doCopy bool) { if roaringParanoia { if c == nil || c.frozen() { panic("setRuns on nil or frozen container") @@ -514,24 +520,24 @@ func (c *Container) setRunsMaybeCopy(runs []interval16, doCopy bool) { c.flags &^= flagPristine // array we can fit in data store: if len(runs) <= stashedRunSize { - newRuns := (*[stashedRunSize]interval16)(unsafe.Pointer(&c.data))[:len(runs)] + newRuns := (*[stashedRunSize]Interval16)(unsafe.Pointer(&c.data))[:len(runs)] copy(newRuns, runs) c.pointer, c.len, c.cap = &c.data[0], int32(len(newRuns)), int32(cap(newRuns)) c.flags &^= flagMapped // this is no longer using a hypothetical mmapped input array return } - if &runs[0].start == c.pointer && !doCopy { + if &runs[0].Start == c.pointer && !doCopy { // nothing to do but update length c.len = int32(len(runs)) return } if doCopy { - runs = append([]interval16(nil), runs...) + runs = append([]Interval16(nil), runs...) } if cap(runs) > 1<<15 { runs = runs[: len(runs) : 1<<15] } - c.pointer, c.len, c.cap = &runs[0].start, int32(len(runs)), int32(cap(runs)) + c.pointer, c.len, c.cap = &runs[0].Start, int32(len(runs)), int32(cap(runs)) } // UpdateOrMake updates the container, yielding a new container if necessary. diff --git a/roaring/containers_test.go b/roaring/containers_test.go index 97f79149c..8e66854d0 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -187,18 +187,18 @@ func TestSliceContainers(t *testing.T) { }) } -func genRun(r *rand.Rand) interval16 { +func genRun(r *rand.Rand) Interval16 { gen: dat := r.Uint32() start, end := uint16(dat>>16), uint16(dat) if start > end { goto gen } - return interval16{start, end} + return Interval16{start, end} } -func splatRunNaive(into []uint64, from interval16) { - for v := int(from.start); v <= int(from.last); v++ { +func splatRunNaive(into []uint64, from Interval16) { + for v := int(from.Start); v <= int(from.Last); v++ { into[v/64] |= (uint64(1) << uint(v%64)) } } @@ -212,21 +212,21 @@ func TestSplat(t *testing.T) { splatRunNaive(a[:], run) splatRun(&b, run) if a != b { - t.Errorf("incorrect splat of run [%d, %d]", run.start, run.last) + t.Errorf("incorrect splat of run [%d, %d]", run.Start, run.Last) } } } -func benchSplat(b *testing.B, run interval16) { +func benchSplat(b *testing.B, run Interval16) { var buf [1024]uint64 for i := 0; i < b.N; i++ { splatRun(&buf, run) } } -func BenchmarkSplatSingle(b *testing.B) { benchSplat(b, interval16{42, 42}) } -func BenchmarkSplatPartword(b *testing.B) { benchSplat(b, interval16{16, 31}) } -func BenchmarkSplatWord(b *testing.B) { benchSplat(b, interval16{16, 31}) } -func BenchmarkSplatEdges(b *testing.B) { benchSplat(b, interval16{15, 16}) } -func BenchmarkSplatMedium(b *testing.B) { benchSplat(b, interval16{13, 65}) } -func BenchmarkSplatAll(b *testing.B) { benchSplat(b, interval16{0, ^uint16(0)}) } +func BenchmarkSplatSingle(b *testing.B) { benchSplat(b, Interval16{42, 42}) } +func BenchmarkSplatPartword(b *testing.B) { benchSplat(b, Interval16{16, 31}) } +func BenchmarkSplatWord(b *testing.B) { benchSplat(b, Interval16{16, 31}) } +func BenchmarkSplatEdges(b *testing.B) { benchSplat(b, Interval16{15, 16}) } +func BenchmarkSplatMedium(b *testing.B) { benchSplat(b, Interval16{13, 65}) } +func BenchmarkSplatAll(b *testing.B) { benchSplat(b, Interval16{0, ^uint16(0)}) } diff --git a/roaring/roaring.go b/roaring/roaring.go index 0bd4da63a..fc75ef110 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -54,7 +54,7 @@ const ( // bitmapN is the number of values in a container.bitmap. bitmapN = (1 << 16) / 64 - maxContainerVal = 0xffff + MaxContainerVal = 0xffff // maxContainerKey is the key representing the last container in a full row. // It is the full bitmap space (2^64) divided by container width (2^16). @@ -75,7 +75,7 @@ var containerTypeNames = map[byte]string{ containerRun: "run", } -var fullContainer = NewContainerRun([]interval16{{start: 0, last: maxContainerVal}}).Freeze() +var fullContainer = NewContainerRun([]Interval16{{Start: 0, Last: MaxContainerVal}}).Freeze() // AdvisoryError is used for the special case where we probably want to *report* // an error reading a file, but don't want to actually count the file as not @@ -524,7 +524,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) { break } if k == skey { - n += uint64(c.countRange(int32(lowbits(start)), maxContainerVal+1)) + n += uint64(c.countRange(int32(lowbits(start)), MaxContainerVal+1)) continue } if k < ekey { @@ -568,21 +568,27 @@ func (b *Bitmap) SliceRange(start, end uint64) []uint64 { } // ForEach executes fn for each value in the bitmap. -func (b *Bitmap) ForEach(fn func(uint64)) { +func (b *Bitmap) ForEach(fn func(uint64) error) error { itr := b.Iterator() itr.Seek(0) for v, eof := itr.Next(); !eof; v, eof = itr.Next() { - fn(v) + if err := fn(v); err != nil { + return err + } } + return nil } // ForEachRange executes fn for each value in the bitmap between [start, end). -func (b *Bitmap) ForEachRange(start, end uint64, fn func(uint64)) { +func (b *Bitmap) ForEachRange(start, end uint64, fn func(uint64) error) error { itr := b.Iterator() itr.Seek(start) for v, eof := itr.Next(); !eof && v < end; v, eof = itr.Next() { - fn(v) + if err := fn(v); err != nil { + return err + } } + return nil } // OffsetRange returns a new bitmap with a containers offset by start. @@ -823,7 +829,7 @@ func (c *Container) intersectInPlace(other *Container) *Container { c = nil return c } - cFull, otherFull := (c.N() == maxContainerVal+1), (other.N() == maxContainerVal+1) + cFull, otherFull := (c.N() == MaxContainerVal+1), (other.N() == MaxContainerVal+1) if cFull && otherFull { return c } @@ -927,9 +933,9 @@ func intersectArrayRunInPlace(a, b *Container) *Container { n := 0 for i, j := 0, 0; i < an && j < bn; { va, vb := aa[i], br[j] - if va < vb.start { + if va < vb.Start { i++ - } else if va > vb.last { + } else if va > vb.Last { j++ } else { aa[n] = va @@ -1033,26 +1039,26 @@ func intersectBitmapRunInPlace(a, b *Container) *Container { n := int32(0) for _, vb := range br { - i := vb.start >> 6 // index into a + i := vb.Start >> 6 // index into a vastart := i << 6 valast := vastart + 63 - for valast >= vb.start && vastart <= vb.last && int(i) < an { - if vastart >= vb.start && valast <= vb.last { // a within b + for valast >= vb.Start && vastart <= vb.Last && int(i) < an { + if vastart >= vb.Start && valast <= vb.Last { // a within b bitmap[i] = ab[i] n += int32(popcount(ab[i])) - } else if vb.start >= vastart && vb.last <= valast { // b within a - var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart) + } else if vb.Start >= vastart && vb.Last <= valast { // b within a + var mask uint64 = ((1 << (vb.Last - vb.Start + 1)) - 1) << (vb.Start - vastart) bits := ab[i] & mask bitmap[i] |= bits n += int32(popcount(bits)) - } else if vastart < vb.start { // a overlaps front of b - offset := 64 - (1 + valast - vb.start) + } else if vastart < vb.Start { // a overlaps front of b + offset := 64 - (1 + valast - vb.Start) bits := (ab[i] >> offset) << offset bitmap[i] |= bits n += int32(popcount(bits)) - } else if vb.start < vastart { // b overlaps front of a - offset := 64 - (1 + vb.last - vastart) + } else if vb.Start < vastart { // b overlaps front of a + offset := 64 - (1 + vb.Last - vastart) bits := (ab[i] << offset) >> offset bitmap[i] |= bits n += int32(popcount(bits)) @@ -1076,42 +1082,42 @@ func intersectRunRunInPlace(a, b *Container) *Container { ar, br := a.runs(), b.runs() an, bn := len(ar), len(br) - var runs []interval16 + var runs []Interval16 if an > bn { - runs = make([]interval16, 0, an) + runs = make([]Interval16, 0, an) } else { - runs = make([]interval16, 0, bn) + runs = make([]Interval16, 0, bn) } n := int32(0) for i, j := 0, 0; i < an && j < bn; { va, vb := ar[i], br[j] - if va.last < vb.start { + if va.Last < vb.Start { // |--va--| |--vb--| i++ - } else if vb.last < va.start { + } else if vb.Last < va.Start { // |--vb--| |--va--| j++ - } else if va.last > vb.last && va.start >= vb.start { + } else if va.Last > vb.Last && va.Start >= vb.Start { // |--vb-|-|-va--| - runs = append(runs, interval16{start: va.start, last: vb.last}) - n += int32(vb.last-va.start) + 1 + runs = append(runs, Interval16{Start: va.Start, Last: vb.Last}) + n += int32(vb.Last-va.Start) + 1 j++ - } else if va.last > vb.last && va.start < vb.start { + } else if va.Last > vb.Last && va.Start < vb.Start { // |--va|--vb--|--| runs = append(runs, vb) - n += int32(vb.last-vb.start) + 1 + n += int32(vb.Last-vb.Start) + 1 j++ - } else if va.last <= vb.last && va.start >= vb.start { + } else if va.Last <= vb.Last && va.Start >= vb.Start { // |--vb|--va--|--| runs = append(runs, va) - n += int32(va.last-va.start) + 1 + n += int32(va.Last-va.Start) + 1 i++ - } else if va.last <= vb.last && va.start < vb.start { + } else if va.Last <= vb.Last && va.Start < vb.Start { // |--va-|-|-vb--| - runs = append(runs, interval16{start: vb.start, last: va.last}) - n += int32(va.last-vb.start) + 1 + runs = append(runs, Interval16{Start: vb.Start, Last: va.Last}) + n += int32(va.Last-vb.Start) + 1 i++ } } @@ -1132,9 +1138,9 @@ func intersectRunArrayInPlace(a, b *Container) *Container { n := 0 for i, j := 0, 0; i < an && j < bn; { va, vb := ar[i], ba[j] - if vb < va.start { + if vb < va.Start { j++ - } else if vb > va.last { + } else if vb > va.Last { i++ } else { array[n] = vb @@ -1356,7 +1362,7 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { tContainer := target.Containers.Get(iKey) // if the target's full, short-circuit out. if tContainer != nil { - if tContainer.N() == maxContainerVal+1 { + if tContainer.N() == MaxContainerVal+1 { bitmapIters.markItersWithKeyAsHandled(i, iKey) continue } @@ -1996,11 +2002,11 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length case containerRun: // official format stores runs as start/len, we want to convert, but since // they might be mmapped, we can't write to that memory - newRuns := make([]interval16, runCount) - oldRuns := (*[65536]interval16)(unsafe.Pointer(r.currentPointer))[:runCount:runCount] + newRuns := make([]Interval16, runCount) + oldRuns := (*[65536]Interval16)(unsafe.Pointer(r.currentPointer))[:runCount:runCount] copy(newRuns, oldRuns) for i := range newRuns { - newRuns[i].last += newRuns[i].start + newRuns[i].Last += newRuns[i].Start } r.currentPointer = (*uint16)(unsafe.Pointer(&newRuns[0])) r.currentLen = int(runCount) @@ -2177,7 +2183,7 @@ func (b *Bitmap) ImportRoaringBits(data []byte, clear bool, log bool, rowSize ui } else { importUpdater = func(oldC *Container, existed bool) (newC *Container, write bool) { existN := oldC.N() - if existN == maxContainerVal+1 { + if existN == MaxContainerVal+1 { return oldC, false } if existN == 0 { @@ -2399,7 +2405,7 @@ func BitmapsToRoaring(bitmaps []*Bitmap) []byte { case containerRun: binary.LittleEndian.PutUint16(nextData[0:2], uint16(c.len)) dataOffset += 2 - dataOffset += 4 * copy((*[1 << 15]interval16)(unsafe.Pointer(&nextData[2]))[:], c.runs()) + dataOffset += 4 * copy((*[1 << 15]Interval16)(unsafe.Pointer(&nextData[2]))[:], c.runs()) } } } @@ -2586,7 +2592,7 @@ func (itr *Iterator) Seek(seek uint64) { j, contains := binSearchRuns(lb, itr.c.runs()) if contains { itr.j = j - itr.k = int32(lb) - int32(itr.c.runs()[j].start) - 1 + itr.k = int32(lb) - int32(itr.c.runs()[j].Start) - 1 return } // If seek is larger than all elements, return. @@ -2660,7 +2666,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) { } r := itr.c.runs()[itr.j] - runLength := int32(r.last - r.start) + runLength := int32(r.Last - r.Start) if itr.k >= runLength { // Reached end of run, move to the next run. @@ -2730,7 +2736,7 @@ func (itr *Iterator) peek() uint64 { return itr.key<<16 | uint64(itr.c.array()[itr.j]) } if itr.c.isRun() { - return itr.key<<16 | uint64(itr.c.runs()[itr.j].start+uint16(itr.k)) + return itr.key<<16 | uint64(itr.c.runs()[itr.j].Start+uint16(itr.k)) } return itr.key<<16 | uint64(itr.j) } @@ -2741,19 +2747,19 @@ const ArrayMaxSize = 4096 // runMaxSize represents the maximum size of run length encoded containers. const runMaxSize = 2048 -type interval16 struct { - start uint16 - last uint16 +type Interval16 struct { + Start uint16 + Last uint16 } // runlen returns the count of integers in the interval. -func (iv interval16) runlen() int32 { - return 1 + int32(iv.last-iv.start) +func (iv Interval16) runlen() int32 { + return 1 + int32(iv.Last-iv.Start) } // count counts all bits in the container. func (c *Container) count() (n int32) { - return c.countRange(0, maxContainerVal+1) + return c.countRange(0, MaxContainerVal+1) } // countRange counts the number of bits set between [start, end). @@ -2837,28 +2843,28 @@ func (c *Container) runCountRange(start, end int32) (n int32) { runs := c.runs() for _, iv := range runs { // iv is before range - if int32(iv.last) < start { + if int32(iv.Last) < start { continue } // iv is after range - if end < int32(iv.start) { + if end < int32(iv.Start) { break } // iv is superset of range - if int32(iv.start) <= start && int32(iv.last) >= end { + if int32(iv.Start) <= start && int32(iv.Last) >= end { return end - start } // iv is subset of range - if int32(iv.start) >= start && int32(iv.last) <= end { + if int32(iv.Start) >= start && int32(iv.Last) <= end { n += iv.runlen() } // iv overlaps beginning of range without being a subset - if int32(iv.start) < start && int32(iv.last) < end { - n += int32(iv.last) - start + 1 + if int32(iv.Start) < start && int32(iv.Last) < end { + n += int32(iv.Last) - start + 1 } // iv overlaps end of range without being a subset - if int32(iv.start) > start && int32(iv.last) >= end { - n += end - int32(iv.start) + if int32(iv.Start) > start && int32(iv.Last) >= end { + n += end - int32(iv.Start) } } return n @@ -2934,49 +2940,49 @@ func (c *Container) runAdd(v uint16) (*Container, bool) { if len(runs) == 0 { c = c.Thaw() - c.setRuns([]interval16{{start: v, last: v}}) + c.setRuns([]Interval16{{Start: v, Last: v}}) c.setN(1) return c, true } i := sort.Search(len(runs), - func(i int) bool { return runs[i].last >= v }) + func(i int) bool { return runs[i].Last >= v }) if i == len(runs) { i-- } iv := runs[i] - if v >= iv.start && iv.last >= v { + if v >= iv.Start && iv.Last >= v { return c, false } c = c.Thaw() runs = c.runs() - if iv.last < v { - if iv.last == v-1 { - runs[i].last++ + if iv.Last < v { + if iv.Last == v-1 { + runs[i].Last++ } else { - runs = append(runs, interval16{start: v, last: v}) + runs = append(runs, Interval16{Start: v, Last: v}) } - } else if v+1 == iv.start { + } else if v+1 == iv.Start { // combining two intervals - if i > 0 && runs[i-1].last == v-1 { - runs[i-1].last = iv.last + if i > 0 && runs[i-1].Last == v-1 { + runs[i-1].Last = iv.Last runs = append(runs[:i], runs[i+1:]...) c.setRuns(runs) c.setN(c.N() + 1) return c, true } // just before an interval - runs[i].start-- - } else if i > 0 && v-1 == runs[i-1].last { + runs[i].Start-- + } else if i > 0 && v-1 == runs[i-1].Last { // just after an interval - runs[i-1].last++ + runs[i-1].Last++ } else { // alone - newIv := interval16{start: v, last: v} - runs = append(runs[:i], append([]interval16{newIv}, runs[i:]...)...) + newIv := Interval16{Start: v, Last: v} + runs = append(runs[:i], append([]Interval16{newIv}, runs[i:]...)...) } c.setRuns(runs) c.setN(c.N() + 1) @@ -3107,7 +3113,7 @@ func (c *Container) unionInPlace(other *Container) *Container { return c } // short-circuit the trivial cases - if c.N() == maxContainerVal+1 || other.N() == maxContainerVal+1 { + if c.N() == MaxContainerVal+1 || other.N() == MaxContainerVal+1 { return fullContainer } switch c.typ() { @@ -3157,11 +3163,11 @@ func (c *Container) bitmapContains(v uint16) bool { // binSearchRuns returns the index of the run containing v, and true, when v is contained; // or the index of the next run starting after v, and false, when v is not contained. -func binSearchRuns(v uint16, a []interval16) (int32, bool) { +func binSearchRuns(v uint16, a []Interval16) (int32, bool) { i := int32(sort.Search(len(a), - func(i int) bool { return a[i].last >= v })) + func(i int) bool { return a[i].Last >= v })) if i < int32(len(a)) { - return i, (v >= a[i].start) && (v <= a[i].last) + return i, (v >= a[i].Start) && (v <= a[i].Last) } return i, false @@ -3244,18 +3250,18 @@ func (c *Container) runRemove(v uint16) (*Container, bool) { } c = c.Thaw() runs = c.runs() - if v == runs[i].last && v == runs[i].start { + if v == runs[i].Last && v == runs[i].Start { runs = append(runs[:i], runs[i+1:]...) - } else if v == runs[i].last { - runs[i].last-- - } else if v == runs[i].start { - runs[i].start++ - } else if v > runs[i].start { - last := runs[i].last - runs[i].last = v - 1 - runs = append(runs, interval16{}) + } else if v == runs[i].Last { + runs[i].Last-- + } else if v == runs[i].Start { + runs[i].Start++ + } else if v > runs[i].Start { + last := runs[i].Last + runs[i].Last = v - 1 + runs = append(runs, Interval16{}) copy(runs[i+2:], runs[i+1:]) - runs[i+1] = interval16{start: v + 1, last: last} + runs[i+1] = Interval16{Start: v + 1, Last: last} // runs = append(runs[:i+1], append([]interval16{{start: v + 1, last: last}}, runs[i+1:]...)...) } c.setN(c.N() - 1) @@ -3303,7 +3309,7 @@ func (c *Container) runMax() uint16 { if len(runs) == 0 { return 0 } - return runs[len(runs)-1].last + return runs[len(runs)-1].Last } // bitmapToArray converts from bitmap format to array format. @@ -3410,8 +3416,8 @@ func (c *Container) runToBitmap() *Container { } bitmap := make([]uint64, bitmapN) for _, iv := range c.runs() { - w1, w2 := iv.start/64, iv.last/64 - b1, b2 := iv.start&63, iv.last&63 + w1, w2 := iv.Start/64, iv.Last/64 + b1, b2 := iv.Start&63, iv.Last&63 // a mask for everything under bit X looks like // (1 << x) - 1. Say b1 is 4; our mask will want // to have the bottom 4 bits be zero, so we shift @@ -3472,7 +3478,7 @@ func (c *Container) bitmapToRun(numRuns int32) *Container { if numRuns == 0 { numRuns = bitmapCountRuns(bitmap) } - runs := make([]interval16, 0, numRuns) + runs := make([]Interval16, 0, numRuns) current := bitmap[0] var i, start, last uint16 @@ -3501,12 +3507,12 @@ func (c *Container) bitmapToRun(numRuns int32) *Container { if current == maxBitmap { // bitmap[1023] == maxBitmap - runs = append(runs, interval16{start, maxContainerVal}) + runs = append(runs, Interval16{start, MaxContainerVal}) break } currentLast := uint16(trailingZeroN(^current)) last = 64*i + currentLast - runs = append(runs, interval16{start, last - 1}) + runs = append(runs, Interval16{start, last - 1}) // pad LSBs with 0s current = current & (current + 1) @@ -3546,17 +3552,17 @@ func (c *Container) arrayToRun(numRuns int32) *Container { numRuns = arrayCountRuns(array) } - runs := make([]interval16, 0, numRuns) + runs := make([]Interval16, 0, numRuns) start := array[0] for i, v := range array[1:] { if v-array[i] > 1 { // if current-previous > 1, one run ends and another begins - runs = append(runs, interval16{start, array[i]}) + runs = append(runs, Interval16{start, array[i]}) start = v } } // append final run - runs = append(runs, interval16{start, array[c.N()-1]}) + runs = append(runs, Interval16{start, array[c.N()-1]}) if c.frozen() { return NewContainerRunN(runs, c.N()) } @@ -3591,7 +3597,7 @@ func (c *Container) runToArray() *Container { array := make([]uint16, c.N()) n := int32(0) for _, r := range runs { - for v := int(r.start); v <= int(r.last); v++ { + for v := int(r.Start); v <= int(r.Last); v++ { array[n] = uint16(v) n++ } @@ -3744,12 +3750,12 @@ func (c *Container) check() error { a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(array), c.N())) } } else if c.isRun() { - n := c.runCountRange(0, maxContainerVal+1) + n := c.runCountRange(0, MaxContainerVal+1) if n != c.N() { a.Append(fmt.Errorf("run count mismatch: count=%d, n=%d", n, c.N())) } } else if c.isBitmap() { - if n := c.bitmapCountRange(0, maxContainerVal+1); n != c.N() { + if n := c.bitmapCountRange(0, MaxContainerVal+1); n != c.N() { a.Append(fmt.Errorf("bitmap count mismatch: count=%d, n=%d", n, c.N())) } } else { @@ -3846,10 +3852,10 @@ func flipRun(b *Container) *Container { } func intersectionCount(a, b *Container) int32 { - if a.N() == maxContainerVal+1 { + if a.N() == MaxContainerVal+1 { return b.N() } - if b.N() == maxContainerVal+1 { + if b.N() == MaxContainerVal+1 { return a.N() } if a.N() == 0 || b.N() == 0 { @@ -3911,12 +3917,12 @@ func intersectionCountArrayRun(a, b *Container) (n int32) { na, nb := len(array), len(runs) for i, j := 0, 0; i < na && j < nb; { va, vb := array[i], runs[j] - if va < vb.start { + if va < vb.Start { i++ - } else if va >= vb.start && va <= vb.last { + } else if va >= vb.Start && va <= vb.Last { i++ n++ - } else if va > vb.last { + } else if va > vb.Last { j++ } } @@ -3929,27 +3935,27 @@ func intersectionCountRunRun(a, b *Container) (n int32) { na, nb := len(ra), len(rb) for i, j := 0, 0; i < na && j < nb; { va, vb := ra[i], rb[j] - if va.last < vb.start { + if va.Last < vb.Start { // |--va--| |--vb--| i++ - } else if va.start > vb.last { + } else if va.Start > vb.Last { // |--vb--| |--va--| j++ - } else if va.last > vb.last && va.start >= vb.start { + } else if va.Last > vb.Last && va.Start >= vb.Start { // |--vb-|-|-va--| - n += 1 + int32(vb.last-va.start) + n += 1 + int32(vb.Last-va.Start) j++ - } else if va.last > vb.last && va.start < vb.start { + } else if va.Last > vb.Last && va.Start < vb.Start { // |--va|--vb--|--| - n += 1 + int32(vb.last-vb.start) + n += 1 + int32(vb.Last-vb.Start) j++ - } else if va.last <= vb.last && va.start >= vb.start { + } else if va.Last <= vb.Last && va.Start >= vb.Start { // |--vb|--va--|--| - n += 1 + int32(va.last-va.start) + n += 1 + int32(va.Last-va.Start) i++ - } else if va.last <= vb.last && va.start < vb.start { + } else if va.Last <= vb.Last && va.Start < vb.Start { // |--va-|-|-vb--| - n += 1 + int32(va.last-vb.start) + n += 1 + int32(va.Last-vb.Start) i++ } } @@ -3959,7 +3965,7 @@ func intersectionCountRunRun(a, b *Container) (n int32) { func intersectionCountBitmapRun(a, b *Container) (n int32) { statsHit("intersectionCount/BitmapRun") for _, iv := range b.runs() { - n += a.bitmapCountRange(int32(iv.start), int32(iv.last)+1) + n += a.bitmapCountRange(int32(iv.Start), int32(iv.Last)+1) } return n } @@ -3985,10 +3991,10 @@ func intersectionCountBitmapBitmap(a, b *Container) (n int32) { } func intersect(a, b *Container) *Container { - if a.N() == maxContainerVal+1 { + if a.N() == MaxContainerVal+1 { return b.Freeze() } - if b.N() == maxContainerVal+1 { + if b.N() == MaxContainerVal+1 { return a.Freeze() } if a.N() == 0 || b.N() == 0 { @@ -4050,9 +4056,9 @@ func intersectArrayRun(a, b *Container) *Container { var output []uint16 for i, j := 0, 0; i < na && j < nb; { va, vb := aa[i], rb[j] - if va < vb.start { + if va < vb.Start { i++ - } else if va > vb.last { + } else if va > vb.Last { j++ } else { output = append(output, va) @@ -4071,27 +4077,27 @@ func intersectRunRun(a, b *Container) *Container { n := int32(0) for i, j := 0, 0; i < na && j < nb; { va, vb := ra[i], rb[j] - if va.last < vb.start { + if va.Last < vb.Start { // |--va--| |--vb--| i++ - } else if vb.last < va.start { + } else if vb.Last < va.Start { // |--vb--| |--va--| j++ - } else if va.last > vb.last && va.start >= vb.start { + } else if va.Last > vb.Last && va.Start >= vb.Start { // |--vb-|-|-va--| - n += output.runAppendInterval(interval16{start: va.start, last: vb.last}) + n += output.runAppendInterval(Interval16{Start: va.Start, Last: vb.Last}) j++ - } else if va.last > vb.last && va.start < vb.start { + } else if va.Last > vb.Last && va.Start < vb.Start { // |--va|--vb--|--| n += output.runAppendInterval(vb) j++ - } else if va.last <= vb.last && va.start >= vb.start { + } else if va.Last <= vb.Last && va.Start >= vb.Start { // |--vb|--va--|--| n += output.runAppendInterval(va) i++ - } else if va.last <= vb.last && va.start < vb.start { + } else if va.Last <= vb.Last && va.Start < vb.Start { // |--va-|-|-vb--| - n += output.runAppendInterval(interval16{start: vb.start, last: va.last}) + n += output.runAppendInterval(Interval16{Start: vb.Start, Last: va.Last}) i++ } } @@ -4115,7 +4121,7 @@ func intersectBitmapRun(a, b *Container) *Container { // output is array container array := make([]uint16, 0, b.N()) for _, iv := range runs { - for i := iv.start; i <= iv.last; i++ { + for i := iv.Start; i <= iv.Last; i++ { if a.bitmapContains(i) { array = append(array, i) } @@ -4137,25 +4143,25 @@ func intersectBitmapRun(a, b *Container) *Container { n := int32(0) for j := 0; j < len(runs); j++ { vb := runs[j] - i := vb.start >> 6 // index into a + i := vb.Start >> 6 // index into a vastart := i << 6 valast := vastart + 63 - for valast >= vb.start && vastart <= vb.last && i < bitmapN { - if vastart >= vb.start && valast <= vb.last { // a within b + for valast >= vb.Start && vastart <= vb.Last && i < bitmapN { + if vastart >= vb.Start && valast <= vb.Last { // a within b bitmap[i] = aBitmap[i] n += int32(popcount(aBitmap[i])) - } else if vb.start >= vastart && vb.last <= valast { // b within a - var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart) + } else if vb.Start >= vastart && vb.Last <= valast { // b within a + var mask uint64 = ((1 << (vb.Last - vb.Start + 1)) - 1) << (vb.Start - vastart) bits := aBitmap[i] & mask bitmap[i] |= bits n += int32(popcount(bits)) - } else if vastart < vb.start { // a overlaps front of b - offset := 64 - (1 + valast - vb.start) + } else if vastart < vb.Start { // a overlaps front of b + offset := 64 - (1 + valast - vb.Start) bits := (aBitmap[i] >> offset) << offset bitmap[i] |= bits n += int32(popcount(bits)) - } else if vb.start < vastart { // b overlaps front of a - offset := 64 - (1 + vb.last - vastart) + } else if vb.Start < vastart { // b overlaps front of a + offset := 64 - (1 + vb.Last - vastart) bits := (aBitmap[i] << offset) >> offset bitmap[i] |= bits n += int32(popcount(bits)) @@ -4207,7 +4213,7 @@ func intersectBitmapBitmap(a, b *Container) *Container { } func union(a, b *Container) *Container { - if a.N() == maxContainerVal+1 || b.N() == maxContainerVal+1 { + if a.N() == MaxContainerVal+1 || b.N() == MaxContainerVal+1 { return fullContainer } if a.isArray() { @@ -4236,7 +4242,9 @@ func union(a, b *Container) *Container { } } } +func Merge(a, b []uint16) { +} func unionArrayArray(a, b *Container) *Container { statsHit("union/ArrayArray") if a.N() == 0 { @@ -4345,7 +4353,7 @@ func unionArrayRun(a, b *Container) *Container { output := NewContainerRun(nil) aa, rb := a.array(), b.runs() na, nb := len(aa), len(rb) - var vb interval16 + var vb Interval16 var va uint16 n := int32(0) for i, j := 0, 0; i < na || j < nb; { @@ -4355,8 +4363,8 @@ func unionArrayRun(a, b *Container) *Container { if j < nb { vb = rb[j] } - if i < na && (j >= nb || va < vb.start) { - n += output.runAppendInterval(interval16{start: va, last: va}) + if i < na && (j >= nb || va < vb.Start) { + n += output.runAppendInterval(Interval16{Start: va, Last: va}) i++ } else { n += output.runAppendInterval(vb) @@ -4378,26 +4386,26 @@ func unionArrayRun(a, b *Container) *Container { // interval is earlier than the start of the last interval in the list of runs. // Its return value is the amount by which the cardinality of the container was // increased. -func (c *Container) runAppendInterval(v interval16) int32 { +func (c *Container) runAppendInterval(v Interval16) int32 { runs := c.runs() if len(runs) == 0 { runs = append(runs, v) c.setRuns(runs) - return int32(v.last-v.start) + 1 + return int32(v.Last-v.Start) + 1 } last := runs[len(runs)-1] - if last.last == maxContainerVal { //protect against overflow + if last.Last == MaxContainerVal { //protect against overflow return 0 } - if last.last+1 >= v.start && v.last > last.last { - runs[len(runs)-1].last = v.last + if last.Last+1 >= v.Start && v.Last > last.Last { + runs[len(runs)-1].Last = v.Last c.setRuns(runs) - return int32(v.last - last.last) - } else if last.last+1 < v.start { + return int32(v.Last - last.Last) + } else if last.Last+1 < v.Start { runs = append(runs, v) c.setRuns(runs) - return int32(v.last-v.start) + 1 + return int32(v.Last-v.Start) + 1 } return 0 } @@ -4406,8 +4414,8 @@ func unionRunRun(a, b *Container) *Container { statsHit("union/RunRun") ra, rb := a.runs(), b.runs() na, nb := len(ra), len(rb) - output := NewContainerRun(make([]interval16, 0, na+nb)) - var va, vb interval16 + output := NewContainerRun(make([]Interval16, 0, na+nb)) + var va, vb Interval16 n := int32(0) for i, j := 0, 0; i < na || j < nb; { if i < na { @@ -4416,7 +4424,7 @@ func unionRunRun(a, b *Container) *Container { if j < nb { vb = rb[j] } - if i < na && (j >= nb || va.start < vb.start) { + if i < na && (j >= nb || va.Start < vb.Start) { n += output.runAppendInterval(va) i++ } else { @@ -4435,7 +4443,7 @@ func unionBitmapRun(a, b *Container) *Container { statsHit("union/BitmapRun") output := a.Clone() for _, run := range b.runs() { - output.bitmapSetRange(uint64(run.start), uint64(run.last)+1) + output.bitmapSetRange(uint64(run.Start), uint64(run.Last)+1) } return output } @@ -4447,7 +4455,7 @@ func unionBitmapRunInPlace(a, b *Container) *Container { bitmap := a.bitmap() statsHit("union/BitmapRun") for _, run := range b.runs() { - bitmapSetRangeIgnoreN(bitmap, uint64(run.start), uint64(run.last)+1) + bitmapSetRangeIgnoreN(bitmap, uint64(run.Start), uint64(run.Last)+1) } return a } @@ -4574,15 +4582,15 @@ func compareArrayBitmap(a []uint16, b []uint64) error { // of the array's values in the run collection. the run collection // can't be empty; if it were, N would have been 0, and we wouldn't // have gotten here. -func compareArrayRuns(a []uint16, r []interval16) error { +func compareArrayRuns(a []uint16, r []Interval16) error { ri := 0 ru := r[ri] ri++ for _, v := range a { - if v < ru.start { + if v < ru.Start { return fmt.Errorf("value %d missing", v) } - if v > ru.last { + if v > ru.Last { if ri >= len(r) { return fmt.Errorf("value %d missing", v) } @@ -4590,7 +4598,7 @@ func compareArrayRuns(a []uint16, r []interval16) error { ri++ // if they're identical, the array value must be // the start of the next run. - if v != ru.start { + if v != ru.Start { return fmt.Errorf("value %d missing", v) } } @@ -4728,7 +4736,7 @@ func unionRunRunInPlace(a, b *Container) *Container { // and count `.start` and `.last` points. // If we get the `state == 0` it means we just built a new interval (`val`), // and we can set it in `a` at the possition `off` -func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { +func unionInterval16InPlace(a, b []Interval16) ([]Interval16, int32) { n := int32(0) an, bn := len(a), len(b) @@ -4742,7 +4750,7 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { // Offset of a - next available index to set. off int = 0 // Value to set/append to a at off - val interval16 + val Interval16 // Current state - state equals 0 means we are clear (out of intervals) // When we start a new interval we add +1 when we get out of interval we add -1. @@ -4761,7 +4769,7 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { eval = func(arr [2]uint16, ii int, onlyB bool) { if state == 0 && ii == 0 { // we are clear and start a new interval - val.start = arr[ii] + val.Start = arr[ii] if onlyB { fromB++ } @@ -4772,7 +4780,7 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { if state == 0 { // we just got out of interval // ii == 1 - val.last = arr[ii] + val.Last = arr[ii] if onlyB { fromB++ } @@ -4787,7 +4795,7 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { eval2 = func(arr [2]uint16, i1, i2 int) { if state == 0 && (i1 == 0 || i2 == 0) { // we are clear and start a new interval - val.start = arr[i1] + val.Start = arr[i1] } @@ -4797,7 +4805,7 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { if state == 0 { // (i1 == 1 || i2 == 1) // we just got out of interval - val.last = arr[i1] + val.Last = arr[i1] } } ) @@ -4808,8 +4816,8 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { var av, bv [2]uint16 if ai < an && bi < bn { - av[0], av[1] = a[ai].start, a[ai].last - bv[0], bv[1] = b[bi].start, b[bi].last + av[0], av[1] = a[ai].Start, a[ai].Last + bv[0], bv[1] = b[bi].Start, b[bi].Last if av[aii] < bv[bii] { // a: |------------------- @@ -4839,11 +4847,11 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { bii++ } } else if ai < an { // only a left - av[0], av[1] = a[ai].start, a[ai].last + av[0], av[1] = a[ai].Start, a[ai].Last eval(av, aii, false) aii++ } else if bi < bn { // only b left - bv[0], bv[1] = b[bi].start, b[bi].last + bv[0], bv[1] = b[bi].Start, b[bi].Last eval(bv, bii, false) bii++ } else { @@ -4853,14 +4861,14 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { if state == 0 { if fromB == 2 { // val.start and val.last come from b, so we need to extend a, first - a = append(a, interval16{}) + a = append(a, Interval16{}) copy(a[off+1:], a[off:]) ai++ an++ } fromB = 0 a, off = appendInterval16At(a, val, off) - n += int32(val.last) - int32(val.start) + 1 + n += int32(val.Last) - int32(val.Start) + 1 } if aii == 2 { @@ -4884,10 +4892,10 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { // appendInterval16At appends or sets val in a at off position // The function returns modified a ([]interval16) and new offset (off) -func appendInterval16At(a []interval16, val interval16, off int) ([]interval16, int) { +func appendInterval16At(a []Interval16, val Interval16, off int) ([]Interval16, int) { - if off > 0 && int32(val.start)-int32(a[off-1].last) <= 1 { - a[off-1].last = val.last + if off > 0 && int32(val.Start)-int32(a[off-1].Last) <= 1 { + a[off-1].Last = val.Last return a, off } @@ -4904,7 +4912,7 @@ func appendInterval16At(a []interval16, val interval16, off int) ([]interval16, } func difference(a, b *Container) *Container { - if a.N() == 0 || b.N() == maxContainerVal+1 { + if a.N() == 0 || b.N() == MaxContainerVal+1 { return nil } if b.N() == 0 { @@ -4979,20 +4987,20 @@ func differenceArrayRun(a, b *Container) *Container { for i < len(aa) { // keep all array elements before beginning of runs - if aa[i] < rb[j].start { + if aa[i] < rb[j].Start { output = append(output, aa[i]) i++ continue } // if array element in run, skip it - if aa[i] >= rb[j].start && aa[i] <= rb[j].last { + if aa[i] >= rb[j].Start && aa[i] <= rb[j].Last { i++ continue } // if array element larger than current run, check next run - if aa[i] > rb[j].last { + if aa[i] > rb[j].Last { j++ if j == len(rb) { break @@ -5014,7 +5022,7 @@ func differenceBitmapRun(a, b *Container) *Container { statsHit("difference/BitmapRun") output := a.Clone() for _, run := range b.runs() { - output.bitmapZeroRange(uint64(run.start), uint64(run.last)+1) + output.bitmapZeroRange(uint64(run.Start), uint64(run.Last)+1) } return output } @@ -5024,22 +5032,22 @@ func differenceBitmapRun(a, b *Container) *Container { func differenceRunArray(a, b *Container) *Container { statsHit("difference/RunArray") ra, ab := a.runs(), b.array() - runs := make([]interval16, 0, len(ra)) + runs := make([]Interval16, 0, len(ra)) bidx := 0 vb := ab[bidx] RUNLOOP: for _, run := range ra { - start := run.start - for vb < run.start { + start := run.Start + for vb < run.Start { bidx++ if bidx >= len(ab) { break } vb = ab[bidx] } - for vb >= run.start && vb <= run.last { + for vb >= run.Start && vb <= run.Last { if vb == start { if vb == 65535 { // overflow break RUNLOOP @@ -5052,7 +5060,7 @@ RUNLOOP: vb = ab[bidx] continue } - runs = append(runs, interval16{start: start, last: vb - 1}) + runs = append(runs, Interval16{Start: start, Last: vb - 1}) if vb == 65535 { // overflow break RUNLOOP } @@ -5064,8 +5072,8 @@ RUNLOOP: vb = ab[bidx] } - if start <= run.last { - runs = append(runs, interval16{start: start, last: run.last}) + if start <= run.Last { + runs = append(runs, Interval16{Start: start, Last: run.Last}) } } output := NewContainerRun(runs) @@ -5078,38 +5086,38 @@ func differenceRunBitmap(a, b *Container) *Container { statsHit("difference/RunBitmap") ra := a.runs() // If a is full, difference is the flip of b. - if len(ra) > 0 && ra[0].start == 0 && ra[0].last == 65535 { + if len(ra) > 0 && ra[0].Start == 0 && ra[0].Last == 65535 { return flipBitmap(b) } bb := b.bitmap()[:1024] - runs := make([]interval16, 0, len(ra)) + runs := make([]Interval16, 0, len(ra)) for _, inputRun := range ra { run := inputRun add := true - for bit := inputRun.start; bit <= inputRun.last; bit++ { + for bit := inputRun.Start; bit <= inputRun.Last; bit++ { idx, exp := int(bit>>6), bit&63 if (bb[idx]>>exp)&1 != 0 { - if run.start == bit { + if run.Start == bit { if bit == 65535 { //overflow add = false } - run.start++ - } else if bit == run.last { - run.last-- + run.Start++ + } else if bit == run.Last { + run.Last-- } else { - run.last = bit - 1 - if run.last >= run.start { + run.Last = bit - 1 + if run.Last >= run.Start { if len(runs) >= runMaxSize { asBitmap := a.runToBitmap() return differenceBitmapBitmap(asBitmap, b) } runs = append(runs, run) } - run.start = bit + 1 - run.last = inputRun.last + run.Start = bit + 1 + run.Last = inputRun.Last } - if run.start > run.last { + if run.Start > run.Last { break } } @@ -5118,7 +5126,7 @@ func differenceRunBitmap(a, b *Container) *Container { break } } - if run.start <= run.last { + if run.Start <= run.Last { if add { if len(runs) >= runMaxSize { asBitmap := a.runToBitmap() @@ -5145,14 +5153,14 @@ func differenceRunRun(a, b *Container) *Container { ra, rb := a.runs(), b.runs() apos := 0 // current a-run index bpos := 0 // current b-run index - astart := ra[apos].start - alast := ra[apos].last - bstart := rb[bpos].start - blast := rb[bpos].last + astart := ra[apos].Start + alast := ra[apos].Last + bstart := rb[bpos].Start + blast := rb[bpos].Last alen := len(ra) blen := len(rb) - runs := make([]interval16, 0, alen+blen) // TODO allocate max then truncate? or something else + runs := make([]Interval16, 0, alen+blen) // TODO allocate max then truncate? or something else // cardinality upper bound: sum of number of runs // each B-run could split an A-run in two, up to len(b.runs) times @@ -5160,37 +5168,37 @@ func differenceRunRun(a, b *Container) *Container { switch { case alast < bstart: // current A-run entirely precedes current B-run: keep full A-run, advance to next A-run - runs = append(runs, interval16{start: astart, last: alast}) + runs = append(runs, Interval16{Start: astart, Last: alast}) apos++ if apos < alen { - astart = ra[apos].start - alast = ra[apos].last + astart = ra[apos].Start + alast = ra[apos].Last } case blast < astart: // current B-run entirely precedes current A-run: advance to next B-run bpos++ if bpos < blen { - bstart = rb[bpos].start - blast = rb[bpos].last + bstart = rb[bpos].Start + blast = rb[bpos].Last } default: // overlap if astart < bstart { - runs = append(runs, interval16{start: astart, last: bstart - 1}) + runs = append(runs, Interval16{Start: astart, Last: bstart - 1}) } if alast > blast { astart = blast + 1 } else { apos++ if apos < alen { - astart = ra[apos].start - alast = ra[apos].last + astart = ra[apos].Start + alast = ra[apos].Last } } } } if apos < alen { - runs = append(runs, interval16{start: astart, last: alast}) + runs = append(runs, Interval16{Start: astart, Last: alast}) apos++ if apos < alen { runs = append(runs, ra[apos:]...) @@ -5423,18 +5431,18 @@ func shiftRun(a *Container) (*Container, bool) { statsHit("shift/Run") carry := false ra := a.runs() - ro := make([]interval16, 0, len(ra)) + ro := make([]Interval16, 0, len(ra)) for _, v := range ra { - if v.start+1 == 0 { // final run was 1 bit on container edge + if v.Start+1 == 0 { // final run was 1 bit on container edge carry = true break - } else if v.last+1 == 0 { // final run ends on container edge - v.start++ + } else if v.Last+1 == 0 { // final run ends on container edge + v.Start++ carry = true } else { - v.start++ - v.last++ + v.Start++ + v.Last++ carry = false } ro = append(ro, v) @@ -5800,7 +5808,7 @@ func xorArrayRun(a, b *Container) *Container { output := NewContainerRun(nil) aa, rb := a.array(), b.runs() na, nb := len(aa), len(rb) - var vb interval16 + var vb Interval16 var va uint16 lastI, lastJ := -1, -1 n := int32((0)) @@ -5814,27 +5822,27 @@ func xorArrayRun(a, b *Container) *Container { lastI = i lastJ = j - if i < na && (j >= nb || va < vb.start) { //before - n += output.runAppendInterval(interval16{start: va, last: va}) + if i < na && (j >= nb || va < vb.Start) { //before + n += output.runAppendInterval(Interval16{Start: va, Last: va}) i++ - } else if j < nb && (i >= na || va > vb.last) { //after + } else if j < nb && (i >= na || va > vb.Last) { //after n += output.runAppendInterval(vb) j++ - } else if va > vb.start { - if va < vb.last { - n += output.runAppendInterval(interval16{start: vb.start, last: va - 1}) + } else if va > vb.Start { + if va < vb.Last { + n += output.runAppendInterval(Interval16{Start: vb.Start, Last: va - 1}) i++ - vb.start = va + 1 + vb.Start = va + 1 - if vb.start > vb.last { + if vb.Start > vb.Last { j++ } - } else if va > vb.last { + } else if va > vb.Last { n += output.runAppendInterval(vb) j++ } else { // va == vb.last - vb.last-- - if vb.start <= vb.last { + vb.Last-- + if vb.Start <= vb.Last { n += output.runAppendInterval(vb) } j++ @@ -5842,11 +5850,11 @@ func xorArrayRun(a, b *Container) *Container { } } else { // we know va == vb.start - if vb.start == maxContainerVal { // protect overflow + if vb.Start == MaxContainerVal { // protect overflow j++ } else { - vb.start++ - if vb.start > vb.last { + vb.Start++ + if vb.Start > vb.Last { j++ } } @@ -5863,7 +5871,7 @@ func xorArrayRun(a, b *Container) *Container { } // xorCompare computes first exclusive run between two runs. -func xorCompare(x *xorstm) (r1 interval16, hasData bool) { +func xorCompare(x *xorstm) (r1 Interval16, hasData bool) { hasData = false if !x.vaValid || !x.vbValid { if x.vbValid { @@ -5877,72 +5885,72 @@ func xorCompare(x *xorstm) (r1 interval16, hasData bool) { return r1, false } - if x.va.last < x.vb.start { //va before + if x.va.Last < x.vb.Start { //va before x.vaValid = false r1 = x.va hasData = true - } else if x.vb.last < x.va.start { //vb before + } else if x.vb.Last < x.va.Start { //vb before x.vbValid = false r1 = x.vb hasData = true - } else if x.va.start == x.vb.start && x.va.last == x.vb.last { // Equal + } else if x.va.Start == x.vb.Start && x.va.Last == x.vb.Last { // Equal x.vaValid = false x.vbValid = false - } else if x.va.start <= x.vb.start && x.va.last >= x.vb.last { //vb inside + } else if x.va.Start <= x.vb.Start && x.va.Last >= x.vb.Last { //vb inside x.vbValid = false - if x.va.start != x.vb.start { - r1 = interval16{start: x.va.start, last: x.vb.start - 1} + if x.va.Start != x.vb.Start { + r1 = Interval16{Start: x.va.Start, Last: x.vb.Start - 1} hasData = true } - if x.vb.last == maxContainerVal { // Check for overflow + if x.vb.Last == MaxContainerVal { // Check for overflow x.vaValid = false } else { - x.va.start = x.vb.last + 1 - if x.va.start > x.va.last { + x.va.Start = x.vb.Last + 1 + if x.va.Start > x.va.Last { x.vaValid = false } } - } else if x.vb.start <= x.va.start && x.vb.last >= x.va.last { //va inside + } else if x.vb.Start <= x.va.Start && x.vb.Last >= x.va.Last { //va inside x.vaValid = false - if x.vb.start != x.va.start { - r1 = interval16{start: x.vb.start, last: x.va.start - 1} + if x.vb.Start != x.va.Start { + r1 = Interval16{Start: x.vb.Start, Last: x.va.Start - 1} hasData = true } - if x.va.last == maxContainerVal { //check for overflow + if x.va.Last == MaxContainerVal { //check for overflow x.vbValid = false } else { - x.vb.start = x.va.last + 1 - if x.vb.start > x.vb.last { + x.vb.Start = x.va.Last + 1 + if x.vb.Start > x.vb.Last { x.vbValid = false } } - } else if x.va.start < x.vb.start && x.va.last <= x.vb.last { //va first overlap + } else if x.va.Start < x.vb.Start && x.va.Last <= x.vb.Last { //va first overlap x.vaValid = false - r1 = interval16{start: x.va.start, last: x.vb.start - 1} + r1 = Interval16{Start: x.va.Start, Last: x.vb.Start - 1} hasData = true - if x.va.last == maxContainerVal { // check for overflow + if x.va.Last == MaxContainerVal { // check for overflow x.vbValid = false } else { - x.vb.start = x.va.last + 1 - if x.vb.start > x.vb.last { + x.vb.Start = x.va.Last + 1 + if x.vb.Start > x.vb.Last { x.vbValid = false } } - } else if x.vb.start < x.va.start && x.vb.last <= x.va.last { //vb first overlap + } else if x.vb.Start < x.va.Start && x.vb.Last <= x.va.Last { //vb first overlap x.vbValid = false - r1 = interval16{start: x.vb.start, last: x.va.start - 1} + r1 = Interval16{Start: x.vb.Start, Last: x.va.Start - 1} hasData = true - if x.vb.last == maxContainerVal { // check for overflow + if x.vb.Last == MaxContainerVal { // check for overflow x.vaValid = false } else { - x.va.start = x.vb.last + 1 - if x.va.start > x.va.last { + x.va.Start = x.vb.Last + 1 + if x.va.Start > x.va.Last { x.vaValid = false } } @@ -5953,7 +5961,7 @@ func xorCompare(x *xorstm) (r1 interval16, hasData bool) { //stm is state machine used to "xor" iterate over runs. type xorstm struct { vaValid, vbValid bool - va, vb interval16 + va, vb Interval16 } // xorRunRun computes the exclusive or of two run containers. @@ -6009,7 +6017,7 @@ func xorBitmapRun(a, b *Container) *Container { output := a.Clone() for _, run := range b.runs() { - output.bitmapXorRange(uint64(run.start), uint64(run.last)+1) + output.bitmapXorRange(uint64(run.Start), uint64(run.Last)+1) } return output @@ -6247,9 +6255,9 @@ func (w handledIters) calculateSummaryStats(key uint64) containerUnionSummarySta summary.c++ summary.n += int64(currContainer.N()) - if currContainer.N() == maxContainerVal+1 { + if currContainer.N() == MaxContainerVal+1 { summary.hasMaxRange = true - summary.n = maxContainerVal + 1 + summary.n = MaxContainerVal + 1 return summary } } @@ -6464,7 +6472,7 @@ func differenceArrayRunInPlace(c, other *Container) { for i < len(aa) { // keep all array elements before beginning of runs - if aa[i] < rb[j].start { + if aa[i] < rb[j].Start { aa[n] = aa[i] n++ i++ @@ -6472,13 +6480,13 @@ func differenceArrayRunInPlace(c, other *Container) { } // if array element in run, skip it - if aa[i] >= rb[j].start && aa[i] <= rb[j].last { + if aa[i] >= rb[j].Start && aa[i] <= rb[j].Last { i++ continue } // if array element larger than current run, check next run - if aa[i] > rb[j].last { + if aa[i] > rb[j].Last { j++ if j == len(rb) { break @@ -6546,7 +6554,7 @@ func differenceBitmapRunInPlace(c, other *Container) { return } for _, run := range other.runs() { - c.bitmapZeroRange(uint64(run.start), uint64(run.last)+1) + c.bitmapZeroRange(uint64(run.Start), uint64(run.Last)+1) } } @@ -6556,21 +6564,21 @@ func differenceRunArrayInPlace(c, other *Container) { if len(ra) == 0 || len(ab) == 0 { return } - runs := make([]interval16, 0, len(ra)) + runs := make([]Interval16, 0, len(ra)) bidx := 0 vb := ab[bidx] RUNLOOP: for _, run := range ra { - start := run.start - for vb < run.start { + start := run.Start + for vb < run.Start { bidx++ if bidx >= len(ab) { break } vb = ab[bidx] } - for vb >= run.start && vb <= run.last { + for vb >= run.Start && vb <= run.Last { if vb == start { if vb == 65535 { // overflow break RUNLOOP @@ -6583,7 +6591,7 @@ RUNLOOP: vb = ab[bidx] continue } - runs = append(runs, interval16{start: start, last: vb - 1}) + runs = append(runs, Interval16{Start: start, Last: vb - 1}) if vb == 65535 { // overflow break RUNLOOP } @@ -6595,14 +6603,14 @@ RUNLOOP: vb = ab[bidx] } - if start <= run.last { - runs = append(runs, interval16{start: start, last: run.last}) + if start <= run.Last { + runs = append(runs, Interval16{Start: start, Last: run.Last}) } } c.setRuns(runs) c.n = 0 for _, run := range runs { - c.n += int32(run.last-run.start) + 1 + c.n += int32(run.Last-run.Start) + 1 } c.optimize() } @@ -6614,7 +6622,7 @@ func differenceRunBitmapInPlace(c, other *Container) { return } // If a is full, difference is the flip of b. - if len(ra) > 0 && ra[0].start == 0 && ra[0].last == 65535 { + if len(ra) > 0 && ra[0].Start == 0 && ra[0].Last == 65535 { clone := other.Clone() bitmap := clone.bitmap() for i, word := range other.bitmap() { @@ -6626,29 +6634,29 @@ func differenceRunBitmapInPlace(c, other *Container) { c.setN(c.count()) return } - runs := make([]interval16, 0, len(ra)) + runs := make([]Interval16, 0, len(ra)) for _, inputRun := range ra { run := inputRun add := true - for bit := inputRun.start; bit <= inputRun.last; bit++ { + for bit := inputRun.Start; bit <= inputRun.Last; bit++ { if other.bitmapContains(bit) { - if run.start == bit { + if run.Start == bit { if bit == 65535 { //overflow add = false } - run.start++ - } else if bit == run.last { - run.last-- + run.Start++ + } else if bit == run.Last { + run.Last-- } else { - run.last = bit - 1 - if run.last >= run.start { + run.Last = bit - 1 + if run.Last >= run.Start { runs = append(runs, run) } - run.start = bit + 1 - run.last = inputRun.last + run.Start = bit + 1 + run.Last = inputRun.Last } - if run.start > run.last { + if run.Start > run.Last { break } } @@ -6657,7 +6665,7 @@ func differenceRunBitmapInPlace(c, other *Container) { break } } - if run.start <= run.last { + if run.Start <= run.Last { if add { runs = append(runs, run) } @@ -6667,7 +6675,7 @@ func differenceRunBitmapInPlace(c, other *Container) { c.setRuns(runs) c.n = 0 for _, run := range runs { - c.n += int32(run.last-run.start) + 1 + c.n += int32(run.Last-run.Start) + 1 } if c.N() < ArrayMaxSize && int32(len(runs)) > c.N()/2 { c.runToArray() @@ -6685,14 +6693,14 @@ func differenceRunRunInPlace(c, other *Container) { } apos := 0 // current a-run index bpos := 0 // current b-run index - astart := ra[apos].start - alast := ra[apos].last - bstart := rb[bpos].start - blast := rb[bpos].last + astart := ra[apos].Start + alast := ra[apos].Last + bstart := rb[bpos].Start + blast := rb[bpos].Last alen := len(ra) blen := len(rb) - runs := make([]interval16, 0, alen+blen) // TODO allocate max then truncate? or something else + runs := make([]Interval16, 0, alen+blen) // TODO allocate max then truncate? or something else // cardinality upper bound: sum of number of runs // each B-run could split an A-run in two, up to len(b.runs) times @@ -6700,37 +6708,37 @@ func differenceRunRunInPlace(c, other *Container) { switch { case alast < bstart: // current A-run entirely precedes current B-run: keep full A-run, advance to next A-run - runs = append(runs, interval16{start: astart, last: alast}) + runs = append(runs, Interval16{Start: astart, Last: alast}) apos++ if apos < alen { - astart = ra[apos].start - alast = ra[apos].last + astart = ra[apos].Start + alast = ra[apos].Last } case blast < astart: // current B-run entirely precedes current A-run: advance to next B-run bpos++ if bpos < blen { - bstart = rb[bpos].start - blast = rb[bpos].last + bstart = rb[bpos].Start + blast = rb[bpos].Last } default: // overlap if astart < bstart { - runs = append(runs, interval16{start: astart, last: bstart - 1}) + runs = append(runs, Interval16{Start: astart, Last: bstart - 1}) } if alast > blast { astart = blast + 1 } else { apos++ if apos < alen { - astart = ra[apos].start - alast = ra[apos].last + astart = ra[apos].Start + alast = ra[apos].Last } } } } if apos < alen { - runs = append(runs, interval16{start: astart, last: alast}) + runs = append(runs, Interval16{Start: astart, Last: alast}) apos++ if apos < alen { runs = append(runs, ra[apos:]...) @@ -6739,6 +6747,39 @@ func differenceRunRunInPlace(c, other *Container) { c.setRuns(runs) c.n = 0 for _, run := range runs { - c.n += int32(run.last-run.start) + 1 + c.n += int32(run.Last-run.Start) + 1 } } + +//RBF exports to be reconsidered as we progress + +func (b *Bitmap) Put(key uint64, c *Container) { + b.Containers.Put(key, c) +} +func AsBitmap(c *Container) []uint64 { + return c.bitmap() +} +func AsArray(c *Container) []uint16 { + return c.array() +} +func ContainerType(c *Container) byte { + return c.typ() +} + +func AsRuns(c *Container) []Interval16 { + return c.runs() +} + +func ConvertArrayToBitmap(c *Container) { + c.arrayToBitmap() +} +func ConvertRunToBitmap(c *Container) { + c.runToBitmap() +} + +func Optimize(c *Container) { + c.optimize() +} +func Union(a, b *Container) *Container { + return union(a, b) +} diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index f8de764fb..6d1b67598 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -175,65 +175,65 @@ func bitmapEvenBitsSet() []uint64 { } ////////////////// run -func runEmpty() []interval16 { - return make([]interval16, 0) +func runEmpty() []Interval16 { + return make([]Interval16, 0) } -func runFull() []interval16 { - run := make([]interval16, 0) - run = append(run, interval16{start: 0, last: 65535}) +func runFull() []Interval16 { + run := make([]Interval16, 0) + run = append(run, Interval16{Start: 0, Last: 65535}) return run } -func runFirstBitSet() []interval16 { - run := make([]interval16, 0) - run = append(run, interval16{start: 0, last: 0}) +func runFirstBitSet() []Interval16 { + run := make([]Interval16, 0) + run = append(run, Interval16{Start: 0, Last: 0}) return run } -func runLastBitSet() []interval16 { - run := make([]interval16, 0) - run = append(run, interval16{start: 65535, last: 65535}) +func runLastBitSet() []Interval16 { + run := make([]Interval16, 0) + run = append(run, Interval16{Start: 65535, Last: 65535}) return run } -func runFirstBitUnset() []interval16 { - run := make([]interval16, 0) - run = append(run, interval16{start: 1, last: 65535}) +func runFirstBitUnset() []Interval16 { + run := make([]Interval16, 0) + run = append(run, Interval16{Start: 1, Last: 65535}) return run } -func runLastBitUnset() []interval16 { - run := make([]interval16, 0) - run = append(run, interval16{start: 0, last: 65534}) +func runLastBitUnset() []Interval16 { + run := make([]Interval16, 0) + run = append(run, Interval16{Start: 0, Last: 65534}) return run } -func runInnerBitsSet() []interval16 { - run := make([]interval16, 0) - run = append(run, interval16{start: 1, last: 65534}) +func runInnerBitsSet() []Interval16 { + run := make([]Interval16, 0) + run = append(run, Interval16{Start: 1, Last: 65534}) return run } -func runOuterBitsSet() []interval16 { - run := make([]interval16, 0) - run = append(run, interval16{start: 0, last: 0}) - run = append(run, interval16{start: 65535, last: 65535}) +func runOuterBitsSet() []Interval16 { + run := make([]Interval16, 0) + run = append(run, Interval16{Start: 0, Last: 0}) + run = append(run, Interval16{Start: 65535, Last: 65535}) return run } -func runOddBitsSet() []interval16 { - run := make([]interval16, containerWidth/2) +func runOddBitsSet() []Interval16 { + run := make([]Interval16, containerWidth/2) for i := 0; i < int(containerWidth/2); i++ { - run[i] = interval16{start: uint16(2*i + 1), last: uint16(2*i + 1)} + run[i] = Interval16{Start: uint16(2*i + 1), Last: uint16(2*i + 1)} } return run } -func runEvenBitsSet() []interval16 { - run := make([]interval16, containerWidth/2) +func runEvenBitsSet() []Interval16 { + run := make([]Interval16, containerWidth/2) for i := 0; i < int(containerWidth/2); i++ { - run[i] = interval16{start: uint16(2 * i), last: uint16(2 * i)} + run[i] = Interval16{Start: uint16(2 * i), Last: uint16(2 * i)} } return run } @@ -258,7 +258,7 @@ func doContainer(typ byte, data interface{}) *Container { c := NewContainerBitmap(-1, data.([]uint64)) return c case containerRun: - return NewContainerRun(data.([]interval16)) + return NewContainerRun(data.([]Interval16)) } return nil } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 085b6e1d1..fdb2db790 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -30,35 +30,35 @@ import ( ) // String produces a human viewable string of the contents. -func (iv interval16) String() string { - return fmt.Sprintf("[%d, %d]", iv.start, iv.last) +func (iv Interval16) String() string { + return fmt.Sprintf("[%d, %d]", iv.Start, iv.Last) } func TestRunAppendInterval(t *testing.T) { a := NewContainerRun(nil) tests := []struct { - base []interval16 - app interval16 + base []Interval16 + app Interval16 exp int32 }{ { - base: []interval16{}, - app: interval16{start: 22, last: 25}, + base: []Interval16{}, + app: Interval16{Start: 22, Last: 25}, exp: 4, }, { - base: []interval16{{start: 20, last: 23}}, - app: interval16{start: 22, last: 25}, + base: []Interval16{{Start: 20, Last: 23}}, + app: Interval16{Start: 22, Last: 25}, exp: 2, }, { - base: []interval16{{start: 20, last: 23}}, - app: interval16{start: 21, last: 22}, + base: []Interval16{{Start: 20, Last: 23}}, + app: Interval16{Start: 21, Last: 22}, exp: 0, }, { - base: []interval16{{start: 20, last: 23}}, - app: interval16{start: 19, last: 25}, + base: []Interval16{{Start: 20, Last: 23}}, + app: Interval16{Start: 19, Last: 25}, exp: 2, // runAppendInterval explicitly does not support intervals whose start is < c.runs[-1].start }, } @@ -73,11 +73,11 @@ func TestRunAppendInterval(t *testing.T) { } func TestInterval16RunLen(t *testing.T) { - iv := interval16{start: 7, last: 9} + iv := Interval16{Start: 7, Last: 9} if iv.runlen() != 3 { t.Fatalf("should be 3") } - iv = interval16{start: 7, last: 7} + iv = Interval16{Start: 7, Last: 7} if iv.runlen() != 1 { t.Fatalf("should be 1") } @@ -87,17 +87,17 @@ func TestContainerRunAdd(t *testing.T) { c := NewContainerRun(nil) tests := []struct { op uint16 - exp []interval16 + exp []Interval16 }{ - {1, []interval16{{start: 1, last: 1}}}, - {2, []interval16{{start: 1, last: 2}}}, - {4, []interval16{{start: 1, last: 2}, {start: 4, last: 4}}}, - {3, []interval16{{start: 1, last: 4}}}, - {10, []interval16{{start: 1, last: 4}, {start: 10, last: 10}}}, - {7, []interval16{{start: 1, last: 4}, {start: 7, last: 7}, {start: 10, last: 10}}}, - {6, []interval16{{start: 1, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}}, - {0, []interval16{{start: 0, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}}, - {8, []interval16{{start: 0, last: 4}, {start: 6, last: 8}, {start: 10, last: 10}}}, + {1, []Interval16{{Start: 1, Last: 1}}}, + {2, []Interval16{{Start: 1, Last: 2}}}, + {4, []Interval16{{Start: 1, Last: 2}, {Start: 4, Last: 4}}}, + {3, []Interval16{{Start: 1, Last: 4}}}, + {10, []Interval16{{Start: 1, Last: 4}, {Start: 10, Last: 10}}}, + {7, []Interval16{{Start: 1, Last: 4}, {Start: 7, Last: 7}, {Start: 10, Last: 10}}}, + {6, []Interval16{{Start: 1, Last: 4}, {Start: 6, Last: 7}, {Start: 10, Last: 10}}}, + {0, []Interval16{{Start: 0, Last: 4}, {Start: 6, Last: 7}, {Start: 10, Last: 10}}}, + {8, []Interval16{{Start: 0, Last: 4}, {Start: 6, Last: 8}, {Start: 10, Last: 10}}}, } for _, test := range tests { c.setMapped(true) @@ -120,7 +120,7 @@ func TestContainerRunAdd2(t *testing.T) { if !ret { t.Fatalf("result of adding new bit should be true: %v", c.runs()) } - if !reflect.DeepEqual(c.runs(), []interval16{{start: 0, last: 0}}) { + if !reflect.DeepEqual(c.runs(), []Interval16{{Start: 0, Last: 0}}) { t.Fatalf("should have 1 run of length 1, but have %v", c.runs()) } c, ret = c.add(0) @@ -270,23 +270,23 @@ func TestBitmapCountRange(t *testing.T) { } func TestIntersectionCountArrayBitmap3(t *testing.T) { - a, b := NewContainerBitmapN(getFullBitmap(), maxContainerVal+1), NewContainerBitmapN(getFullBitmap(), maxContainerVal+1) + a, b := NewContainerBitmapN(getFullBitmap(), MaxContainerVal+1), NewContainerBitmapN(getFullBitmap(), MaxContainerVal+1) res := intersectBitmapBitmap(a, b) - if res.N() != res.count() || res.N() != maxContainerVal+1 { - t.Fatalf("test #1 intersectCountBitmapBitmap fail orig: %v new: %v exp: %v", res.N(), res.count(), maxContainerVal+1) + if res.N() != res.count() || res.N() != MaxContainerVal+1 { + t.Fatalf("test #1 intersectCountBitmapBitmap fail orig: %v new: %v exp: %v", res.N(), res.count(), MaxContainerVal+1) } a = a.bitmapToRun(0) res = intersectBitmapRun(b, a) - if res.N() != res.count() || res.N() != maxContainerVal+1 { - t.Fatalf("test #2 intersectCountBitmapRun fail orig: %v new: %v exp: %v", res.N(), res.count(), maxContainerVal+1) + if res.N() != res.count() || res.N() != MaxContainerVal+1 { + t.Fatalf("test #2 intersectCountBitmapRun fail orig: %v new: %v exp: %v", res.N(), res.count(), MaxContainerVal+1) } b.bitmapToRun(0) res = intersectRunRun(a, b) n := intersectionCountRunRun(a, b) - if res.N() != res.count() || res.N() != maxContainerVal+1 || res.N() != int32(n) { - t.Fatalf("test #3 intersectCountRunRun fail orig: %v new: %v exp: %v", res.N(), res.count(), maxContainerVal+1) + if res.N() != res.count() || res.N() != MaxContainerVal+1 || res.N() != int32(n) { + t.Fatalf("test #3 intersectCountRunRun fail orig: %v new: %v exp: %v", res.N(), res.count(), MaxContainerVal+1) } } @@ -335,22 +335,22 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { } func TestRunRemove(t *testing.T) { - c := NewContainerRun([]interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}) + c := NewContainerRun([]Interval16{{Start: 2, Last: 10}, {Start: 12, Last: 13}, {Start: 15, Last: 16}}) tests := []struct { op uint16 - exp []interval16 + exp []Interval16 expRet bool }{ - {2, []interval16{{start: 3, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, true}, - {10, []interval16{{start: 3, last: 9}, {start: 12, last: 13}, {start: 15, last: 16}}, true}, - {12, []interval16{{start: 3, last: 9}, {start: 13, last: 13}, {start: 15, last: 16}}, true}, - {13, []interval16{{start: 3, last: 9}, {start: 15, last: 16}}, true}, - {16, []interval16{{start: 3, last: 9}, {start: 15, last: 15}}, true}, - {6, []interval16{{start: 3, last: 5}, {start: 7, last: 9}, {start: 15, last: 15}}, true}, - {8, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, true}, - {8, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false}, - {1, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false}, - {44, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false}, + {2, []Interval16{{Start: 3, Last: 10}, {Start: 12, Last: 13}, {Start: 15, Last: 16}}, true}, + {10, []Interval16{{Start: 3, Last: 9}, {Start: 12, Last: 13}, {Start: 15, Last: 16}}, true}, + {12, []Interval16{{Start: 3, Last: 9}, {Start: 13, Last: 13}, {Start: 15, Last: 16}}, true}, + {13, []Interval16{{Start: 3, Last: 9}, {Start: 15, Last: 16}}, true}, + {16, []Interval16{{Start: 3, Last: 9}, {Start: 15, Last: 15}}, true}, + {6, []Interval16{{Start: 3, Last: 5}, {Start: 7, Last: 9}, {Start: 15, Last: 15}}, true}, + {8, []Interval16{{Start: 3, Last: 5}, {Start: 7, Last: 7}, {Start: 9, Last: 9}, {Start: 15, Last: 15}}, true}, + {8, []Interval16{{Start: 3, Last: 5}, {Start: 7, Last: 7}, {Start: 9, Last: 9}, {Start: 15, Last: 15}}, false}, + {1, []Interval16{{Start: 3, Last: 5}, {Start: 7, Last: 7}, {Start: 9, Last: 9}, {Start: 15, Last: 15}}, false}, + {44, []Interval16{{Start: 3, Last: 5}, {Start: 7, Last: 7}, {Start: 9, Last: 9}, {Start: 15, Last: 15}}, false}, } for i, test := range tests { @@ -370,7 +370,7 @@ func TestRunRemove(t *testing.T) { } func TestRunMax(t *testing.T) { - c := NewContainerRun([]interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}) + c := NewContainerRun([]Interval16{{Start: 2, Last: 10}, {Start: 12, Last: 13}, {Start: 15, Last: 16}}) max := c.max() if max != 16 { t.Fatalf("max for %v should be 16", c.runs()) @@ -385,7 +385,7 @@ func TestRunMax(t *testing.T) { func TestIntersectionCountArrayRun(t *testing.T) { a := NewContainerArray([]uint16{1, 5, 10, 11, 12}) - b := NewContainerRun([]interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}) + b := NewContainerRun([]Interval16{{Start: 2, Last: 10}, {Start: 12, Last: 13}, {Start: 15, Last: 16}}) ret := intersectionCountArrayRun(a, b) if ret != 3 { @@ -397,7 +397,7 @@ func TestIntersectionCountBitmapRun(t *testing.T) { ob := make([]uint64, bitmapN) ob[0] = 1 << 63 a := NewContainerBitmap(1, ob) - b := NewContainerRun([]interval16{{start: 63, last: 64}}) + b := NewContainerRun([]Interval16{{Start: 63, Last: 64}}) ret := intersectionCountBitmapRun(a, b) if ret != 1 { @@ -405,7 +405,7 @@ func TestIntersectionCountBitmapRun(t *testing.T) { } a = NewContainerBitmap(-1, []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}) - b = NewContainerRun([]interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}) + b = NewContainerRun([]Interval16{{Start: 29, Last: 31}, {Start: 125, Last: 134}, {Start: 191, Last: 197}, {Start: 200, Last: 300}}) ret = intersectionCountBitmapRun(a, b) if ret != 14 { @@ -415,40 +415,40 @@ func TestIntersectionCountBitmapRun(t *testing.T) { func TestIntersectionCountRunRun(t *testing.T) { tests := []struct { - aruns []interval16 - bruns []interval16 + aruns []Interval16 + bruns []Interval16 exp int32 }{ { - aruns: []interval16{}, - bruns: []interval16{{start: 3, last: 8}}, exp: 0}, + aruns: []Interval16{}, + bruns: []Interval16{{Start: 3, Last: 8}}, exp: 0}, { - aruns: []interval16{{start: 2, last: 10}}, - bruns: []interval16{{start: 3, last: 8}}, exp: 6}, + aruns: []Interval16{{Start: 2, Last: 10}}, + bruns: []Interval16{{Start: 3, Last: 8}}, exp: 6}, { - aruns: []interval16{{start: 2, last: 10}}, - bruns: []interval16{{start: 1, last: 11}}, exp: 9}, + aruns: []Interval16{{Start: 2, Last: 10}}, + bruns: []Interval16{{Start: 1, Last: 11}}, exp: 9}, { - aruns: []interval16{{start: 2, last: 10}}, - bruns: []interval16{{start: 0, last: 2}}, exp: 1}, + aruns: []Interval16{{Start: 2, Last: 10}}, + bruns: []Interval16{{Start: 0, Last: 2}}, exp: 1}, { - aruns: []interval16{{start: 2, last: 10}}, - bruns: []interval16{{start: 1, last: 10}}, exp: 9}, + aruns: []Interval16{{Start: 2, Last: 10}}, + bruns: []Interval16{{Start: 1, Last: 10}}, exp: 9}, { - aruns: []interval16{{start: 2, last: 10}}, - bruns: []interval16{{start: 5, last: 12}}, exp: 6}, + aruns: []Interval16{{Start: 2, Last: 10}}, + bruns: []Interval16{{Start: 5, Last: 12}}, exp: 6}, { - aruns: []interval16{{start: 2, last: 10}}, - bruns: []interval16{{start: 10, last: 99}}, exp: 1}, + aruns: []Interval16{{Start: 2, Last: 10}}, + bruns: []Interval16{{Start: 10, Last: 99}}, exp: 1}, { - aruns: []interval16{{start: 2, last: 10}, {start: 44, last: 99}}, - bruns: []interval16{{start: 12, last: 14}}, exp: 0}, + aruns: []Interval16{{Start: 2, Last: 10}, {Start: 44, Last: 99}}, + bruns: []Interval16{{Start: 12, Last: 14}}, exp: 0}, { - aruns: []interval16{{start: 2, last: 10}, {start: 12, last: 13}}, - bruns: []interval16{{start: 2, last: 10}, {start: 12, last: 13}}, exp: 11}, + aruns: []Interval16{{Start: 2, Last: 10}, {Start: 12, Last: 13}}, + bruns: []Interval16{{Start: 2, Last: 10}, {Start: 12, Last: 13}}, exp: 11}, { - aruns: []interval16{{start: 8, last: 12}, {start: 15, last: 19}}, - bruns: []interval16{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6}, + aruns: []Interval16{{Start: 8, Last: 12}, {Start: 15, Last: 19}}, + bruns: []Interval16{{Start: 9, Last: 9}, {Start: 11, Last: 17}}, exp: 6}, } for i, test := range tests { a := NewContainerRun(test.aruns) @@ -465,27 +465,27 @@ func TestIntersectArrayRun(t *testing.T) { b := NewContainerRun(nil) tests := []struct { array []uint16 - runs []interval16 + runs []Interval16 exp []uint16 }{ { array: []uint16{1, 4, 5, 7, 10, 11, 12}, - runs: []interval16{{start: 5, last: 10}}, + runs: []Interval16{{Start: 5, Last: 10}}, exp: []uint16{5, 7, 10}, }, { array: []uint16{}, - runs: []interval16{{start: 5, last: 10}}, + runs: []Interval16{{Start: 5, Last: 10}}, exp: []uint16(nil), }, { array: []uint16{1, 4, 5, 7, 10, 11, 12}, - runs: []interval16{}, + runs: []Interval16{}, exp: []uint16(nil), }, { array: []uint16{0, 1, 4, 5, 7, 10, 11, 12}, - runs: []interval16{{start: 0, last: 5}, {start: 7, last: 7}}, + runs: []Interval16{{Start: 0, Last: 5}, {Start: 7, Last: 7}}, exp: []uint16{0, 1, 4, 5, 7}, }, } @@ -508,45 +508,45 @@ func TestIntersectRunRun(t *testing.T) { a := NewContainerRun(nil) b := NewContainerRun(nil) tests := []struct { - aruns []interval16 - bruns []interval16 - exp []interval16 + aruns []Interval16 + bruns []Interval16 + exp []Interval16 expN int32 }{ { - aruns: []interval16{}, - bruns: []interval16{{start: 5, last: 10}}, - exp: []interval16(nil), + aruns: []Interval16{}, + bruns: []Interval16{{Start: 5, Last: 10}}, + exp: []Interval16(nil), expN: 0, }, { - aruns: []interval16{{start: 5, last: 12}}, - bruns: []interval16{{start: 5, last: 10}}, - exp: []interval16{{start: 5, last: 10}}, + aruns: []Interval16{{Start: 5, Last: 12}}, + bruns: []Interval16{{Start: 5, Last: 10}}, + exp: []Interval16{{Start: 5, Last: 10}}, expN: 6, }, { - aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}}, - bruns: []interval16{{start: 5, last: 10}}, - exp: []interval16{{start: 5, last: 5}, {start: 7, last: 10}}, + aruns: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 5}, {Start: 7, Last: 8}, {Start: 9, Last: 12}}, + bruns: []Interval16{{Start: 5, Last: 10}}, + exp: []Interval16{{Start: 5, Last: 5}, {Start: 7, Last: 10}}, expN: 5, }, { - aruns: []interval16{{start: 20, last: 30}}, - bruns: []interval16{{start: 5, last: 10}, {start: 19, last: 21}}, - exp: []interval16{{start: 20, last: 21}}, + aruns: []Interval16{{Start: 20, Last: 30}}, + bruns: []Interval16{{Start: 5, Last: 10}, {Start: 19, Last: 21}}, + exp: []Interval16{{Start: 20, Last: 21}}, expN: 2, }, { - aruns: []interval16{{start: 5, last: 10}}, - bruns: []interval16{{start: 7, last: 12}}, - exp: []interval16{{start: 7, last: 10}}, + aruns: []Interval16{{Start: 5, Last: 10}}, + bruns: []Interval16{{Start: 7, Last: 12}}, + exp: []Interval16{{Start: 7, Last: 10}}, expN: 4, }, { - aruns: []interval16{{start: 5, last: 12}}, - bruns: []interval16{{start: 7, last: 10}}, - exp: []interval16{{start: 7, last: 10}}, + aruns: []Interval16{{Start: 5, Last: 12}}, + bruns: []Interval16{{Start: 7, Last: 10}}, + exp: []Interval16{{Start: 7, Last: 10}}, expN: 4, }, } @@ -570,37 +570,37 @@ func TestIntersectRunRun(t *testing.T) { func TestIntersectBitmapRunBitmap(t *testing.T) { tests := []struct { bitmap []uint64 - runs []interval16 + runs []Interval16 exp []uint64 expN int32 }{ { bitmap: []uint64{1}, - runs: []interval16{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 4096}}, + runs: []Interval16{{Start: 0, Last: 0}, {Start: 2, Last: 5}, {Start: 62, Last: 71}, {Start: 77, Last: 4096}}, exp: []uint64{1}, expN: 1, }, { bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, - runs: []interval16{{start: 1, last: 1}}, + runs: []Interval16{{Start: 1, Last: 1}}, exp: []uint64{2}, expN: 1, }, { bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, - runs: []interval16{{start: 1, last: 1}, {start: 10, last: 12}, {start: 61, last: 77}}, + runs: []Interval16{{Start: 1, Last: 1}, {Start: 10, Last: 12}, {Start: 61, Last: 77}}, exp: []uint64{0xe000000000001C02}, expN: 7, }, { bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF}, - runs: []interval16{{start: 1, last: 1}, {start: 61, last: 77}}, + runs: []Interval16{{Start: 1, Last: 1}, {Start: 61, Last: 77}}, exp: []uint64{0xE000000000000002, 0x00000000000003FFF}, expN: 18, }, { bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 1, 1, 1, 0xA, 1, 1, 0, 1}, - runs: []interval16{{start: 63, last: 10000}}, + runs: []Interval16{{Start: 63, Last: 10000}}, exp: []uint64{0x8000000000000000, 1, 1, 1, 0xA, 1, 1, 0, 1}, expN: 9, }, @@ -630,37 +630,37 @@ func TestIntersectBitmapRunArray(t *testing.T) { b := NewContainerRun(nil) tests := []struct { bitmap []uint64 - runs []interval16 + runs []Interval16 exp []uint16 expN int32 }{ { bitmap: []uint64{1}, - runs: []interval16{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 4096}}, + runs: []Interval16{{Start: 0, Last: 0}, {Start: 2, Last: 5}, {Start: 62, Last: 71}, {Start: 77, Last: 4096}}, exp: []uint16{0}, expN: 1, }, { bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, - runs: []interval16{{start: 1, last: 1}}, + runs: []Interval16{{Start: 1, Last: 1}}, exp: []uint16{1}, expN: 1, }, { bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, - runs: []interval16{{start: 1, last: 1}, {start: 10, last: 12}, {start: 61, last: 77}}, + runs: []Interval16{{Start: 1, Last: 1}, {Start: 10, Last: 12}, {Start: 61, Last: 77}}, exp: []uint16{1, 10, 11, 12, 61, 62, 63}, expN: 7, }, { bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF}, - runs: []interval16{{start: 1, last: 1}, {start: 61, last: 68}}, + runs: []Interval16{{Start: 1, Last: 1}, {Start: 61, Last: 68}}, exp: []uint16{1, 61, 62, 63, 64, 65, 66, 67, 68}, expN: 9, }, { bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 1, 1, 1, 0xA, 1, 1, 0, 1}, - runs: []interval16{{start: 63, last: 10000}, {start: 65000, last: 65535}}, + runs: []Interval16{{Start: 63, Last: 10000}, {Start: 65000, Last: 65535}}, exp: []uint16{63, 64, 128, 192, 257, 259, 320, 384, 512}, expN: 9, }, @@ -688,7 +688,7 @@ func TestUnionMixed(t *testing.T) { b := NewContainerBitmap(2, []uint64{0x3}) // run container - r := NewContainerRun([]interval16{{start: 5, last: 10}}) + r := NewContainerRun([]Interval16{{Start: 5, Last: 10}}) t.Run("various container Unions", func(t *testing.T) { tests := []struct { @@ -724,163 +724,163 @@ func TestUnionMixed(t *testing.T) { func TestUnionInterval16InPlace(t *testing.T) { tests := []struct { name string - a []interval16 - b []interval16 - expected []interval16 + a []Interval16 + b []Interval16 + expected []Interval16 expectedN int32 }{ { name: "firstBitUnset lastBitSet", - a: []interval16{interval16{1, 10}}, - b: []interval16{interval16{10, 10}}, - expected: []interval16{interval16{1, 10}}, + a: []Interval16{Interval16{1, 10}}, + b: []Interval16{Interval16{10, 10}}, + expected: []Interval16{Interval16{1, 10}}, expectedN: 10, }, { name: "single overlap", - a: []interval16{interval16{1, 10}, interval16{21, 28}}, - b: []interval16{interval16{8, 12}}, - expected: []interval16{interval16{1, 12}, interval16{21, 28}}, + a: []Interval16{Interval16{1, 10}, Interval16{21, 28}}, + b: []Interval16{Interval16{8, 12}}, + expected: []Interval16{Interval16{1, 12}, Interval16{21, 28}}, expectedN: 20, }, { name: "nested intervals", - a: []interval16{interval16{3, 13}, interval16{17, 20}}, - b: []interval16{interval16{1, 4}, interval16{6, 7}, interval16{8, 9}, interval16{10, 11}, interval16{14, 17}}, - expected: []interval16{interval16{1, 20}}, + a: []Interval16{Interval16{3, 13}, Interval16{17, 20}}, + b: []Interval16{Interval16{1, 4}, Interval16{6, 7}, Interval16{8, 9}, Interval16{10, 11}, Interval16{14, 17}}, + expected: []Interval16{Interval16{1, 20}}, expectedN: 20, }, { name: "no overlap", - a: []interval16{interval16{3, 4}, interval16{7, 8}}, - b: []interval16{interval16{1, 2}, interval16{5, 6}, interval16{9, 10}}, - expected: []interval16{interval16{1, 10}}, + a: []Interval16{Interval16{3, 4}, Interval16{7, 8}}, + b: []Interval16{Interval16{1, 2}, Interval16{5, 6}, Interval16{9, 10}}, + expected: []Interval16{Interval16{1, 10}}, expectedN: 10, }, { name: "b in a", - a: []interval16{interval16{1, 10}}, - b: []interval16{interval16{5, 7}}, - expected: []interval16{interval16{1, 10}}, + a: []Interval16{Interval16{1, 10}}, + b: []Interval16{Interval16{5, 7}}, + expected: []Interval16{Interval16{1, 10}}, expectedN: 10, }, { name: "a eq b", - a: []interval16{interval16{1, 10}}, - b: []interval16{interval16{1, 10}}, - expected: []interval16{interval16{1, 10}}, + a: []Interval16{Interval16{1, 10}}, + b: []Interval16{Interval16{1, 10}}, + expected: []Interval16{Interval16{1, 10}}, expectedN: 10, }, { name: "a in b", - a: []interval16{interval16{5, 7}}, - b: []interval16{interval16{1, 10}}, - expected: []interval16{interval16{1, 10}}, + a: []Interval16{Interval16{5, 7}}, + b: []Interval16{Interval16{1, 10}}, + expected: []Interval16{Interval16{1, 10}}, expectedN: 10, }, { name: "a ahead b", - a: []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}}, - b: []interval16{interval16{10, 11}, interval16{12, 13}, interval16{14, 15}}, - expected: []interval16{interval16{1, 7}, interval16{10, 15}}, + a: []Interval16{Interval16{1, 2}, Interval16{3, 4}, Interval16{5, 7}}, + b: []Interval16{Interval16{10, 11}, Interval16{12, 13}, Interval16{14, 15}}, + expected: []Interval16{Interval16{1, 7}, Interval16{10, 15}}, expectedN: 13, }, { name: "b ahead a", - a: []interval16{interval16{10, 11}, interval16{12, 13}, interval16{14, 15}}, - b: []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}}, - expected: []interval16{interval16{1, 7}, interval16{10, 15}}, + a: []Interval16{Interval16{10, 11}, Interval16{12, 13}, Interval16{14, 15}}, + b: []Interval16{Interval16{1, 2}, Interval16{3, 4}, Interval16{5, 7}}, + expected: []Interval16{Interval16{1, 7}, Interval16{10, 15}}, expectedN: 13, }, { name: "empty a and b", - a: []interval16{}, - b: []interval16{}, - expected: []interval16{}, + a: []Interval16{}, + b: []Interval16{}, + expected: []Interval16{}, expectedN: 0, }, { name: "empty a", - a: []interval16{}, - b: []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}}, - expected: []interval16{interval16{1, 7}}, + a: []Interval16{}, + b: []Interval16{Interval16{1, 2}, Interval16{3, 4}, Interval16{5, 7}}, + expected: []Interval16{Interval16{1, 7}}, expectedN: 7, }, { name: "empty b", - a: []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}}, - b: []interval16{}, - expected: []interval16{interval16{1, 7}}, + a: []Interval16{Interval16{1, 2}, Interval16{3, 4}, Interval16{5, 7}}, + b: []Interval16{}, + expected: []Interval16{Interval16{1, 7}}, expectedN: 7, }, { name: "single a", - a: []interval16{interval16{1, 2}}, - b: []interval16{}, - expected: []interval16{interval16{1, 2}}, + a: []Interval16{Interval16{1, 2}}, + b: []Interval16{}, + expected: []Interval16{Interval16{1, 2}}, expectedN: 2, }, { name: "single b", - a: []interval16{}, - b: []interval16{interval16{1, 2}}, - expected: []interval16{interval16{1, 2}}, + a: []Interval16{}, + b: []Interval16{Interval16{1, 2}}, + expected: []Interval16{Interval16{1, 2}}, expectedN: 2, }, { name: "single a single b", - a: []interval16{interval16{3, 4}}, - b: []interval16{interval16{1, 2}}, - expected: []interval16{interval16{1, 4}}, + a: []Interval16{Interval16{3, 4}}, + b: []Interval16{Interval16{1, 2}}, + expected: []Interval16{Interval16{1, 4}}, expectedN: 4, }, { name: "oddBitsSet lastBitUnset", - a: []interval16{interval16{1, 1}, interval16{3, 3}, interval16{5, 5}}, - b: []interval16{interval16{0, 4}}, - expected: []interval16{interval16{0, 5}}, + a: []Interval16{Interval16{1, 1}, Interval16{3, 3}, Interval16{5, 5}}, + b: []Interval16{Interval16{0, 4}}, + expected: []Interval16{Interval16{0, 5}}, expectedN: 6, }, { name: "all bits", - a: []interval16{interval16{1, 1}, interval16{3, 3}, interval16{5, 5}}, - b: []interval16{interval16{0, 0}, interval16{2, 2}, interval16{4, 4}}, - expected: []interval16{interval16{0, 5}}, + a: []Interval16{Interval16{1, 1}, Interval16{3, 3}, Interval16{5, 5}}, + b: []Interval16{Interval16{0, 0}, Interval16{2, 2}, Interval16{4, 4}}, + expected: []Interval16{Interval16{0, 5}}, expectedN: 6, }, { name: "short a long b", - a: []interval16{interval16{5, 5}, interval16{7, 7}, interval16{9, 10}, interval16{12, 12}, interval16{15, 17}, interval16{19, 20}}, - b: []interval16{interval16{1, 10}, interval16{12, 12}, interval16{14, 18}}, - expected: []interval16{interval16{1, 10}, interval16{12, 12}, interval16{14, 20}}, + a: []Interval16{Interval16{5, 5}, Interval16{7, 7}, Interval16{9, 10}, Interval16{12, 12}, Interval16{15, 17}, Interval16{19, 20}}, + b: []Interval16{Interval16{1, 10}, Interval16{12, 12}, Interval16{14, 18}}, + expected: []Interval16{Interval16{1, 10}, Interval16{12, 12}, Interval16{14, 20}}, expectedN: 18, }, { name: "common endings", - a: []interval16{interval16{1, 5}, interval16{15, 20}, interval16{25, 35}}, - b: []interval16{interval16{1, 10}, interval16{15, 20}, interval16{30, 35}}, - expected: []interval16{interval16{1, 10}, interval16{15, 20}, interval16{25, 35}}, + a: []Interval16{Interval16{1, 5}, Interval16{15, 20}, Interval16{25, 35}}, + b: []Interval16{Interval16{1, 10}, Interval16{15, 20}, Interval16{30, 35}}, + expected: []Interval16{Interval16{1, 10}, Interval16{15, 20}, Interval16{25, 35}}, expectedN: 27, }, { name: "common endings and overlap", - a: []interval16{interval16{1, 5}, interval16{10, 15}}, - b: []interval16{interval16{5, 10}, interval16{12, 17}}, - expected: []interval16{interval16{1, 17}}, + a: []Interval16{Interval16{1, 5}, Interval16{10, 15}}, + b: []Interval16{Interval16{5, 10}, Interval16{12, 17}}, + expected: []Interval16{Interval16{1, 17}}, expectedN: 17, }, { name: "no common endings and overlap", - a: []interval16{interval16{5, 10}, interval16{12, 17}}, - b: []interval16{interval16{0, 11}, interval16{15, 20}}, - expected: []interval16{interval16{0, 20}}, + a: []Interval16{Interval16{5, 10}, Interval16{12, 17}}, + b: []Interval16{Interval16{0, 11}, Interval16{15, 20}}, + expected: []Interval16{Interval16{0, 20}}, expectedN: 21, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - bb := make([]interval16, len(tc.b)) + bb := make([]Interval16, len(tc.b)) copy(bb, tc.b) runs, n := unionInterval16InPlace(tc.a, tc.b) @@ -904,7 +904,7 @@ func TestUnionInterval16InPlace(t *testing.T) { } func TestIntersectMixed(t *testing.T) { - a := NewContainerRun([]interval16{{start: 5, last: 10}}) + a := NewContainerRun([]Interval16{{Start: 5, Last: 10}}) b := NewContainerArray([]uint16{1, 4, 5, 7, 10, 11, 12}) c := NewContainerBitmap(2, []uint64{0x60}) @@ -917,8 +917,8 @@ func TestIntersectMixed(t *testing.T) { t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array()) } res = intersect(a, a) - if !reflect.DeepEqual(res.runs(), []interval16{{start: 5, last: 10}}) { - t.Fatalf("test #3 expected %v, but got %v", []interval16{{start: 5, last: 10}}, res.runs()) + if !reflect.DeepEqual(res.runs(), []Interval16{{Start: 5, Last: 10}}) { + t.Fatalf("test #3 expected %v, but got %v", []Interval16{{Start: 5, Last: 10}}, res.runs()) } res = intersect(c, a) @@ -942,7 +942,7 @@ func TestIntersectMixed(t *testing.T) { } func TestDifferenceMixed(t *testing.T) { - a := NewContainerRun([]interval16{{start: 5, last: 10}}) + a := NewContainerRun([]Interval16{{Start: 5, Last: 10}}) b := NewContainerArray([]uint16{0, 2, 4, 6, 8, 10, 12}) @@ -962,7 +962,7 @@ func TestDifferenceMixed(t *testing.T) { } res = difference(a, a) - if !reflect.DeepEqual(res.runs(), []interval16{}) { + if !reflect.DeepEqual(res.runs(), []Interval16{}) { t.Fatalf("test #3 expected empty but got %v", res.runs()) } @@ -972,8 +972,8 @@ func TestDifferenceMixed(t *testing.T) { } res = difference(a, c) - if !reflect.DeepEqual(res.runs(), []interval16{{start: 7, last: 10}}) { - t.Fatalf("test #5 expected %v, but got %v", []interval16{{start: 7, last: 10}}, res.runs()) + if !reflect.DeepEqual(res.runs(), []Interval16{{Start: 7, Last: 10}}) { + t.Fatalf("test #5 expected %v, but got %v", []Interval16{{Start: 7, Last: 10}}, res.runs()) } res = difference(b, c) @@ -1012,49 +1012,49 @@ func TestUnionRunRun(t *testing.T) { a := NewContainerRun(nil) b := NewContainerRun(nil) tests := []struct { - aruns []interval16 - bruns []interval16 - exp []interval16 + aruns []Interval16 + bruns []Interval16 + exp []Interval16 }{ { - aruns: []interval16{}, - bruns: []interval16{{start: 5, last: 10}}, - exp: []interval16{{start: 5, last: 10}}, + aruns: []Interval16{}, + bruns: []Interval16{{Start: 5, Last: 10}}, + exp: []Interval16{{Start: 5, Last: 10}}, }, { - aruns: []interval16{{start: 5, last: 12}}, - bruns: []interval16{{start: 5, last: 10}}, - exp: []interval16{{start: 5, last: 12}}, + aruns: []Interval16{{Start: 5, Last: 12}}, + bruns: []Interval16{{Start: 5, Last: 10}}, + exp: []Interval16{{Start: 5, Last: 12}}, }, { - aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}}, - bruns: []interval16{{start: 5, last: 10}}, - exp: []interval16{{start: 1, last: 3}, {start: 5, last: 12}}, + aruns: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 5}, {Start: 7, Last: 8}, {Start: 9, Last: 12}}, + bruns: []Interval16{{Start: 5, Last: 10}}, + exp: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 12}}, }, { - aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}}, - bruns: []interval16{{start: 2, last: 65535}}, - exp: []interval16{{start: 1, last: 65535}}, + aruns: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 5}, {Start: 7, Last: 8}, {Start: 9, Last: 12}}, + bruns: []Interval16{{Start: 2, Last: 65535}}, + exp: []Interval16{{Start: 1, Last: 65535}}, }, { - aruns: []interval16{{start: 2, last: 65535}}, - bruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}}, - exp: []interval16{{start: 1, last: 65535}}, + aruns: []Interval16{{Start: 2, Last: 65535}}, + bruns: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 5}, {Start: 7, Last: 8}, {Start: 9, Last: 12}}, + exp: []Interval16{{Start: 1, Last: 65535}}, }, { - aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}}, - bruns: []interval16{{start: 0, last: 65535}}, - exp: []interval16{{start: 0, last: 65535}}, + aruns: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 5}, {Start: 7, Last: 8}, {Start: 9, Last: 12}}, + bruns: []Interval16{{Start: 0, Last: 65535}}, + exp: []Interval16{{Start: 0, Last: 65535}}, }, { - aruns: []interval16{{start: 0, last: 65535}}, - bruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}}, - exp: []interval16{{start: 0, last: 65535}}, + aruns: []Interval16{{Start: 0, Last: 65535}}, + bruns: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 5}, {Start: 7, Last: 8}, {Start: 9, Last: 12}}, + exp: []Interval16{{Start: 0, Last: 65535}}, }, { - aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 9}, {start: 12, last: 22}}, - bruns: []interval16{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}}, - exp: []interval16{{start: 1, last: 9}, {start: 12, last: 27}, {start: 33, last: 34}}, + aruns: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 5}, {Start: 7, Last: 9}, {Start: 12, Last: 22}}, + bruns: []Interval16{{Start: 2, Last: 8}, {Start: 16, Last: 27}, {Start: 33, Last: 34}}, + exp: []Interval16{{Start: 1, Last: 9}, {Start: 12, Last: 27}, {Start: 33, Last: 34}}, }, } for i, test := range tests { @@ -1072,27 +1072,27 @@ func TestUnionArrayRun(t *testing.T) { b := NewContainerRun(nil) tests := []struct { array []uint16 - runs []interval16 + runs []Interval16 exp []uint16 }{ { array: []uint16{1, 4, 5, 7, 10, 11, 12}, - runs: []interval16{{start: 5, last: 10}}, + runs: []Interval16{{Start: 5, Last: 10}}, exp: []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}, }, { array: []uint16{}, - runs: []interval16{{start: 5, last: 10}}, + runs: []Interval16{{Start: 5, Last: 10}}, exp: []uint16{5, 6, 7, 8, 9, 10}, }, { array: []uint16{1, 4, 5, 7, 10, 11, 12}, - runs: []interval16{}, + runs: []Interval16{}, exp: []uint16{1, 4, 5, 7, 10, 11, 12}, }, { array: []uint16{0, 1, 4, 5, 7, 10, 11, 12}, - runs: []interval16{{start: 0, last: 5}, {start: 7, last: 7}}, + runs: []Interval16{{Start: 0, Last: 5}, {Start: 7, Last: 7}}, exp: []uint16{0, 1, 2, 3, 4, 5, 7, 10, 11, 12}, }, } @@ -1195,27 +1195,27 @@ func TestBitmapToArray(t *testing.T) { func TestRunToBitmap(t *testing.T) { tests := []struct { - runs []interval16 + runs []Interval16 exp []uint64 }{ { - runs: []interval16{}, + runs: []Interval16{}, exp: []uint64{}, }, { - runs: []interval16{{start: 0, last: 0}}, + runs: []Interval16{{Start: 0, Last: 0}}, exp: []uint64{1}, }, { - runs: []interval16{{start: 0, last: 4}}, + runs: []Interval16{{Start: 0, Last: 4}}, exp: []uint64{31}, }, { - runs: []interval16{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}}, + runs: []Interval16{{Start: 2, Last: 2}, {Start: 5, Last: 7}, {Start: 13, Last: 14}, {Start: 17, Last: 17}}, exp: []uint64{155876}, }, { - runs: []interval16{{start: 0, last: 3}, {start: 60, last: 67}}, + runs: []Interval16{{Start: 0, Last: 3}, {Start: 60, Last: 67}}, exp: []uint64{0xF00000000000000F, 0x000000000000000F}, }, } @@ -1247,55 +1247,55 @@ func getFullBitmap() []uint64 { func TestBitmapToRun(t *testing.T) { tests := []struct { bitmap []uint64 - exp []interval16 + exp []Interval16 }{ { // empty run bitmap: []uint64{}, - exp: []interval16{}, + exp: []Interval16{}, }, { // single-bit run bitmap: []uint64{1}, - exp: []interval16{{start: 0, last: 0}}, + exp: []Interval16{{Start: 0, Last: 0}}, }, { // single multi-bit run in one word bitmap: []uint64{31}, - exp: []interval16{{start: 0, last: 4}}, + exp: []Interval16{{Start: 0, Last: 4}}, }, { // multiple runs in one word bitmap: []uint64{155876}, - exp: []interval16{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}}, + exp: []Interval16{{Start: 2, Last: 2}, {Start: 5, Last: 7}, {Start: 13, Last: 14}, {Start: 17, Last: 17}}, }, { // span two words, both mixed bitmap: []uint64{0xF00000000000000F, 0x000000000000000F}, - exp: []interval16{{start: 0, last: 3}, {start: 60, last: 67}}, + exp: []Interval16{{Start: 0, Last: 3}, {Start: 60, Last: 67}}, }, { // span two words, first = maxBitmap bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xF}, - exp: []interval16{{start: 0, last: 67}}, + exp: []Interval16{{Start: 0, Last: 67}}, }, { // span two words, second = maxBitmap bitmap: []uint64{0xF000000000000000, 0xFFFFFFFFFFFFFFFF}, - exp: []interval16{{start: 60, last: 127}}, + exp: []Interval16{{Start: 60, Last: 127}}, }, { // span three words bitmap: []uint64{0xF000000000000000, 0xFFFFFFFFFFFFFFFF, 0xF}, - exp: []interval16{{start: 60, last: 131}}, + exp: []Interval16{{Start: 60, Last: 131}}, }, { bitmap: make([]uint64, bitmapN), - exp: []interval16{{start: 65408, last: 65535}}, + exp: []Interval16{{Start: 65408, Last: 65535}}, }, { bitmap: getFullBitmap(), - exp: []interval16{{start: 0, last: 65535}}, + exp: []Interval16{{Start: 0, Last: 65535}}, }, } tests[8].bitmap[1022] = 0xFFFFFFFFFFFFFFFF @@ -1318,23 +1318,23 @@ func TestBitmapToRun(t *testing.T) { func TestArrayToRun(t *testing.T) { tests := []struct { array []uint16 - exp []interval16 + exp []Interval16 }{ { array: []uint16{}, - exp: []interval16{}, + exp: []Interval16{}, }, { array: []uint16{0}, - exp: []interval16{{start: 0, last: 0}}, + exp: []Interval16{{Start: 0, Last: 0}}, }, { array: []uint16{0, 1, 2, 3, 4}, - exp: []interval16{{start: 0, last: 4}}, + exp: []Interval16{{Start: 0, Last: 4}}, }, { array: []uint16{2, 5, 6, 7, 13, 14, 17}, - exp: []interval16{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}}, + exp: []Interval16{{Start: 2, Last: 2}, {Start: 5, Last: 7}, {Start: 13, Last: 14}, {Start: 17, Last: 17}}, }, } @@ -1349,23 +1349,23 @@ func TestArrayToRun(t *testing.T) { func TestRunToArray(t *testing.T) { tests := []struct { - runs []interval16 + runs []Interval16 exp []uint16 }{ { - runs: []interval16{}, + runs: []Interval16{}, exp: []uint16{}, }, { - runs: []interval16{{start: 0, last: 0}}, + runs: []Interval16{{Start: 0, Last: 0}}, exp: []uint16{0}, }, { - runs: []interval16{{start: 0, last: 4}}, + runs: []Interval16{{Start: 0, Last: 4}}, exp: []uint16{0, 1, 2, 3, 4}, }, { - runs: []interval16{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}}, + runs: []Interval16{{Start: 2, Last: 2}, {Start: 5, Last: 7}, {Start: 13, Last: 14}, {Start: 17, Last: 17}}, exp: []uint16{2, 5, 6, 7, 13, 14, 17}, }, } @@ -1423,13 +1423,13 @@ func TestBitmapZeroRange(t *testing.T) { func TestUnionBitmapRun(t *testing.T) { tests := []struct { bitmap []uint64 - runs []interval16 + runs []Interval16 exp []uint64 expN int32 }{ { bitmap: []uint64{2}, - runs: []interval16{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 78}}, + runs: []Interval16{{Start: 0, Last: 0}, {Start: 2, Last: 5}, {Start: 62, Last: 71}, {Start: 77, Last: 78}}, exp: []uint64{0xC00000000000003F, 0x60FF}, expN: 18, }, @@ -1545,12 +1545,12 @@ func TestArrayCountRuns(t *testing.T) { func TestDifferenceArrayRun(t *testing.T) { tests := []struct { array []uint16 - runs []interval16 + runs []Interval16 exp []uint16 }{ { array: []uint16{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, - runs: []interval16{{start: 5, last: 10}}, + runs: []Interval16{{Start: 5, Last: 10}}, exp: []uint16{0, 1, 2, 3, 4, 11, 12}, }, } @@ -1566,54 +1566,54 @@ func TestDifferenceArrayRun(t *testing.T) { func TestDifferenceRunArray(t *testing.T) { tests := []struct { - runs []interval16 + runs []Interval16 array []uint16 - exp []interval16 + exp []Interval16 }{ { - runs: []interval16{{start: 0, last: 12}}, + runs: []Interval16{{Start: 0, Last: 12}}, array: []uint16{5, 6, 7, 8, 9, 10}, - exp: []interval16{{start: 0, last: 4}, {start: 11, last: 12}}, + exp: []Interval16{{Start: 0, Last: 4}, {Start: 11, Last: 12}}, }, { - runs: []interval16{{start: 0, last: 12}}, + runs: []Interval16{{Start: 0, Last: 12}}, array: []uint16{0, 1, 2, 3}, - exp: []interval16{{start: 4, last: 12}}, + exp: []Interval16{{Start: 4, Last: 12}}, }, { - runs: []interval16{{start: 0, last: 12}}, + runs: []Interval16{{Start: 0, Last: 12}}, array: []uint16{9, 10, 11, 12, 13}, - exp: []interval16{{start: 0, last: 8}}, + exp: []Interval16{{Start: 0, Last: 8}}, }, { - runs: []interval16{{start: 1, last: 12}}, + runs: []Interval16{{Start: 1, Last: 12}}, array: []uint16{0, 9, 10, 11, 12, 13}, - exp: []interval16{{start: 1, last: 8}}, + exp: []Interval16{{Start: 1, Last: 8}}, }, { - runs: []interval16{{start: 1, last: 12}, {start: 14, last: 14}, {start: 18, last: 18}}, + runs: []Interval16{{Start: 1, Last: 12}, {Start: 14, Last: 14}, {Start: 18, Last: 18}}, array: []uint16{0, 9, 10, 11, 12, 13, 14, 17}, - exp: []interval16{{start: 1, last: 8}, {start: 18, last: 18}}, + exp: []Interval16{{Start: 1, Last: 8}, {Start: 18, Last: 18}}, }, { - runs: []interval16{{start: 1, last: 12}, {start: 14, last: 14}, {start: 18, last: 18}}, + runs: []Interval16{{Start: 1, Last: 12}, {Start: 14, Last: 14}, {Start: 18, Last: 18}}, array: []uint16{0, 9, 10, 11, 12, 13, 14, 17, 19}, - exp: []interval16{{start: 1, last: 8}, {start: 18, last: 18}}, + exp: []Interval16{{Start: 1, Last: 8}, {Start: 18, Last: 18}}, }, { - runs: []interval16{{start: 1, last: 12}, {start: 14, last: 17}, {start: 19, last: 28}}, + runs: []Interval16{{Start: 1, Last: 12}, {Start: 14, Last: 17}, {Start: 19, Last: 28}}, array: []uint16{0, 9, 10, 11, 12, 13, 14, 17, 19, 25, 27}, - exp: []interval16{{start: 1, last: 8}, {start: 15, last: 16}, {start: 20, last: 24}, {start: 26, last: 26}, {start: 28, last: 28}}, + exp: []Interval16{{Start: 1, Last: 8}, {Start: 15, Last: 16}, {Start: 20, Last: 24}, {Start: 26, Last: 26}, {Start: 28, Last: 28}}, }, { - runs: []interval16{{start: 0, last: 20}, {start: 65533, last: 65535}}, + runs: []Interval16{{Start: 0, Last: 20}, {Start: 65533, Last: 65535}}, array: []uint16{65533, 65534, 65535}, - exp: []interval16{{start: 0, last: 20}}, + exp: []Interval16{{Start: 0, Last: 20}}, }, { - runs: []interval16{{start: 0, last: 20}, {start: 65530, last: 65535}}, + runs: []Interval16{{Start: 0, Last: 20}, {Start: 65530, Last: 65535}}, array: []uint16{37, 65535}, - exp: []interval16{{start: 0, last: 20}, {start: 65530, last: 65534}}, + exp: []Interval16{{Start: 0, Last: 20}, {Start: 65530, Last: 65534}}, }, } for i, test := range tests { @@ -1639,49 +1639,49 @@ func MakeLastBitSet() []uint64 { func TestDifferenceRunBitmap(t *testing.T) { tests := []struct { - runs []interval16 + runs []Interval16 bitmap []uint64 - exp []interval16 + exp []Interval16 }{ { - runs: []interval16{{start: 0, last: 63}}, + runs: []Interval16{{Start: 0, Last: 63}}, bitmap: MakeBitmap([]uint64{0x0000FFFF000000F0}), - exp: []interval16{{start: 0, last: 3}, {start: 8, last: 31}, {start: 48, last: 63}}, + exp: []Interval16{{Start: 0, Last: 3}, {Start: 8, Last: 31}, {Start: 48, Last: 63}}, }, { - runs: []interval16{{start: 0, last: 63}}, + runs: []Interval16{{Start: 0, Last: 63}}, bitmap: MakeBitmap([]uint64{0x8000000000000000}), - exp: []interval16{{start: 0, last: 62}}, + exp: []Interval16{{Start: 0, Last: 62}}, }, { - runs: []interval16{{start: 0, last: 63}}, + runs: []Interval16{{Start: 0, Last: 63}}, bitmap: MakeBitmap([]uint64{0x0000000000000001}), - exp: []interval16{{start: 1, last: 63}}, + exp: []Interval16{{Start: 1, Last: 63}}, }, { - runs: []interval16{{start: 0, last: 63}}, + runs: []Interval16{{Start: 0, Last: 63}}, bitmap: MakeBitmap([]uint64{0x0, 0x0000000000000001}), - exp: []interval16{{start: 0, last: 63}}, + exp: []Interval16{{Start: 0, Last: 63}}, }, { - runs: []interval16{{start: 0, last: 65}}, + runs: []Interval16{{Start: 0, Last: 65}}, bitmap: MakeBitmap([]uint64{0x0, 0x0000000000000001}), - exp: []interval16{{start: 0, last: 63}, {start: 65, last: 65}}, + exp: []Interval16{{Start: 0, Last: 63}, {Start: 65, Last: 65}}, }, { - runs: []interval16{{start: 0, last: 65}}, + runs: []Interval16{{Start: 0, Last: 65}}, bitmap: MakeBitmap([]uint64{0x0, 0x8000000000000000}), - exp: []interval16{{start: 0, last: 65}}, + exp: []Interval16{{Start: 0, Last: 65}}, }, { - runs: []interval16{{start: 1, last: 65535}}, + runs: []Interval16{{Start: 1, Last: 65535}}, bitmap: MakeBitmap([]uint64{0x0000000000000001}), - exp: []interval16{{start: 1, last: 65535}}, + exp: []Interval16{{Start: 1, Last: 65535}}, }, { - runs: []interval16{{start: 0, last: 65533}, {start: 65535, last: 65535}}, + runs: []Interval16{{Start: 0, Last: 65533}, {Start: 65535, Last: 65535}}, bitmap: MakeLastBitSet(), - exp: []interval16{{start: 0, last: 65533}}, + exp: []Interval16{{Start: 0, Last: 65533}}, }, } for i, test := range tests { @@ -1697,67 +1697,66 @@ func TestDifferenceRunBitmap(t *testing.T) { func TestDifferenceBitmapRun(t *testing.T) { tests := []struct { bitmap []uint64 - runs []interval16 + runs []Interval16 exp []uint64 }{ { bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, - runs: []interval16{{start: 4, last: 7}, {start: 32, last: 47}}, + runs: []Interval16{{Start: 4, Last: 7}, {Start: 32, Last: 47}}, exp: []uint64{0xFFFF0000FFFFFF0F}, }, { bitmap: []uint64{0xFFFFFFFFFFFFFFBF}, - runs: []interval16{{start: 0, last: 5}, {start: 7, last: 63}}, + runs: []Interval16{{Start: 0, Last: 5}, {Start: 7, Last: 63}}, exp: []uint64{0x0000000000000000}, }, { bitmap: []uint64{0xFFFFFFFFFFFFFFBF}, - runs: []interval16{{start: 0, last: 5}}, + runs: []Interval16{{Start: 0, Last: 5}}, exp: []uint64{0xFFFFFFFFFFFFFF80}, }, { bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, - runs: []interval16{{start: 60, last: 63}}, + runs: []Interval16{{Start: 60, Last: 63}}, exp: []uint64{0x0FFFFFFFFFFFFFFF}, }, { bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, - runs: []interval16{{start: 60, last: 65}}, + runs: []Interval16{{Start: 60, Last: 65}}, exp: []uint64{0x0FFFFFFFFFFFFFFF}, }, { bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF}, - runs: []interval16{{start: 60, last: 65}, {start: 67, last: 72}, {start: 126, last: 130}}, + runs: []Interval16{{Start: 60, Last: 65}, {Start: 67, Last: 72}, {Start: 126, Last: 130}}, exp: []uint64{0x0FFFFFFFFFFFFFFF, 0x3FFFFFFFFFFFFE04, 0xFFFFFFFFFFFFFFF8}, }, { bitmap: []uint64{0x0000000000000001}, - runs: []interval16{{start: 0, last: 0}}, + runs: []Interval16{{Start: 0, Last: 0}}, exp: []uint64{0x0000000000000000}, }, { bitmap: []uint64{0x8000000000000000}, - runs: []interval16{{start: 63, last: 63}}, + runs: []Interval16{{Start: 63, Last: 63}}, exp: []uint64{0x0000000000000000}, }, { bitmap: []uint64{0xC000000000000000, 0x0000000000000003}, - runs: []interval16{{start: 63, last: 64}}, + runs: []Interval16{{Start: 63, Last: 64}}, exp: []uint64{0x4000000000000000, 0x0000000000000002}, }, { bitmap: []uint64{0x0000000000000000}, - runs: []interval16{{start: 5, last: 7}}, + runs: []Interval16{{Start: 5, Last: 7}}, exp: []uint64{0x0000000000000000}, - }, - { + }, { bitmap: bitmapLastBitSet(), - runs: []interval16{{start: 65535, last: 65535}}, + runs: []Interval16{{Start: 65535, Last: 65535}}, exp: bitmapEmpty(), }, { bitmap: bitmapFull(), - runs: []interval16{{start: 0, last: 65535}}, + runs: []Interval16{{Start: 0, Last: 65535}}, exp: bitmapEmpty(), }, } @@ -1847,18 +1846,18 @@ func TestDifferenceBitmapBitmap(t *testing.T) { func TestDifferenceRunRun(t *testing.T) { tests := []struct { - aruns []interval16 - bruns []interval16 - exp []interval16 + aruns []Interval16 + bruns []Interval16 + exp []Interval16 expn int32 }{ { // this tests all six overlap combinations // A [ ] [ ] [ ] [ ] [ ] [ ] // B [ ] [ ] [ ] [ ] [ ] [ ] - aruns: []interval16{{start: 3, last: 6}, {start: 13, last: 16}, {start: 24, last: 26}, {start: 33, last: 38}, {start: 43, last: 46}, {start: 53, last: 56}}, - bruns: []interval16{{start: 1, last: 8}, {start: 11, last: 14}, {start: 21, last: 23}, {start: 35, last: 37}, {start: 44, last: 48}, {start: 57, last: 59}}, - exp: []interval16{{start: 15, last: 16}, {start: 24, last: 26}, {start: 33, last: 34}, {start: 38, last: 38}, {start: 43, last: 43}, {start: 53, last: 56}}, + aruns: []Interval16{{Start: 3, Last: 6}, {Start: 13, Last: 16}, {Start: 24, Last: 26}, {Start: 33, Last: 38}, {Start: 43, Last: 46}, {Start: 53, Last: 56}}, + bruns: []Interval16{{Start: 1, Last: 8}, {Start: 11, Last: 14}, {Start: 21, Last: 23}, {Start: 35, Last: 37}, {Start: 44, Last: 48}, {Start: 57, Last: 59}}, + exp: []Interval16{{Start: 15, Last: 16}, {Start: 24, Last: 26}, {Start: 33, Last: 34}, {Start: 38, Last: 38}, {Start: 43, Last: 43}, {Start: 53, Last: 56}}, expn: 13, }, } @@ -1951,7 +1950,7 @@ func TestWriteReadFullBitmap(t *testing.T) { } func TestWriteReadRun(t *testing.T) { - cr := NewContainerRun([]interval16{{start: 3, last: 13}, {start: 100, last: 109}}) + cr := NewContainerRun([]Interval16{{Start: 3, Last: 13}, {Start: 100, Last: 109}}) br := NewFileBitmap() br.Containers.Put(0, cr) br2 := NewFileBitmap() @@ -1977,19 +1976,19 @@ func TestXorArrayRun(t *testing.T) { }{ { a: NewContainerArray([]uint16{1, 5, 10, 11, 12}), - b: NewContainerRun([]interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}), + b: NewContainerRun([]Interval16{{Start: 2, Last: 10}, {Start: 12, Last: 13}, {Start: 15, Last: 16}}), exp: NewContainerArray([]uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}), }, { a: NewContainerArray([]uint16{1, 5, 10, 11, 12, 13, 14}), - b: NewContainerRun([]interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}), + b: NewContainerRun([]Interval16{{Start: 2, Last: 10}, {Start: 12, Last: 13}, {Start: 15, Last: 16}}), exp: NewContainerArray([]uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}), }, { a: NewContainerArray([]uint16{65535}), - b: NewContainerRun([]interval16{{start: 65534, last: 65535}}), + b: NewContainerRun([]Interval16{{Start: 65534, Last: 65535}}), exp: NewContainerArray([]uint16{65534}), }, { a: NewContainerArray([]uint16{65535}), - b: NewContainerRun([]interval16{{start: 65535, last: 65535}}), + b: NewContainerRun([]Interval16{{Start: 65535, Last: 65535}}), exp: NewContainerArray([]uint16{}), }, } @@ -2011,8 +2010,8 @@ func TestXorArrayRun(t *testing.T) { //special case that didn't fit the xorrunrun table testing below. func TestXorRunRun1(t *testing.T) { - a := NewContainerRun([]interval16{{start: 4, last: 10}}) - b := NewContainerRun([]interval16{{start: 5, last: 10}}) + a := NewContainerRun([]Interval16{{Start: 4, Last: 10}}) + b := NewContainerRun([]Interval16{{Start: 5, Last: 10}}) ret := xorRunRun(a, b) if !reflect.DeepEqual(ret.array(), []uint16{4}) { t.Fatalf("test #1 expected %v, but got %v", []uint16{4}, ret.array()) @@ -2027,84 +2026,84 @@ func TestXorRunRun(t *testing.T) { a := NewContainerRun(nil) b := NewContainerRun(nil) tests := []struct { - aruns []interval16 - bruns []interval16 - exp []interval16 + aruns []Interval16 + bruns []Interval16 + exp []Interval16 }{ { - aruns: []interval16{}, - bruns: []interval16{{start: 5, last: 10}}, - exp: []interval16{{start: 5, last: 10}}, + aruns: []Interval16{}, + bruns: []Interval16{{Start: 5, Last: 10}}, + exp: []Interval16{{Start: 5, Last: 10}}, }, { - aruns: []interval16{{start: 0, last: 4}}, - bruns: []interval16{{start: 6, last: 10}}, - exp: []interval16{{start: 0, last: 4}, {start: 6, last: 10}}, + aruns: []Interval16{{Start: 0, Last: 4}}, + bruns: []Interval16{{Start: 6, Last: 10}}, + exp: []Interval16{{Start: 0, Last: 4}, {Start: 6, Last: 10}}, }, { - aruns: []interval16{{start: 0, last: 6}}, - bruns: []interval16{{start: 4, last: 10}}, - exp: []interval16{{start: 0, last: 3}, {start: 7, last: 10}}, + aruns: []Interval16{{Start: 0, Last: 6}}, + bruns: []Interval16{{Start: 4, Last: 10}}, + exp: []Interval16{{Start: 0, Last: 3}, {Start: 7, Last: 10}}, }, { - aruns: []interval16{{start: 4, last: 10}}, - bruns: []interval16{{start: 0, last: 6}}, - exp: []interval16{{start: 0, last: 3}, {start: 7, last: 10}}, + aruns: []Interval16{{Start: 4, Last: 10}}, + bruns: []Interval16{{Start: 0, Last: 6}}, + exp: []Interval16{{Start: 0, Last: 3}, {Start: 7, Last: 10}}, }, { - aruns: []interval16{{start: 0, last: 10}}, - bruns: []interval16{{start: 0, last: 6}}, - exp: []interval16{{start: 7, last: 10}}, + aruns: []Interval16{{Start: 0, Last: 10}}, + bruns: []Interval16{{Start: 0, Last: 6}}, + exp: []Interval16{{Start: 7, Last: 10}}, }, { - aruns: []interval16{{start: 0, last: 6}}, - bruns: []interval16{{start: 0, last: 10}}, - exp: []interval16{{start: 7, last: 10}}, + aruns: []Interval16{{Start: 0, Last: 6}}, + bruns: []Interval16{{Start: 0, Last: 10}}, + exp: []Interval16{{Start: 7, Last: 10}}, }, { - aruns: []interval16{{start: 0, last: 6}}, - bruns: []interval16{{start: 0, last: 10}}, - exp: []interval16{{start: 7, last: 10}}, + aruns: []Interval16{{Start: 0, Last: 6}}, + bruns: []Interval16{{Start: 0, Last: 10}}, + exp: []Interval16{{Start: 7, Last: 10}}, }, { - aruns: []interval16{{start: 5, last: 12}}, - bruns: []interval16{{start: 5, last: 10}}, - exp: []interval16{{start: 11, last: 12}}, + aruns: []Interval16{{Start: 5, Last: 12}}, + bruns: []Interval16{{Start: 5, Last: 10}}, + exp: []Interval16{{Start: 11, Last: 12}}, }, { - aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, - bruns: []interval16{{start: 5, last: 10}}, - exp: []interval16{{start: 1, last: 3}, {start: 6, last: 6}, {start: 11, last: 12}}, + aruns: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 5}, {Start: 7, Last: 12}}, + bruns: []Interval16{{Start: 5, Last: 10}}, + exp: []Interval16{{Start: 1, Last: 3}, {Start: 6, Last: 6}, {Start: 11, Last: 12}}, }, { - aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, - bruns: []interval16{{start: 2, last: 65535}}, - exp: []interval16{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}}, + aruns: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 5}, {Start: 7, Last: 12}}, + bruns: []Interval16{{Start: 2, Last: 65535}}, + exp: []Interval16{{Start: 1, Last: 1}, {Start: 4, Last: 4}, {Start: 6, Last: 6}, {Start: 13, Last: 65535}}, }, { - aruns: []interval16{{start: 2, last: 65535}}, - bruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, - exp: []interval16{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}}, + aruns: []Interval16{{Start: 2, Last: 65535}}, + bruns: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 5}, {Start: 7, Last: 12}}, + exp: []Interval16{{Start: 1, Last: 1}, {Start: 4, Last: 4}, {Start: 6, Last: 6}, {Start: 13, Last: 65535}}, }, { - aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, - bruns: []interval16{{start: 0, last: 65535}}, - exp: []interval16{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}}, + aruns: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 5}, {Start: 7, Last: 12}}, + bruns: []Interval16{{Start: 0, Last: 65535}}, + exp: []Interval16{{Start: 0, Last: 0}, {Start: 4, Last: 4}, {Start: 6, Last: 6}, {Start: 13, Last: 65535}}, }, { - aruns: []interval16{{start: 0, last: 65535}}, - bruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, - exp: []interval16{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}}, + aruns: []Interval16{{Start: 0, Last: 65535}}, + bruns: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 5}, {Start: 7, Last: 12}}, + exp: []Interval16{{Start: 0, Last: 0}, {Start: 4, Last: 4}, {Start: 6, Last: 6}, {Start: 13, Last: 65535}}, }, { - aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 9}, {start: 12, last: 22}}, - bruns: []interval16{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}}, - exp: []interval16{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 9, last: 9}, {start: 12, last: 15}, {start: 23, last: 27}, {start: 33, last: 34}}, + aruns: []Interval16{{Start: 1, Last: 3}, {Start: 5, Last: 5}, {Start: 7, Last: 9}, {Start: 12, Last: 22}}, + bruns: []Interval16{{Start: 2, Last: 8}, {Start: 16, Last: 27}, {Start: 33, Last: 34}}, + exp: []Interval16{{Start: 1, Last: 1}, {Start: 4, Last: 4}, {Start: 6, Last: 6}, {Start: 9, Last: 9}, {Start: 12, Last: 15}, {Start: 23, Last: 27}, {Start: 33, Last: 34}}, }, { - aruns: []interval16{{start: 65530, last: 65535}}, - bruns: []interval16{{start: 65532, last: 65535}}, - exp: []interval16{{start: 65530, last: 65531}}, + aruns: []Interval16{{Start: 65530, Last: 65535}}, + bruns: []Interval16{{Start: 65532, Last: 65535}}, + exp: []Interval16{{Start: 65530, Last: 65531}}, }, } for i, test := range tests { @@ -2188,12 +2187,12 @@ func TestBitmapXorRange(t *testing.T) { func TestXorBitmapRun(t *testing.T) { tests := []struct { bitmap []uint64 - runs []interval16 + runs []Interval16 exp []uint64 }{ { bitmap: []uint64{0x0, 0x0, 0x0}, - runs: []interval16{{start: 129, last: 131}}, + runs: []Interval16{{Start: 129, Last: 131}}, exp: []uint64{0x0, 0x0, 0x00000000000000E}, }, } @@ -2523,7 +2522,7 @@ func TestIteratorVarious(t *testing.T) { func TestRunBinSearchContains(t *testing.T) { tests := []struct { - runs []interval16 + runs []Interval16 index uint16 exp struct { index int32 @@ -2531,7 +2530,7 @@ func TestRunBinSearchContains(t *testing.T) { } }{ { - runs: []interval16{{start: 0, last: 10}}, + runs: []Interval16{{Start: 0, Last: 10}}, index: uint16(3), exp: struct { index int32 @@ -2539,7 +2538,7 @@ func TestRunBinSearchContains(t *testing.T) { }{index: 0, found: true}, }, { - runs: []interval16{{start: 0, last: 10}}, + runs: []Interval16{{Start: 0, Last: 10}}, index: uint16(13), exp: struct { index int32 @@ -2547,7 +2546,7 @@ func TestRunBinSearchContains(t *testing.T) { }{index: 0, found: false}, }, { - runs: []interval16{{start: 0, last: 10}, {start: 20, last: 30}}, + runs: []Interval16{{Start: 0, Last: 10}, {Start: 20, Last: 30}}, index: uint16(13), exp: struct { index int32 @@ -2555,7 +2554,7 @@ func TestRunBinSearchContains(t *testing.T) { }{index: 0, found: false}, }, { - runs: []interval16{{start: 0, last: 10}, {start: 20, last: 30}}, + runs: []Interval16{{Start: 0, Last: 10}, {Start: 20, Last: 30}}, index: uint16(36), exp: struct { index int32 @@ -2576,55 +2575,55 @@ func TestRunBinSearchContains(t *testing.T) { func TestRunBinSearch(t *testing.T) { tests := []struct { - runs []interval16 + runs []Interval16 search uint16 exp bool expi int32 }{ { - runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + runs: []Interval16{{2, 10}, {50, 60}, {80, 90}}, search: 1, exp: false, expi: 0, }, { - runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + runs: []Interval16{{2, 10}, {50, 60}, {80, 90}}, search: 2, exp: true, expi: 0, }, { - runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + runs: []Interval16{{2, 10}, {50, 60}, {80, 90}}, search: 5, exp: true, expi: 0, }, { - runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + runs: []Interval16{{2, 10}, {50, 60}, {80, 90}}, search: 10, exp: true, expi: 0, }, { - runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + runs: []Interval16{{2, 10}, {50, 60}, {80, 90}}, search: 20, exp: false, expi: 1, }, { - runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + runs: []Interval16{{2, 10}, {50, 60}, {80, 90}}, search: 55, exp: true, expi: 1, }, { - runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + runs: []Interval16{{2, 10}, {50, 60}, {80, 90}}, search: 70, exp: false, expi: 2, }, { - runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + runs: []Interval16{{2, 10}, {50, 60}, {80, 90}}, search: 100, exp: false, expi: 3, @@ -3973,31 +3972,31 @@ func TestShiftBitmap(t *testing.T) { } func TestShiftRun(t *testing.T) { tests := []struct { - runs []interval16 + runs []Interval16 n int32 en int32 - exp []interval16 + exp []Interval16 carry bool }{ { - runs: []interval16{{start: 5, last: 10}}, + runs: []Interval16{{Start: 5, Last: 10}}, n: 5, en: 5, - exp: []interval16{{start: 6, last: 11}}, + exp: []Interval16{{Start: 6, Last: 11}}, carry: false, }, { - runs: []interval16{{start: 5, last: 65535}}, + runs: []Interval16{{Start: 5, Last: 65535}}, n: 65530, en: 65529, - exp: []interval16{{start: 6, last: 65535}}, + exp: []Interval16{{Start: 6, Last: 65535}}, carry: true, }, { - runs: []interval16{{start: 65535, last: 65535}}, + runs: []Interval16{{Start: 65535, Last: 65535}}, n: 1, en: 0, - exp: []interval16{}, + exp: []Interval16{}, carry: true, }, } @@ -4337,7 +4336,7 @@ func BenchmarkUnionRunRunInPlace(bm *testing.B) { runs := []struct { name string - fn func() []interval16 + fn func() []Interval16 }{ {"FirstBitSet", runFirstBitSet}, {"LastBitSet", runLastBitSet}, @@ -4376,7 +4375,7 @@ func BenchmarkUnionRunRunInPlace(bm *testing.B) { func TestUnionRunRunInPlaceBitwiseCompare(t *testing.T) { runs := []struct { name string - run []interval16 + run []Interval16 }{ {name: "FirstBitSet", run: runFirstBitSet()}, {name: "LastBitSet", run: runLastBitSet()}, diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 7dda0f5c6..8a0d3ead4 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -315,8 +315,9 @@ func TestBitmap_SliceRange(t *testing.T) { // Ensure a bitmap can loop over a set of values. func TestBitmap_ForEach(t *testing.T) { var a []uint64 - roaring.NewFileBitmap(1, 2, 3).ForEach(func(v uint64) { + _ = roaring.NewFileBitmap(1, 2, 3).ForEach(func(v uint64) error { a = append(a, v) + return nil }) if !reflect.DeepEqual(a, []uint64{1, 2, 3}) { t.Fatalf("unexpected values: %+v", a) @@ -326,8 +327,9 @@ func TestBitmap_ForEach(t *testing.T) { // Ensure a bitmap can loop over a set of values in a range. func TestBitmap_ForEachRange(t *testing.T) { var a []uint64 - roaring.NewFileBitmap(1, 2, 3, 4).ForEachRange(2, 4, func(v uint64) { + _ = roaring.NewFileBitmap(1, 2, 3, 4).ForEachRange(2, 4, func(v uint64) error { a = append(a, v) + return nil }) if !reflect.DeepEqual(a, []uint64{2, 3}) { t.Fatalf("unexpected values: %+v", a) diff --git a/roaring/unmarshal_binary.go b/roaring/unmarshal_binary.go index f95f2a41f..53aff37ee 100644 --- a/roaring/unmarshal_binary.go +++ b/roaring/unmarshal_binary.go @@ -52,7 +52,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) (err error) { case containerArray: newC = NewContainerArray((*[4096]uint16)(unsafe.Pointer(itrPointer))[:itrLen:itrLen]) case containerRun: - newC = NewContainerRunN((*[2048]interval16)(unsafe.Pointer(itrPointer))[:itrLen:itrLen], int32(itrN)) + newC = NewContainerRunN((*[2048]Interval16)(unsafe.Pointer(itrPointer))[:itrLen:itrLen], int32(itrN)) case containerBitmap: newC = NewContainerBitmapN((*[1024]uint64)(unsafe.Pointer(itrPointer))[:1024:itrLen], int32(itrN)) default: @@ -144,7 +144,7 @@ func InspectBinary(data []byte, mapped bool, info *BitmapInfo) (b *Bitmap, mappe case containerArray: newC = NewContainerArray((*[4096]uint16)(unsafe.Pointer(itrPointer))[:itrLen:itrLen]) case containerRun: - newC = NewContainerRunN((*[2048]interval16)(unsafe.Pointer(itrPointer))[:itrLen:itrLen], int32(itrN)) + newC = NewContainerRunN((*[2048]Interval16)(unsafe.Pointer(itrPointer))[:itrLen:itrLen], int32(itrN)) case containerBitmap: newC = NewContainerBitmapN((*[1024]uint64)(unsafe.Pointer(itrPointer))[:1024:itrLen], int32(itrN)) default: diff --git a/server/grpc.go b/server/grpc.go index 70df01429..8d7638da5 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -272,6 +272,9 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe return errToStatusError(err) } + // Obtain transaction. + tx := pilosa.NewMultiTxWithIndex(true, index) + var fields []*pilosa.Field for _, field := range index.Fields() { // exclude internal fields (starting with "_") @@ -440,7 +443,7 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe } } } else { - value, exists, err = field.StringValue(col) + value, exists, err = field.StringValue(tx, col) if err != nil { return errors.Wrap(err, "getting string field value for column") } @@ -689,7 +692,7 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe } } } else { - value, exists, err = field.StringValue(id) + value, exists, err = field.StringValue(tx, id) if err != nil { return errors.Wrap(err, "getting string field value for column") } diff --git a/server/handler_test.go b/server/handler_test.go index 479856fad..e893aa371 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -173,22 +173,32 @@ func TestHandler_Endpoints(t *testing.T) { } }) + tx, err := holder.Begin(true) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) if f, err := i0.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(0, 0, nil); err != nil { + } else if _, err := f.SetBit(tx, 0, 0, nil); err != nil { t.Fatal(err) } if f, err := i1.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(0, 0, nil); err != nil { + } else if _, err := f.SetBit(tx, 0, 0, nil); err != nil { t.Fatal(err) } if _, err := i0.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + t.Run("Schema", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) diff --git a/test/holder.go b/test/holder.go index 3d73a0bf0..b48016f9f 100644 --- a/test/holder.go +++ b/test/holder.go @@ -86,7 +86,9 @@ func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row { if err != nil { panic(err) } - row, err := f.Row(rowID) + tx := &pilosa.RoaringTx{Index: idx.Index} + + row, err := f.Row(tx, rowID) if err != nil { panic(err) } @@ -100,7 +102,9 @@ func (h *Holder) ReadRow(index, field string, rowID uint64) *pilosa.Row { if f == nil { panic(pilosa.ErrFieldNotFound) } - row, err := f.Row(rowID) + tx := &pilosa.RoaringTx{Field: f} + + row, err := f.Row(tx, rowID) if err != nil { panic(err) } @@ -122,7 +126,9 @@ func (h *Holder) RowTime(index, field string, rowID uint64, t time.Time, quantum if err != nil { panic(err) } - row, err := f.RowTime(rowID, t, quantum) + tx := &pilosa.RoaringTx{Index: idx.Index} + + row, err := f.RowTime(tx, rowID, t, quantum) if err != nil { panic(err) } @@ -141,7 +147,9 @@ func (h *Holder) SetBitTime(index, field string, rowID, columnID uint64, t *time if err != nil { panic(err) } - _, err = f.SetBit(rowID, columnID, t) + tx := &pilosa.RoaringTx{Index: idx.Index} + + _, err = f.SetBit(tx, rowID, columnID, t) if err != nil { panic(err) } @@ -154,7 +162,9 @@ func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) { if err != nil { panic(err) } - _, err = f.ClearBit(rowID, columnID) + tx := &pilosa.RoaringTx{Index: idx.Index} + + _, err = f.ClearBit(tx, rowID, columnID) if err != nil { panic(err) } @@ -175,7 +185,9 @@ func (h *Holder) SetValue(index, field string, columnID uint64, value int64) { if err != nil { panic(err) } - _, err = f.SetValue(columnID, value) + tx := &pilosa.RoaringTx{Index: idx.Index} + + _, err = f.SetValue(tx, columnID, value) if err != nil { panic(err) } @@ -188,7 +200,9 @@ func (h *Holder) Value(index, field string, columnID uint64) (int64, bool) { if err != nil { panic(err) } - val, exists, err := f.Value(columnID) + tx := &pilosa.RoaringTx{Index: idx.Index} + + val, exists, err := f.Value(tx, columnID) if err != nil { panic(err) } @@ -203,7 +217,9 @@ func (h *Holder) Range(index, field string, op pql.Token, predicate int64) *pilo if err != nil { panic(err) } - row, err := f.Range(field, op, predicate) + tx := &pilosa.RoaringTx{Index: idx.Index} + + row, err := f.Range(tx, field, op, predicate) if err != nil { panic(err) } diff --git a/tx.go b/tx.go new file mode 100644 index 000000000..b8b74d427 --- /dev/null +++ b/tx.go @@ -0,0 +1,433 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "fmt" + "sync" + + "github.com/pilosa/pilosa/v2/roaring" + "github.com/pkg/errors" +) + +type Tx interface { + Rollback() error + Commit() error + + RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) + + Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) + PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error + RemoveContainer(index, field, view string, shard uint64, key uint64) error + + Add(index, field, view string, shard uint64, a ...uint64) (changed bool, err error) + Remove(index, field, view string, shard uint64, a ...uint64) (changed bool, err error) + Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) + + ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) + ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error + ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error + + Count(index, field, view string, shard uint64) (uint64, error) + Max(index, field, view string, shard uint64) (uint64, error) + Min(index, field, view string, shard uint64) (uint64, bool, error) + UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error + CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) + OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) +} + +// MultiTx implements the transaction interface to combine multiple transactions. +type MultiTx struct { + mu sync.Mutex + writable bool + holder *Holder + index *Index + txs map[multiTxKey]Tx +} + +// NewMultiTx returns a new instance of MultiTx for a Holder. +func NewMultiTx(writable bool, holder *Holder) *MultiTx { + return &MultiTx{ + writable: writable, + holder: holder, + txs: make(map[multiTxKey]Tx), + } +} + +// NewMultiTxWithIndex returns a new instance of MultiTx for a single index. +func NewMultiTxWithIndex(writable bool, index *Index) *MultiTx { + return &MultiTx{ + writable: writable, + index: index, + txs: make(map[multiTxKey]Tx), + } +} + +var _ Tx = (*MultiTx)(nil) + +// Rollback rolls back all underlying transactions. +func (mtx *MultiTx) Rollback() (err error) { + for _, tx := range mtx.txs { + if e := tx.Rollback(); e != nil && err == nil { + err = e + } + } + return err +} + +// Commit commits all underlying transactions. +func (mtx *MultiTx) Commit() (err error) { + for _, tx := range mtx.txs { + if e := tx.Commit(); e != nil && err == nil { + err = e + } + } + return err +} + +func (mtx *MultiTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { + tx, err := mtx.tx(index, shard) + if err != nil { + return nil, err + } + return tx.RoaringBitmap(index, field, view, shard) +} + +func (mtx *MultiTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) { + tx, err := mtx.tx(index, shard) + if err != nil { + return nil, err + } + return tx.Container(index, field, view, shard, key) +} + +func (mtx *MultiTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error { + tx, err := mtx.tx(index, shard) + if err != nil { + return err + } + return tx.PutContainer(index, field, view, shard, key, c) +} + +func (mtx *MultiTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error { + tx, err := mtx.tx(index, shard) + if err != nil { + return err + } + return tx.RemoveContainer(index, field, view, shard, key) +} + +func (mtx *MultiTx) Add(index, field, view string, shard uint64, a ...uint64) (changed bool, err error) { + tx, err := mtx.tx(index, shard) + if err != nil { + return false, err + } + return tx.Add(index, field, view, shard, a...) +} + +func (mtx *MultiTx) Remove(index, field, view string, shard uint64, a ...uint64) (changed bool, err error) { + tx, err := mtx.tx(index, shard) + if err != nil { + return false, err + } + return tx.Remove(index, field, view, shard, a...) +} + +func (mtx *MultiTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) { + tx, err := mtx.tx(index, shard) + if err != nil { + return false, err + } + return tx.Contains(index, field, view, shard, v) +} + +func (mtx *MultiTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) { + tx, err := mtx.tx(index, shard) + if err != nil { + return nil, false, err + } + return tx.ContainerIterator(index, field, view, shard, key) +} + +func (mtx *MultiTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { + tx, err := mtx.tx(index, shard) + if err != nil { + return err + } + return tx.ForEach(index, field, view, shard, fn) +} + +func (mtx *MultiTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { + tx, err := mtx.tx(index, shard) + if err != nil { + return err + } + return tx.ForEachRange(index, field, view, shard, start, end, fn) +} + +func (mtx *MultiTx) Count(index, field, view string, shard uint64) (uint64, error) { + tx, err := mtx.tx(index, shard) + if err != nil { + return 0, err + } + return tx.Count(index, field, view, shard) +} + +func (mtx *MultiTx) Max(index, field, view string, shard uint64) (uint64, error) { + tx, err := mtx.tx(index, shard) + if err != nil { + return 0, err + } + return tx.Max(index, field, view, shard) +} + +func (mtx *MultiTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { + tx, err := mtx.tx(index, shard) + if err != nil { + return 0, false, err + } + return tx.Min(index, field, view, shard) +} + +func (mtx *MultiTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { + tx, err := mtx.tx(index, shard) + if err != nil { + return err + } + return tx.UnionInPlace(index, field, view, shard, others...) +} + +func (mtx *MultiTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) { + tx, err := mtx.tx(index, shard) + if err != nil { + return 0, err + } + return tx.CountRange(index, field, view, shard, start, end) +} + +func (mtx *MultiTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) { + tx, err := mtx.tx(index, shard) + if err != nil { + return nil, err + } + return tx.OffsetRange(index, field, view, shard, offset, start, end) +} + +// tx returns a transaction by index/shard. Reuses transaction if already open. +// Otherwise begins a new transaction. +func (mtx *MultiTx) tx(index string, shard uint64) (_ Tx, err error) { + mtx.mu.Lock() + defer mtx.mu.Unlock() + + // Lookup transaction from cache. + tx := mtx.txs[multiTxKey{index, shard}] + if tx != nil { + return tx, nil + } + + // If transaction doesn't exist, lookup the index. + idx := mtx.index + if mtx.holder != nil { + if idx = mtx.holder.Index(index); idx == nil { + return nil, ErrIndexNotFound + } + } + + // Begin tranaction & cache it. + if tx, err = idx.Begin(mtx.writable, shard); err != nil { + return nil, err + } + mtx.txs[multiTxKey{index, shard}] = tx + + return tx, nil +} + +type multiTxKey struct { + index string + shard uint64 +} + +// RoaringTx represents a fake transaction object for Roaring storage. +type RoaringTx struct { + Index *Index + Field *Field + fragment *fragment +} + +// Rollback is a no-op as Roaring does not support transactions. +func (tx *RoaringTx) Rollback() error { + return nil +} + +// Commit is a no-op as Roaring does not support transactions. +func (tx *RoaringTx) Commit() error { + return nil +} + +func (tx *RoaringTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { + return tx.bitmap(field, view, shard) +} + +func (tx *RoaringTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return nil, err + } + return b.Containers.Get(key), nil +} + +func (tx *RoaringTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return err + } + b.Containers.Put(key, c) + return nil +} + +func (tx *RoaringTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return err + } + b.Containers.Remove(key) + return nil +} + +func (tx *RoaringTx) Add(index, field, view string, shard uint64, a ...uint64) (changed bool, err error) { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return false, err + } + return b.Add(a...) +} + +func (tx *RoaringTx) Remove(index, field, view string, shard uint64, a ...uint64) (changed bool, err error) { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return false, err + } + return b.Remove(a...) +} + +func (tx *RoaringTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return false, err + } + return b.Contains(v), nil +} + +func (tx *RoaringTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return nil, false, err + } + citer, found = b.Containers.Iterator(key) + return citer, found, nil +} + +func (tx *RoaringTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return err + } + return b.ForEach(fn) +} + +func (tx *RoaringTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return err + } + return b.ForEachRange(start, end, fn) +} + +func (tx *RoaringTx) Count(index, field, view string, shard uint64) (uint64, error) { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return 0, err + } + return b.Count(), nil +} + +func (tx *RoaringTx) Max(index, field, view string, shard uint64) (uint64, error) { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return 0, err + } + return b.Max(), nil +} + +func (tx *RoaringTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return 0, false, err + } + v, ok := b.Min() + return v, ok, nil +} + +func (tx *RoaringTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return err + } + b.UnionInPlace(others...) + return nil +} + +func (tx *RoaringTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return 0, err + } + return b.CountRange(start, end), nil +} + +func (tx *RoaringTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) { + b, err := tx.bitmap(field, view, shard) + if err != nil { + return nil, err + } + return b.OffsetRange(offset, start, end), nil +} + +func (tx *RoaringTx) bitmap(field, view string, shard uint64) (*roaring.Bitmap, error) { + // If a fragment is attached, always use it. + if tx.fragment != nil { + return tx.fragment.storage, nil + } + + // If a field is attached, start from there. + // Otherwise look up the field from the index. + f := tx.Field + if f == nil { + if f = tx.Index.Field(field); f == nil { + return nil, ErrFieldNotFound + } + } + + v := f.view(view) + if v == nil { + return nil, errors.Errorf("view not found: %q", view) + } + + frag := v.Fragment(shard) + if frag == nil { + panic(fmt.Sprintf("fragment not found: %q / %q / %d", field, view, shard)) + } + return frag.storage, nil +} diff --git a/utils_internal_test.go b/utils_internal_test.go index 83751f4bf..386282adb 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -133,8 +133,21 @@ func (t *ClusterCluster) SetBit(index, field string, rowID, colID uint64, x *tim if f == nil { return fmt.Errorf("index/field does not exist: %s/%s", index, field) } - _, err := f.SetBit(rowID, colID, x) - if err != nil { + + if err := func() error { + tx, err := c.holder.Begin(true) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + if _, err := f.SetBit(tx, rowID, colID, x); err != nil { + return err + } else if err := tx.Commit(); err != nil { + return err + } + return nil + }(); err != nil { return err } } diff --git a/view.go b/view.go index af73b7fe2..f127338c3 100644 --- a/view.go +++ b/view.go @@ -402,74 +402,76 @@ func (v *view) deleteFragment(shard uint64) error { } // row returns a row for a shard of the view. -func (v *view) row(rowID uint64) *Row { +func (v *view) row(tx Tx, rowID uint64) (*Row, error) { row := NewRow() for _, frag := range v.allFragments() { - fr := frag.row(rowID) - if fr == nil { + fr, err := frag.row(tx, rowID) + if err != nil { + return nil, err + } else if fr == nil { continue } row.Merge(fr) } - return row + return row, nil } // setBit sets a bit within the view. -func (v *view) setBit(rowID, columnID uint64) (changed bool, err error) { +func (v *view) setBit(tx Tx, rowID, columnID uint64) (changed bool, err error) { shard := columnID / ShardWidth frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return changed, err } - return frag.setBit(rowID, columnID) + return frag.setBit(tx, rowID, columnID) } // clearBit clears a bit within the view. -func (v *view) clearBit(rowID, columnID uint64) (changed bool, err error) { +func (v *view) clearBit(tx Tx, rowID, columnID uint64) (changed bool, err error) { shard := columnID / ShardWidth frag := v.Fragment(shard) if frag == nil { return false, nil } - return frag.clearBit(rowID, columnID) + return frag.clearBit(tx, rowID, columnID) } // value uses a column of bits to read a multi-bit value. -func (v *view) value(columnID uint64, bitDepth uint) (value int64, exists bool, err error) { +func (v *view) value(tx Tx, columnID uint64, bitDepth uint) (value int64, exists bool, err error) { shard := columnID / ShardWidth frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return value, exists, err } - return frag.value(columnID, bitDepth) + return frag.value(tx, columnID, bitDepth) } // setValue uses a column of bits to set a multi-bit value. -func (v *view) setValue(columnID uint64, bitDepth uint, value int64) (changed bool, err error) { +func (v *view) setValue(tx Tx, columnID uint64, bitDepth uint, value int64) (changed bool, err error) { shard := columnID / ShardWidth frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return changed, err } - return frag.setValue(columnID, bitDepth, value) + return frag.setValue(tx, columnID, bitDepth, value) } // clearValue removes a specific value assigned to columnID -func (v *view) clearValue(columnID uint64, bitDepth uint, value int64) (changed bool, err error) { +func (v *view) clearValue(tx Tx, columnID uint64, bitDepth uint, value int64) (changed bool, err error) { shard := columnID / ShardWidth frag := v.Fragment(shard) if frag == nil { return false, nil } - return frag.clearValue(columnID, bitDepth, value) + return frag.clearValue(tx, columnID, bitDepth, value) } // rangeOp returns rows with a field value encoding matching the predicate. -func (v *view) rangeOp(op pql.Token, bitDepth uint, predicate int64) (*Row, error) { +func (v *view) rangeOp(tx Tx, op pql.Token, bitDepth uint, predicate int64) (*Row, error) { r := NewRow() for _, frag := range v.allFragments() { - other, err := frag.rangeOp(op, bitDepth, predicate) + other, err := frag.rangeOp(tx, op, bitDepth, predicate) if err != nil { return nil, err }