diff --git a/api.go b/api.go index 56eaece56..3edb1b915 100644 --- a/api.go +++ b/api.go @@ -306,7 +306,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin cw := csv.NewWriter(w) // Iterate over each column. - if err := f.ForEachBit(func(rowID, columnID uint64) error { + if err := f.forEachBit(func(rowID, columnID uint64) error { return cw.Write([]string{ strconv.FormatUint(rowID, 10), strconv.FormatUint(columnID, 10), @@ -403,7 +403,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } var resp = internal.BlockDataResponse{} - resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) + resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block)) // Encode response. buf, err := proto.Marshal(&resp) diff --git a/cluster.go b/cluster.go index 81ac03424..a5574e2cd 100644 --- a/cluster.go +++ b/cluster.go @@ -633,7 +633,7 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost { for _, field := range idx.Fields() { for _, view := range field.Views() { - fieldViews.addView(field.Name(), view.Name()) + fieldViews.addView(field.Name(), view.name) } } diff --git a/executor.go b/executor.go index 4e2ffe9bf..25fc98ac9 100644 --- a/executor.go +++ b/executor.go @@ -399,7 +399,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq return ValCount{}, nil } - vsum, vcount, err := fragment.Sum(filter, bsig.BitDepth()) + vsum, vcount, err := fragment.sum(filter, bsig.BitDepth()) if err != nil { return ValCount{}, errors.Wrap(err, "computing sum") } @@ -437,7 +437,7 @@ func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fmin, fcount, err := fragment.Min(filter, bsig.BitDepth()) + fmin, fcount, err := fragment.min(filter, bsig.BitDepth()) if err != nil { return ValCount{}, err } @@ -475,7 +475,7 @@ func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fmax, fcount, err := fragment.Max(filter, bsig.BitDepth()) + fmax, fcount, err := fragment.max(filter, bsig.BitDepth()) if err != nil { return ValCount{}, err } @@ -602,7 +602,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca if tanimotoThreshold > 100 { return nil, errors.New("Tanimoto Threshold is from 1 to 100 only") } - return f.Top(TopOptions{ + return f.top(TopOptions{ N: int(n), Src: src, RowIDs: rowIDs, @@ -664,7 +664,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. if frag == nil { return NewRow(), nil } - return frag.Row(rowID), nil + return frag.row(rowID), nil } // executeIntersectSlice executes a intersect() call for a local slice. @@ -756,7 +756,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C if f == nil { continue } - row = row.Union(f.Row(rowID)) + row = row.Union(f.row(rowID)) } f.Stats.Count("range", 1, 1.0) return row, nil @@ -808,7 +808,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, return NewRow(), nil } - return frag.NotNull(bsig.BitDepth()) + return frag.notNull(bsig.BitDepth()) } else if cond.Op == pql.BETWEEN { @@ -846,10 +846,10 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, // If the query is asking for the entire valid range, just return // the not-null bitmap for the bsiGroup. if predicates[0] <= bsig.Min && predicates[1] >= bsig.Max { - return frag.NotNull(bsig.BitDepth()) + return frag.notNull(bsig.BitDepth()) } - return frag.RangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax) + return frag.rangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax) } else { @@ -879,16 +879,16 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range. if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) || (cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) { - return frag.NotNull(bsig.BitDepth()) + return frag.notNull(bsig.BitDepth()) } // outOfRange for NEQ should return all not-null. if outOfRange && cond.Op == pql.NEQ { - return frag.NotNull(bsig.BitDepth()) + return frag.notNull(bsig.BitDepth()) } f.Stats.Count("range:bsigroup", 1, 1.0) - return frag.RangeOp(cond.Op, bsig.BitDepth(), baseValue) + return frag.rangeOp(cond.Op, bsig.BitDepth(), baseValue) } } diff --git a/executor_test.go b/executor_test.go index 01c99e9c5..221a554e8 100644 --- a/executor_test.go +++ b/executor_test.go @@ -107,11 +107,11 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { func TestExecutor_Execute_Difference(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) + hldr.SetBit("i", "general", 10, 1) + hldr.SetBit("i", "general", 10, 2) + hldr.SetBit("i", "general", 10, 3) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, 4) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { @@ -125,7 +125,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { func TestExecutor_Execute_Empty_Difference(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + hldr.SetBit("i", "general", 10, 1) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil { @@ -137,13 +137,13 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { func TestExecutor_Execute_Intersect(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.SetBit("i", "general", 10, 1) + hldr.SetBit("i", "general", 10, SliceWidth+1) + hldr.SetBit("i", "general", 10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.SetBit("i", "general", 11, 1) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { @@ -168,12 +168,12 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { func TestExecutor_Execute_Union(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, SliceWidth+1) + hldr.SetBit("i", "general", 10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { @@ -187,7 +187,7 @@ func TestExecutor_Execute_Union(t *testing.T) { func TestExecutor_Execute_Empty_Union(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + hldr.SetBit("i", "general", 10, 0) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil { @@ -201,12 +201,12 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { func TestExecutor_Execute_Xor(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, SliceWidth+1) + hldr.SetBit("i", "general", 10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { @@ -220,9 +220,9 @@ func TestExecutor_Execute_Xor(t *testing.T) { func TestExecutor_Execute_Count(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.SetBit("i", "f", 10, 3) + hldr.SetBit("i", "f", 10, SliceWidth+1) + hldr.SetBit("i", "f", 10, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil { @@ -237,9 +237,11 @@ func TestExecutor_Execute_SetBit(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() + // set a bit so the view gets created. + hldr.SetBit("i", "f", 1, 0) + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - if n := f.Row(11).Count(); n != 0 { + if n := hldr.Row("i", "f", 11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) } @@ -251,7 +253,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { } } - if n := f.Row(11).Count(); n != 1 { + if n := hldr.Row("i", "f", 11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil { @@ -427,12 +429,12 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { defer hldr.Close() // Set columns for rows 0, 10, & 20 across two slices. - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 0, 2) + hldr.SetBit("i", "f", 0, SliceWidth) + hldr.SetBit("i", "f", 1, SliceWidth+2) + hldr.SetBit("i", "f", 1, SliceWidth) // Execute query. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) @@ -450,23 +452,23 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, SliceWidth) + hldr.SetBit("i", "f", 0, 2*SliceWidth) + hldr.SetBit("i", "f", 0, 3*SliceWidth) + hldr.SetBit("i", "f", 0, 4*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 1) + hldr.SetBit("i", "f", 1, 0) + hldr.SetBit("i", "f", 1, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1) + hldr.SetBit("i", "f", 2, SliceWidth) + hldr.SetBit("i", "f", 2, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1) + hldr.SetBit("i", "f", 3, 2*SliceWidth) + hldr.SetBit("i", "f", 3, 2*SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1) + hldr.SetBit("i", "f", 4, 3*SliceWidth) + hldr.SetBit("i", "f", 4, 3*SliceWidth+1) // Execute query. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) @@ -485,19 +487,19 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { defer hldr.Close() // Set columns for rows 0, 10, & 20 across two slices. - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 0, SliceWidth) + hldr.SetBit("i", "f", 10, SliceWidth) + hldr.SetBit("i", "f", 10, SliceWidth+1) + hldr.SetBit("i", "f", 20, SliceWidth) + hldr.SetBit("i", "f", 20, SliceWidth+1) + hldr.SetBit("i", "f", 20, SliceWidth+2) // Create an intersecting row. - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2) + hldr.SetBit("i", "other", 100, SliceWidth) + hldr.SetBit("i", "other", 100, SliceWidth+1) + hldr.SetBit("i", "other", 100, SliceWidth+2) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() @@ -521,9 +523,9 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { // hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 10, SliceWidth) if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) @@ -544,9 +546,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 10, SliceWidth) if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) @@ -989,7 +991,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) + hldr.SetBit("i", "f", 10, SliceWidth+1) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil { @@ -1023,8 +1025,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) + hldr.SetBit("i", "f", 10, (2*SliceWidth)+1) + hldr.SetBit("i", "f", 10, (2*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil { @@ -1078,7 +1080,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } // Verify that one column is set on both node's holder. - if n := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 { + if n := hldr.Row("i", "f", 10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -1132,7 +1134,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } // Verify that one column is set on both node's holder. - if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 { + if n := hldr.ViewRow("i", "f", "standard_2016", 10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -1192,8 +1194,8 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2) + hldr.SetBit("i", "f", 30, (2*SliceWidth)+1) + hldr.SetBit("i", "f", 30, (4*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=3)`), nil, nil); err != nil { diff --git a/field.go b/field.go index 0e1f79d66..8c7683cd1 100644 --- a/field.go +++ b/field.go @@ -137,7 +137,7 @@ func (f *Field) MaxSlice() uint64 { var max uint64 for _, view := range f.views { - if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max { + if viewMaxSlice := view.calculateMaxSlice(); viewMaxSlice > max { max = viewMaxSlice } } @@ -249,11 +249,11 @@ func (f *Field) openViews() error { name := filepath.Base(fi.Name()) view := f.newView(f.ViewPath(name), name) - if err := view.Open(); err != nil { - return fmt.Errorf("opening view: view=%s, err=%s", view.Name(), err) + if err := view.open(); err != nil { + return fmt.Errorf("opening view: view=%s, err=%s", view.name, err) } view.RowAttrStore = f.rowAttrStore - f.views[view.Name()] = view + f.views[view.name] = view } return nil @@ -369,7 +369,7 @@ func (f *Field) Close() error { // Close all views. for _, view := range f.views { - if err := view.Close(); err != nil { + if err := view.close(); err != nil { return err } } @@ -447,9 +447,9 @@ func (f *Field) deleteBSIGroupAndView(name string) error { if view := f.views[viewName]; view != nil { delete(f.views, viewName) - if err := view.Close(); err != nil { + if err := view.close(); err != nil { return errors.Wrap(err, "closing view") - } else if err := os.RemoveAll(view.Path()); err != nil { + } else if err := os.RemoveAll(view.path); err != nil { return errors.Wrap(err, "deleting directory") } } @@ -538,7 +538,7 @@ func (f *Field) viewNames() []string { // RecalculateCaches recalculates caches on every view in the field. func (f *Field) RecalculateCaches() { for _, view := range f.Views() { - view.RecalculateCaches() + view.recalculateCaches() } } @@ -579,11 +579,11 @@ func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) { view := f.newView(f.ViewPath(name), name) - if err := view.Open(); err != nil { + if err := view.open(); err != nil { return nil, false, errors.Wrap(err, "opening view") } view.RowAttrStore = f.rowAttrStore - f.views[view.Name()] = view + f.views[view.name] = view return view, true, nil } @@ -606,12 +606,12 @@ func (f *Field) DeleteView(name string) error { } // Close data files before deletion. - if err := view.Close(); err != nil { + if err := view.close(); err != nil { return errors.Wrap(err, "closing view") } // Delete view directory. - if err := os.RemoveAll(view.Path()); err != nil { + if err := os.RemoveAll(view.path); err != nil { return errors.Wrap(err, "deleting directory") } @@ -620,6 +620,28 @@ func (f *Field) DeleteView(name string) error { return nil } +// Row returns a row of the standard view. +func (f *Field) Row(rowID uint64) (*Row, error) { + if f.Type() != FieldTypeSet { + return nil, errors.Errorf("row method unsupported for field type: %s", f.Type()) + } + view := f.View(ViewStandard) + if view == nil { + return nil, ErrInvalidView + } + return view.row(rowID), nil +} + +// ViewRow returns a row for a view and slice. +// TODO: unexport this with views (it's only used in tests). +func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) { + view := f.View(viewName) + if view == nil { + return nil, ErrInvalidView + } + return view.row(rowID), nil +} + // SetBit sets a bit on a view within the field. func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. @@ -634,7 +656,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed } // Set non-time bit. - if v, err := view.SetBit(rowID, colID); err != nil { + if v, err := view.setBit(rowID, colID); err != nil { return changed, errors.Wrap(err, "setting on view") } else if v { changed = v @@ -652,7 +674,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed return changed, errors.Wrapf(err, "creating view %s", subname) } - if c, err := view.SetBit(rowID, colID); err != nil { + if c, err := view.setBit(rowID, colID); err != nil { return changed, errors.Wrapf(err, "setting on view %s", subname) } else if c { changed = true @@ -676,7 +698,7 @@ func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (change } // Clear non-time bit. - if v, err := view.ClearBit(rowID, colID); err != nil { + if v, err := view.clearBit(rowID, colID); err != nil { return changed, errors.Wrap(err, "clearing on view") } else if v { changed = v @@ -694,7 +716,7 @@ func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (change return changed, errors.Wrapf(err, "creating view %s", subname) } - if c, err := view.ClearBit(rowID, colID); err != nil { + if c, err := view.clearBit(rowID, colID); err != nil { return changed, errors.Wrapf(err, "clearing on view %s", subname) } else if c { changed = true @@ -905,7 +927,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro return errors.Wrap(err, "creating view") } - if err := frag.Import(data.RowIDs, data.ColumnIDs); err != nil { + if err := frag.bulkImport(data.RowIDs, data.ColumnIDs); err != nil { return err } } @@ -962,7 +984,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { baseValues[i] = uint64(value - bsig.Min) } - if err := frag.ImportValue(data.ColumnIDs, baseValues, bsig.BitDepth()); err != nil { + if err := frag.importValue(data.ColumnIDs, baseValues, bsig.BitDepth()); err != nil { return err } } diff --git a/fragment.go b/fragment.go index afa80f5f1..830033c63 100644 --- a/fragment.go +++ b/fragment.go @@ -130,27 +130,8 @@ func NewFragment(path, index, field, view string, slice uint64) *Fragment { } } -// Path returns the path the fragment was initialized with. -func (f *Fragment) Path() string { return f.path } - -// CachePath returns the path to the fragment's cache data. -func (f *Fragment) CachePath() string { return f.path + CacheExt } - -// Index returns the index that the fragment was initialized with. -func (f *Fragment) Index() string { return f.index } - -// Field returns the field the fragment was initialized with. -func (f *Fragment) Field() string { return f.field } - -// View returns the view the fragment was initialized with. -func (f *Fragment) View() string { return f.view } - -// Slice returns the slice the fragment was initialized with. -func (f *Fragment) Slice() uint64 { return f.slice } - -// Cache returns the fragment's cache. -// This is not safe for concurrent use. -func (f *Fragment) Cache() Cache { return f.cache } +// cachePath returns the path to the fragment's cache data. +func (f *Fragment) cachePath() string { return f.path + CacheExt } // Open opens the underlying storage. func (f *Fragment) Open() error { @@ -261,7 +242,7 @@ func (f *Fragment) openCache() error { } // Read cache data from disk. - path := f.CachePath() + path := f.cachePath() buf, err := ioutil.ReadFile(path) if os.IsNotExist(err) { return nil @@ -342,14 +323,14 @@ func (f *Fragment) closeStorage() error { return nil } -// Row returns a row by ID. -func (f *Fragment) Row(rowID uint64) *Row { +// row returns a row by ID. +func (f *Fragment) row(rowID uint64) *Row { f.mu.Lock() defer f.mu.Unlock() - return f.row(rowID, true, true) + return f.unprotectedRow(rowID, true, true) } -func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *Row { +func (f *Fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCache bool) *Row { if checkRowCache { r, ok := f.rowCache.Fetch(rowID) if ok && r != nil { @@ -380,15 +361,15 @@ func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *R return row } -// SetBit sets a bit for a given column & row within the fragment. +// setBit sets a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) { +func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - return f.setBit(rowID, columnID) + return f.unprotectedSetBit(rowID, columnID) } -func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { +func (f *Fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) @@ -415,7 +396,7 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { } // Get the row from row cache or fragment.storage. - row := f.row(rowID, true, true) + row := f.unprotectedRow(rowID, true, true) row.SetBit(columnID) // Update the cache. @@ -432,15 +413,15 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { return changed, nil } -// ClearBit clears a bit for a given column & row within the fragment. +// clearBit clears a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) { +func (f *Fragment) clearBit(rowID, columnID uint64) (bool, error) { f.mu.Lock() defer f.mu.Unlock() - return f.clearBit(rowID, columnID) + return f.unprotectedClearBit(rowID, columnID) } -func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { +func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) @@ -467,7 +448,7 @@ func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { } // Get the row from cache or fragment.storage. - row := f.row(rowID, true, true) + row := f.unprotectedRow(rowID, true, true) row.ClearBit(columnID) // Update the cache. @@ -486,8 +467,8 @@ func (f *Fragment) bit(rowID, columnID uint64) (bool, error) { return f.storage.Contains(pos), nil } -// Value uses a column of bits to read a multi-bit value. -func (f *Fragment) Value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { +// value uses a column of bits to read a multi-bit value. +func (f *Fragment) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { f.mu.Lock() defer f.mu.Unlock() @@ -510,20 +491,20 @@ func (f *Fragment) Value(columnID uint64, bitDepth uint) (value uint64, exists b return value, true, nil } -// SetValue uses a column of bits to set a multi-bit value. -func (f *Fragment) SetValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +// setValue uses a column of bits to set a multi-bit value. +func (f *Fragment) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() for i := uint(0); i < bitDepth; i++ { if value&(1< uint(0); i-- { ii := i - 1 // allow for uint range: (bitDepth-1) to 0 - row := f.Row(uint64(ii)) + row := f.row(uint64(ii)) x := consider.Difference(row) count = x.Count() @@ -647,11 +628,11 @@ func (f *Fragment) Min(filter *Row, bitDepth uint) (min, count uint64, err error return min, count, nil } -// Max returns the max of a given bsiGroup as well as the number of columns involved. +// max returns the max of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *Fragment) Max(filter *Row, bitDepth uint) (max, count uint64, err error) { +func (f *Fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error) { - consider := f.Row(uint64(bitDepth)) + consider := f.row(uint64(bitDepth)) if filter != nil { consider = consider.Intersect(filter) } @@ -663,7 +644,7 @@ func (f *Fragment) Max(filter *Row, bitDepth uint) (max, count uint64, err error for i := bitDepth; i > uint(0); i-- { ii := i - 1 // allow for uint range: (bitDepth-1) to 0 - row := f.Row(uint64(ii)) + row := f.row(uint64(ii)) x := row.Intersect(consider) count = x.Count() @@ -678,8 +659,8 @@ func (f *Fragment) Max(filter *Row, bitDepth uint) (max, count uint64, err error return max, count, nil } -// RangeOp returns bitmaps with a bsiGroup value encoding matching the predicate. -func (f *Fragment) RangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +// rangeOp returns bitmaps with a bsiGroup value encoding matching the predicate. +func (f *Fragment) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { switch op { case pql.EQ: return f.rangeEQ(bitDepth, predicate) @@ -696,11 +677,11 @@ func (f *Fragment) RangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, func (f *Fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit := (predicate >> uint(i)) & 1 if bit == 1 { @@ -715,7 +696,7 @@ func (f *Fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) { func (f *Fragment) rangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) // Get the equal bitmap. eq, err := f.rangeEQ(bitDepth, predicate) @@ -733,12 +714,12 @@ func (f *Fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool) keep := NewRow() // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) // Filter any bits that don't match the current bit value. leadingZeros := true for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit := (predicate >> uint(i)) & 1 // Remove any columns with higher bits set. @@ -778,12 +759,12 @@ func (f *Fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool) } func (f *Fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) keep := NewRow() // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit := (predicate >> uint(i)) & 1 // Handle last bit differently. @@ -812,20 +793,20 @@ func (f *Fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) return b, nil } -// NotNull returns the not-null row (stored at bitDepth). -func (f *Fragment) NotNull(bitDepth uint) (*Row, error) { - return f.Row(uint64(bitDepth)), nil +// notNull returns the not-null row (stored at bitDepth). +func (f *Fragment) notNull(bitDepth uint) (*Row, error) { + return f.row(uint64(bitDepth)), nil } -// RangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax. -func (f *Fragment) RangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { - b := f.Row(uint64(bitDepth)) +// rangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax. +func (f *Fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { + b := f.row(uint64(bitDepth)) keep1 := NewRow() // GTE keep2 := NewRow() // LTE // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit1 := (predicateMin >> uint(i)) & 1 bit2 := (predicateMax >> uint(i)) & 1 @@ -864,12 +845,12 @@ func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) { if columnID < minColumnID || columnID >= minColumnID+SliceWidth { return 0, errors.New("column out of bounds") } - return Pos(rowID, columnID), nil + return pos(rowID, columnID), nil } -// ForEachBit executes fn for every bit set in the fragment. +// forEachBit executes fn for every bit set in the fragment. // Errors returned from fn are passed through. -func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error { +func (f *Fragment) forEachBit(fn func(rowID, columnID uint64) error) error { f.mu.Lock() defer f.mu.Unlock() @@ -886,10 +867,10 @@ func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error { return err } -// Top returns the top rows from the fragment. +// top returns the top rows from the fragment. // If opt.Src is specified then only rows which intersect src are returned. // If opt.FilterValues exist then the row attribute specified by field is matched. -func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { +func (f *Fragment) top(opt TopOptions) ([]Pair, error) { // Retrieve pairs. If no row ids specified then return from cache. pairs := f.topBitmapPairs(opt.RowIDs) @@ -960,7 +941,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Calculate count and append. count := cnt if opt.Src != nil { - count = opt.Src.IntersectionCount(f.Row(rowID)) + count = opt.Src.IntersectionCount(f.row(rowID)) } if count == 0 { continue @@ -1004,7 +985,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Calculate the intersecting column count and skip if it's below our // last row in our current result set. - count := opt.Src.IntersectionCount(f.Row(rowID)) + count := opt.Src.IntersectionCount(f.row(rowID)) if count < threshold { continue } @@ -1048,7 +1029,7 @@ func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair { continue } - row := f.Row(rowID) + row := f.row(rowID) if row.Count() > 0 { // Otherwise load from storage. pairs = append(pairs, BitmapPair{ @@ -1089,13 +1070,6 @@ func (f *Fragment) Checksum() []byte { return h.Sum(nil) } -// BlockN returns the number of blocks in the fragment. -func (f *Fragment) BlockN() int { - f.mu.Lock() - defer f.mu.Unlock() - return int(f.storage.Max() / (HashBlockSize * SliceWidth)) -} - // InvalidateChecksums clears all cached block checksums. func (f *Fragment) InvalidateChecksums() { f.mu.Lock() @@ -1184,8 +1158,8 @@ func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i } } -// BlockData returns bits in a block as row & column ID pairs. -func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) { +// blockData returns bits in a block as row & column ID pairs. +func (f *Fragment) blockData(id int) (rowIDs, columnIDs []uint64) { f.mu.Lock() defer f.mu.Unlock() @@ -1196,17 +1170,17 @@ func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) { return } -// MergeBlock compares the block's bits and computes a diff with another set of block bits. +// mergeBlock compares the block's bits and computes a diff with another set of block bits. // The state of a bit is determined by consensus from all blocks being considered. // // For example, if 3 blocks are compared and two have a set bit and one has a // cleared bit then the bit is considered cleared. The function returns the // diff per incoming block so that all can be in sync. -func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, err error) { +func (f *Fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, err error) { // Ensure that all pair sets are of equal length. for i := range data { - if len(data[i].RowIDs) != len(data[i].ColumnIDs) { - return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].RowIDs), len(data[i].ColumnIDs)) + if len(data[i].rowIDs) != len(data[i].columnIDs) { + return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].rowIDs), len(data[i].columnIDs)) } } @@ -1214,8 +1188,8 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e defer f.mu.Unlock() // Track sets and clears for all blocks (including local). - sets = make([]PairSet, len(data)+1) - clears = make([]PairSet, len(data)+1) + sets = make([]pairSet, len(data)+1) + clears = make([]pairSet, len(data)+1) // Limit upper row/column pair. maxRowID := uint64(id+1) * HashBlockSize @@ -1231,7 +1205,7 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e // Append buffered iterators for each incoming block. for i := range data { - var itr Iterator = NewSliceIterator(data[i].RowIDs, data[i].ColumnIDs) + var itr Iterator = NewSliceIterator(data[i].rowIDs, data[i].columnIDs) itr = NewLimitIterator(itr, maxRowID, maxColumnID) itrs = append(itrs, NewBufIterator(itr)) } @@ -1296,25 +1270,25 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e // Append to either the set or clear diff. if newValue { - sets[i].RowIDs = append(sets[i].RowIDs, min.rowID) - sets[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID) + sets[i].rowIDs = append(sets[i].rowIDs, min.rowID) + sets[i].columnIDs = append(sets[i].columnIDs, min.columnID) } else { - clears[i].RowIDs = append(sets[i].RowIDs, min.rowID) - clears[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID) + clears[i].rowIDs = append(sets[i].rowIDs, min.rowID) + clears[i].columnIDs = append(sets[i].columnIDs, min.columnID) } } } // Set local bits. - for i := range sets[0].ColumnIDs { - if _, err := f.setBit(sets[0].RowIDs[i], (f.Slice()*SliceWidth)+sets[0].ColumnIDs[i]); err != nil { + for i := range sets[0].columnIDs { + if _, err := f.unprotectedSetBit(sets[0].rowIDs[i], (f.slice*SliceWidth)+sets[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "setting") } } // Clear local bits. - for i := range clears[0].ColumnIDs { - if _, err := f.clearBit(clears[0].RowIDs[i], (f.Slice()*SliceWidth)+clears[0].ColumnIDs[i]); err != nil { + for i := range clears[0].columnIDs { + if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.slice*SliceWidth)+clears[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "clearing") } } @@ -1322,9 +1296,9 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e return sets[1:], clears[1:], nil } -// Import bulk imports a set of bits and then snapshots the storage. +// bulkImport bulk imports a set of bits and then snapshots the storage. // This does not affect the fragment's cache. -func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { +func (f *Fragment) bulkImport(rowIDs, columnIDs []uint64) error { f.mu.Lock() defer f.mu.Unlock() // Verify that there are an equal number of row ids and column ids. @@ -1373,7 +1347,7 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { // Import should ALWAYS have row() load a new row from fragment.storage // because the row that's in rowCache hasn't been updated with // this import's data. - f.cache.BulkAdd(rowID, f.row(rowID, false, false).Count()) + f.cache.BulkAdd(rowID, f.unprotectedRow(rowID, false, false).Count()) } f.cache.Invalidate() @@ -1392,8 +1366,8 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { return nil } -// ImportValue bulk imports a set of range-encoded values. -func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error { +// importValue bulk imports a set of range-encoded values. +func (f *Fragment) importValue(columnIDs, values []uint64, bitDepth uint) error { f.mu.Lock() defer f.mu.Unlock() // Verify that there are an equal number of column ids and values. @@ -1529,7 +1503,7 @@ func (f *Fragment) flushCache() error { } // Write to disk. - if err := ioutil.WriteFile(f.CachePath(), buf, 0666); err != nil { + if err := ioutil.WriteFile(f.cachePath(), buf, 0666); err != nil { return errors.Wrap(err, "writing") } @@ -1603,7 +1577,7 @@ func (f *Fragment) writeCacheToArchive(tw *tar.Writer) error { defer f.mu.Unlock() // Read cache into buffer. - buf, err := ioutil.ReadFile(f.CachePath()) + buf, err := ioutil.ReadFile(f.cachePath()) if os.IsNotExist(err) { return nil } else if err != nil { @@ -1697,7 +1671,7 @@ func (f *Fragment) readCacheFromArchive(r io.Reader) error { buf, err := ioutil.ReadAll(r) if err != nil { return errors.Wrap(err, "reading") - } else if err := ioutil.WriteFile(f.CachePath(), buf, 0666); err != nil { + } else if err := ioutil.WriteFile(f.cachePath(), buf, 0666); err != nil { return errors.Wrap(err, "writing") } @@ -1762,11 +1736,11 @@ func (s *FragmentSyncer) isClosing() bool { } } -// SyncFragment compares checksums for the local and remote fragments and +// syncFragment compares checksums for the local and remote fragments and // then merges any blocks which have differences. -func (s *FragmentSyncer) SyncFragment() error { +func (s *FragmentSyncer) syncFragment() error { // Determine replica set. - nodes := s.Cluster.SliceNodes(s.Fragment.Index(), s.Fragment.Slice()) + nodes := s.Cluster.SliceNodes(s.Fragment.index, s.Fragment.slice) if len(nodes) == 1 { return nil } @@ -1782,7 +1756,7 @@ func (s *FragmentSyncer) SyncFragment() error { } // Retrieve remote blocks. - blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.Index(), s.Fragment.Field(), s.Fragment.Slice()) + blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.index, s.Fragment.field, s.Fragment.slice) if err != nil && err != ErrFragmentNotFound { return errors.Wrap(err, "getting blocks") } @@ -1845,9 +1819,9 @@ func (s *FragmentSyncer) syncBlock(id int) error { f := s.Fragment // Read pairs from each remote block. - var pairSets []PairSet var uris []*URI - for _, node := range s.Cluster.SliceNodes(f.Index(), f.Slice()) { + var pairSets []pairSet + for _, node := range s.Cluster.SliceNodes(f.index, f.slice) { if s.Node.ID == node.ID { continue } @@ -1861,14 +1835,14 @@ func (s *FragmentSyncer) syncBlock(id int) error { uris = append(uris, uri) // Only sync the standard block. - rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.Index(), f.Field(), f.Slice(), id) + rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.index, f.field, f.slice, id) if err != nil { return errors.Wrap(err, "getting block") } - pairSets = append(pairSets, PairSet{ - ColumnIDs: columnIDs, - RowIDs: rowIDs, + pairSets = append(pairSets, pairSet{ + columnIDs: columnIDs, + rowIDs: rowIDs, }) } @@ -1878,7 +1852,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { } // Merge blocks together. - sets, clears, err := f.MergeBlock(id, pairSets) + sets, clears, err := f.mergeBlock(id, pairSets) if err != nil { return errors.Wrap(err, "merging") } @@ -1889,12 +1863,12 @@ func (s *FragmentSyncer) syncBlock(id int) error { count := 0 // Ignore if there are no differences. - if len(set.ColumnIDs) == 0 && len(clear.ColumnIDs) == 0 { + if len(set.columnIDs) == 0 && len(clear.columnIDs) == 0 { continue } // Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest. - total := len(set.ColumnIDs) + len(clear.ColumnIDs) + total := len(set.columnIDs) + len(clear.columnIDs) maxWrites := s.Cluster.MaxWritesPerRequest if maxWrites <= 0 { maxWrites = 5000 @@ -1902,12 +1876,12 @@ func (s *FragmentSyncer) syncBlock(id int) error { buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites)))) // Only sync the standard block. - for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(field=%q, row=%d, col=%d)\n", f.Field(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + for j := 0; j < len(set.columnIDs); j++ { + fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(field=%q, row=%d, col=%d)\n", f.field, set.rowIDs[j], (f.slice*SliceWidth)+set.columnIDs[j]) count++ } - for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(field=%q, row=%d, col=%d)\n", f.Field(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + for j := 0; j < len(clear.columnIDs); j++ { + fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(field=%q, row=%d, col=%d)\n", f.field, clear.rowIDs[j], (f.slice*SliceWidth)+clear.columnIDs[j]) count++ } @@ -1923,7 +1897,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { Query: buffers[k].String(), Remote: true, } - _, err := s.Cluster.InternalClient.QueryNode(context.Background(), uris[i], f.Index(), queryRequest) + _, err := s.Cluster.InternalClient.QueryNode(context.Background(), uris[i], f.index, queryRequest) if err != nil { return errors.Wrap(err, "executing") } @@ -1941,10 +1915,10 @@ func madvise(b []byte, advice int) (err error) { return } -// PairSet is a list of equal length row and column id lists. -type PairSet struct { - RowIDs []uint64 - ColumnIDs []uint64 +// pairSet is a list of equal length row and column id lists. +type pairSet struct { + rowIDs []uint64 + columnIDs []uint64 } // byteSlicesEqual returns true if all slices are equal. @@ -1961,7 +1935,7 @@ func byteSlicesEqual(a [][]byte) bool { return true } -// Pos returns the row position of a row/column pair. -func Pos(rowID, columnID uint64) uint64 { +// pos returns the row position of a row/column pair. +func pos(rowID, columnID uint64) uint64 { return (rowID * SliceWidth) + (columnID % SliceWidth) } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 20f3f7b6a..e7a77dafb 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -40,27 +40,27 @@ func TestFragment_SetBit(t *testing.T) { defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(120, 1); err != nil { + if _, err := f.setBit(120, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 6); err != nil { + } else if _, err := f.setBit(120, 6); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(121, 0); err != nil { + } else if _, err := f.setBit(121, 0); err != nil { t.Fatal(err) } // Verify counts on rows. - if n := f.Row(120).Count(); n != 2 { + if n := f.row(120).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) - } else if n := f.Row(121).Count(); n != 1 { + } else if n := f.row(121).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(120).Count(); n != 2 { + } else if n := f.row(120).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) - } else if n := f.Row(121).Count(); n != 1 { + } else if n := f.row(121).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -71,23 +71,23 @@ func TestFragment_ClearBit(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1); err != nil { + if _, err := f.setBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2); err != nil { + } else if _, err := f.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, err := f.ClearBit(1000, 1); err != nil { + } else if _, err := f.clearBit(1000, 1); err != nil { t.Fatal(err) } // Verify count on row. - if n := f.Row(1000).Count(); n != 1 { + if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 1 { + } else if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -99,14 +99,14 @@ func TestFragment_SetValue(t *testing.T) { defer f.Close() // Set value. - if changed, err := f.SetValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.Value(100, 16); err != nil { + if value, exists, err := f.value(100, 16); err != nil { t.Fatal(err) } else if value != 3829 { t.Fatalf("unexpected value: %d", value) @@ -115,7 +115,7 @@ func TestFragment_SetValue(t *testing.T) { } // Setting value should return no change. - if changed, err := f.SetValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(100, 16, 3829); err != nil { t.Fatal(err) } else if changed { t.Fatal("expected no change") @@ -127,21 +127,21 @@ func TestFragment_SetValue(t *testing.T) { defer f.Close() // Set value. - if changed, err := f.SetValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Overwriting value should overwrite all bits. - if changed, err := f.SetValue(100, 16, 2028); err != nil { + if changed, err := f.setValue(100, 16, 2028); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.Value(100, 16); err != nil { + if value, exists, err := f.value(100, 16); err != nil { t.Fatal(err) } else if value != 2028 { t.Fatalf("unexpected value: %d", value) @@ -155,14 +155,14 @@ func TestFragment_SetValue(t *testing.T) { defer f.Close() // Set value. - if changed, err := f.SetValue(100, 10, 20); err != nil { + if changed, err := f.setValue(100, 10, 20); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Non-existent value. - if value, exists, err := f.Value(100, 11); err != nil { + if value, exists, err := f.value(100, 11); err != nil { t.Fatal(err) } else if value != 0 { t.Fatalf("unexpected value: %d", value) @@ -191,14 +191,14 @@ func TestFragment_SetValue(t *testing.T) { m[columnID] = int64(value) - if _, err := f.SetValue(columnID, bitDepth, value); err != nil { + if _, err := f.setValue(columnID, bitDepth, value); err != nil { t.Fatal(err) } } // Ensure values are set. for columnID, value := range m { - v, exists, err := f.Value(columnID, bitDepth) + v, exists, err := f.value(columnID, bitDepth) if err != nil { t.Fatal(err) } else if value != int64(v) { @@ -223,18 +223,18 @@ func TestFragment_Sum(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } t.Run("NoFilter", func(t *testing.T) { - if sum, n, err := f.Sum(nil, bitDepth); err != nil { + if sum, n, err := f.sum(nil, bitDepth); err != nil { t.Fatal(err) } else if n != 4 { t.Fatalf("unexpected count: %d", n) @@ -244,7 +244,7 @@ func TestFragment_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if sum, n, err := f.Sum(NewRow(2000, 4000, 5000), bitDepth); err != nil { + if sum, n, err := f.sum(NewRow(2000, 4000, 5000), bitDepth); err != nil { t.Fatal(err) } else if n != 2 { t.Fatalf("unexpected count: %d", n) @@ -262,19 +262,19 @@ func TestFragment_MinMax(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(5000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(6000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(7000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(7000, bitDepth, 0); err != nil { t.Fatal(err) } @@ -292,7 +292,7 @@ func TestFragment_MinMax(t *testing.T) { {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if min, cnt, err := f.Min(test.filter, bitDepth); err != nil { + if min, cnt, err := f.min(test.filter, bitDepth); err != nil { t.Fatal(err) } else if min != test.exp { t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min) @@ -316,7 +316,7 @@ func TestFragment_MinMax(t *testing.T) { {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if max, cnt, err := f.Max(test.filter, bitDepth); err != nil { + if max, cnt, err := f.max(test.filter, bitDepth); err != nil { t.Fatal(err) } else if max != test.exp { t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max) @@ -336,18 +336,18 @@ func TestFragment_Range(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for equality. - if b, err := f.RangeOp(pql.EQ, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.EQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -359,18 +359,18 @@ func TestFragment_Range(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for inequality. - if b, err := f.RangeOp(pql.NEQ, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.NEQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -382,43 +382,43 @@ func TestFragment_Range(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for values less than (ending with set column). - if b, err := f.RangeOp(pql.LT, bitDepth, 301); err != nil { + if b, err := f.rangeOp(pql.LT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values less than (ending with unset column). - if b, err := f.RangeOp(pql.LT, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.LT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values less than or equal to (ending with set column). - if b, err := f.RangeOp(pql.LTE, bitDepth, 301); err != nil { + if b, err := f.rangeOp(pql.LTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values less than or equal to (ending with unset column). - if b, err := f.RangeOp(pql.LTE, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.LTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -430,43 +430,43 @@ func TestFragment_Range(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for values greater than (ending with unset bit). - if b, err := f.RangeOp(pql.GT, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.GT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than (ending with set bit). - if b, err := f.RangeOp(pql.GT, bitDepth, 301); err != nil { + if b, err := f.rangeOp(pql.GT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with unset bit). - if b, err := f.RangeOp(pql.GTE, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.GTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with set bit). - if b, err := f.RangeOp(pql.GTE, bitDepth, 301); err != nil { + if b, err := f.rangeOp(pql.GTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -478,43 +478,43 @@ func TestFragment_Range(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for values greater than (ending with unset column). - if b, err := f.RangeBetween(bitDepth, 300, 2817); err != nil { + if b, err := f.rangeBetween(bitDepth, 300, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than (ending with set column). - if b, err := f.RangeBetween(bitDepth, 301, 2817); err != nil { + if b, err := f.rangeBetween(bitDepth, 301, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with unset column). - if b, err := f.RangeBetween(bitDepth, 301, 2816); err != nil { + if b, err := f.rangeBetween(bitDepth, 301, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with set column). - if b, err := f.RangeBetween(bitDepth, 300, 2816); err != nil { + if b, err := f.rangeBetween(bitDepth, 300, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -528,25 +528,25 @@ func TestFragment_Snapshot(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1); err != nil { + if _, err := f.setBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2); err != nil { + } else if _, err := f.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, err := f.ClearBit(1000, 1); err != nil { + } else if _, err := f.clearBit(1000, 1); err != nil { t.Fatal(err) } // Snapshot bitmap and verify data. if err := f.Snapshot(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 1 { + } else if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 1 { + } else if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -557,17 +557,17 @@ func TestFragment_ForEachBit(t *testing.T) { defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(100, 20); err != nil { + if _, err := f.setBit(100, 20); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 38); err != nil { + } else if _, err := f.setBit(2, 38); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 37); err != nil { + } else if _, err := f.setBit(2, 37); err != nil { t.Fatal(err) } // Iterate over bits. var result [][2]uint64 - if err := f.ForEachBit(func(rowID, columnID uint64) error { + if err := f.forEachBit(func(rowID, columnID uint64) error { result = append(result, [2]uint64{rowID, columnID}) return nil }); err != nil { @@ -591,7 +591,7 @@ func TestFragment_Top(t *testing.T) { f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(TopOptions{N: 2}); err != nil { + if pairs, err := f.top(TopOptions{N: 2}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -617,7 +617,7 @@ func TestFragment_Top_Filter(t *testing.T) { f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": int64(20)}) // Retrieve top rows. - if pairs, err := f.Top(TopOptions{ + if pairs, err := f.top(TopOptions{ N: 2, FilterName: "x", FilterValues: []interface{}{int64(10), int64(15), int64(20)}, @@ -648,7 +648,7 @@ func TestFragment_TopN_Intersect(t *testing.T) { f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(TopOptions{N: 3, Src: src}); err != nil { + if pairs, err := f.top(TopOptions{N: 3, Src: src}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 3}, @@ -683,7 +683,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(TopOptions{N: 10, Src: src}); err != nil { + if pairs, err := f.top(TopOptions{N: 10, Src: src}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 999, Count: 19}, @@ -712,7 +712,7 @@ func TestFragment_TopN_IDs(t *testing.T) { f.mustSetBits(102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.Top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 4}, @@ -733,7 +733,7 @@ func TestFragment_TopN_NopCache(t *testing.T) { f.mustSetBits(102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.Top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{}) { t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs)) @@ -792,7 +792,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { } // Retrieve top rows. - if pairs, err := f.Top(TopOptions{N: 5}); err != nil { + if pairs, err := f.top(TopOptions{N: 5}); err != nil { t.Fatal(err) } else if len(pairs) > int(cacheSize) { t.Fatalf("TopN count cannot exceed cache size: %d", cacheSize) @@ -810,9 +810,9 @@ func TestFragment_Checksum(t *testing.T) { // Retrieve checksum and set bits. orig := f.Checksum() - if _, err := f.SetBit(1, 200); err != nil { + if _, err := f.setBit(1, 200); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(HashBlockSize*2, 200); err != nil { + } else if _, err := f.setBit(HashBlockSize*2, 200); err != nil { t.Fatal(err) } @@ -831,7 +831,7 @@ func TestFragment_Blocks(t *testing.T) { var prev []FragmentBlock // Set first bit. - if _, err := f.SetBit(0, 0); err != nil { + if _, err := f.setBit(0, 0); err != nil { t.Fatal(err) } blocks := f.Blocks() @@ -841,7 +841,7 @@ func TestFragment_Blocks(t *testing.T) { prev = blocks // Set bit on different row. - if _, err := f.SetBit(20, 0); err != nil { + if _, err := f.setBit(20, 0); err != nil { t.Fatal(err) } blocks = f.Blocks() @@ -851,7 +851,7 @@ func TestFragment_Blocks(t *testing.T) { prev = blocks // Set bit on different column. - if _, err := f.SetBit(20, 100); err != nil { + if _, err := f.setBit(20, 100); err != nil { t.Fatal(err) } blocks = f.Blocks() @@ -866,7 +866,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { defer f.Close() // Set bits on a different block. - if _, err := f.SetBit(100, 1); err != nil { + if _, err := f.setBit(100, 1); err != nil { t.Fatal(err) } @@ -885,14 +885,14 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0); err != nil { + if _, err := f.setBit(i, 0); err != nil { t.Fatal(err) } } // Verify correct cache type and size. - if cache, ok := f.Cache().(*LRUCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*LRUCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } @@ -903,8 +903,8 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { } // Re-verify correct cache type and size. - if cache, ok := f.Cache().(*LRUCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*LRUCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } @@ -935,14 +935,14 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0); err != nil { + if _, err := f.setBit(i, 0); err != nil { t.Fatal(err) } } // Verify correct cache type and size. - if cache, ok := f.Cache().(*RankCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*RankCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } @@ -956,8 +956,8 @@ func TestFragment_RankCache_Persistence(t *testing.T) { f = index.Field("f").View(ViewStandard).Fragment(0) // Re-verify correct cache type and size. - if cache, ok := f.Cache().(*RankCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*RankCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } @@ -969,16 +969,16 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { defer f0.Close() // Set and then clear bits on the fragment. - if _, err := f0.SetBit(1000, 1); err != nil { + if _, err := f0.setBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f0.SetBit(1000, 2); err != nil { + } else if _, err := f0.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, err := f0.ClearBit(1000, 1); err != nil { + } else if _, err := f0.clearBit(1000, 1); err != nil { t.Fatal(err) } // Verify cache is populated. - if n := f0.Cache().Len(); n != 1 { + if n := f0.cache.Len(); n != 1 { t.Fatalf("unexpected cache size: %d", n) } @@ -998,21 +998,21 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Verify cache is in other fragment. - if n := f1.Cache().Len(); n != 1 { + if n := f1.cache.Len(); n != 1 { t.Fatalf("unexpected cache size: %d", n) } // Verify data in other fragment. - if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + if a := f1.row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected columns: %+v", a) } // Close and reopen the fragment & verify the data. if err := f1.reopen(); err != nil { t.Fatal(err) - } else if n := f1.Cache().Len(); n != 1 { + } else if n := f1.cache.Len(); n != 1 { t.Fatalf("unexpected cache size (reopen): %d", n) - } else if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + } else if a := f1.row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected columns (reopen): %+v", a) } } @@ -1045,12 +1045,12 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Generate some intersecting data. for i := 0; i < 10000; i += 2 { - if _, err := f.SetBit(1, uint64(i)); err != nil { + if _, err := f.setBit(1, uint64(i)); err != nil { b.Fatal(err) } } for i := 0; i < 10000; i += 3 { - if _, err := f.SetBit(2, uint64(i)); err != nil { + if _, err := f.setBit(2, uint64(i)); err != nil { b.Fatal(err) } } @@ -1063,7 +1063,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Start benchmark b.ResetTimer() for i := 0; i < b.N; i++ { - if n := f.Row(1).IntersectionCount(f.Row(2)); n == 0 { + if n := f.row(1).IntersectionCount(f.row(2)); n == 0 { b.Fatalf("unexpected count: %d", n) } } @@ -1081,7 +1081,7 @@ func TestFragment_Tanimoto(t *testing.T) { f.mustSetBits(102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.Top(TopOptions{TanimotoThreshold: 50, Src: src}); err != nil { + if pairs, err := f.top(TopOptions{TanimotoThreshold: 50, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -1104,7 +1104,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { f.mustSetBits(102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.Top(TopOptions{TanimotoThreshold: 0, Src: src}); err != nil { + if pairs, err := f.top(TopOptions{TanimotoThreshold: 0, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 3 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -1123,7 +1123,7 @@ func TestFragment_Snapshot_Run(t *testing.T) { // Set bits on the fragment. for i := uint64(1); i < 3; i++ { - if _, err := f.SetBit(1000, i); err != nil { + if _, err := f.setBit(1000, i); err != nil { t.Fatal(err) } } @@ -1131,14 +1131,14 @@ func TestFragment_Snapshot_Run(t *testing.T) { // Snapshot bitmap and verify data. if err := f.Snapshot(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 2 { + } else if n := f.row(1000).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 2 { + } else if n := f.row(1000).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -1187,7 +1187,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { val += 2 i++ } - if err := f.Import(rows, cols); err != nil { + if err := f.bulkImport(rows, cols); err != nil { b.Fatalf("Error Building Sample: %s", err) } if row > max { @@ -1228,7 +1228,7 @@ func BenchmarkFragment_Import(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - if err := f.Import(rows, cols); err != nil { + if err := f.bulkImport(rows, cols); err != nil { b.Fatalf("Error Building Sample: %s", err) } } @@ -1273,7 +1273,7 @@ func (f *Fragment) reopen() error { // This function does not accept a timestamp or quantum. func (f *Fragment) mustSetBits(rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { - if _, err := f.SetBit(rowID, columnID); err != nil { + if _, err := f.setBit(rowID, columnID); err != nil { panic(err) } } diff --git a/holder.go b/holder.go index 2330d59e7..b3147d897 100644 --- a/holder.go +++ b/holder.go @@ -217,7 +217,7 @@ func (h *Holder) Schema() []*IndexInfo { for _, field := range index.Fields() { fi := &FieldInfo{Name: field.Name(), Options: field.Options()} for _, view := range field.Views() { - fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()}) + fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) } sort.Sort(viewInfoSlice(fi.Views)) di.Fields = append(di.Fields, fi) @@ -437,7 +437,7 @@ func (h *Holder) flushCaches() { for _, index := range h.Indexes() { for _, field := range index.Fields() { for _, view := range field.Views() { - for _, fragment := range view.Fragments() { + for _, fragment := range view.allFragments() { select { case <-h.closing: return @@ -445,7 +445,7 @@ func (h *Holder) flushCaches() { } if err := fragment.FlushCache(); err != nil { - h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) + h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.cachePath()) } } } @@ -761,7 +761,7 @@ func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) err Closing: s.Closing, RemoteClient: s.RemoteClient, } - if err := fs.SyncFragment(); err != nil { + if err := fs.syncFragment(); err != nil { return errors.Wrap(err, "syncing fragment") } @@ -804,14 +804,14 @@ func (c *HolderCleaner) CleanHolder() error { // Get the fragments registered in memory. for _, field := range index.Fields() { for _, view := range field.Views() { - for _, fragment := range view.Fragments() { - fragSlice := fragment.Slice() + for _, fragment := range view.allFragments() { + fragSlice := fragment.slice // Ignore fragments that should be present. if uint64InSlice(fragSlice, containedSlices) { continue } // Delete fragment. - if err := view.DeleteFragment(fragSlice); err != nil { + if err := view.deleteFragment(fragSlice); err != nil { return errors.Wrap(err, "deleting fragment") } } diff --git a/holder_test.go b/holder_test.go index 70fecf245..b47e019eb 100644 --- a/holder_test.go +++ b/holder_test.go @@ -210,9 +210,7 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if _, err := view.SetBit(0, 0); err != nil { + } else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -233,9 +231,7 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if _, err := view.SetBit(0, 0); err != nil { + } else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -261,7 +257,7 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) - } else if _, err := view.SetBit(0, 0); err != nil { + } else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } else if err := view.Fragment(0).FlushCache(); err != nil { t.Fatal(err) @@ -331,14 +327,8 @@ func TestHolder_DeleteIndex(t *testing.T) { defer hldr.Close() // Write bits to separate indexes. - f0 := hldr.MustCreateFragmentIfNotExists("i0", "f", pilosa.ViewStandard, 0) - if _, err := f0.SetBit(100, 200); err != nil { - t.Fatal(err) - } - f1 := hldr.MustCreateFragmentIfNotExists("i1", "f", pilosa.ViewStandard, 0) - if _, err := f1.SetBit(100, 200); err != nil { - t.Fatal(err) - } + hldr.SetBit("i0", "f", 100, 200) + hldr.SetBit("i1", "f", 100, 200) // Ensure i0 exists. if _, err := os.Stat(hldr.IndexPath("i0")); err != nil { @@ -404,42 +394,24 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { } // Set data on the local holder. - f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - if _, err := f.SetBit(0, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(2, 20); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(120, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(200, 4); err != nil { - t.Fatal(err) - } + hldr0.SetBit("i", "f", 0, 10) + hldr0.SetBit("i", "f", 2, 20) + hldr0.SetBit("i", "f", 120, 10) + hldr0.SetBit("i", "f", 200, 4) - f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1) - if _, err := f.SetBit(9, SliceWidth+5); err != nil { - t.Fatal(err) - } + hldr0.SetBit("i", "f0", 9, SliceWidth+5) - hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0) + // Set a bit to create the fragment. + hldr0.SetBit("y", "z", 0, 0) // Set data on the remote holder. - f = hldr1.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - if _, err := f.SetBit(0, 4000); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(3, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(120, 10); err != nil { - t.Fatal(err) - } + hldr1.SetBit("i", "f", 0, 4000) + hldr1.SetBit("i", "f", 3, 10) + hldr1.SetBit("i", "f", 120, 10) - f = hldr1.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 3) - if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(10, (3*SliceWidth)+7); err != nil { - t.Fatal(err) - } + hldr1.SetBit("y", "z", 10, (3*SliceWidth)+4) + hldr1.SetBit("y", "z", 10, (3*SliceWidth)+5) + hldr1.SetBit("y", "z", 10, (3*SliceWidth)+7) // Set highest slice. hldr0.Index("i").SetRemoteMaxSlice(1) @@ -460,29 +432,23 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0, hldr1} { - f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - a := f.Row(9).Columns() - if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a) - } - if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } - f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3) - if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { + + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -509,34 +475,18 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { } // Set data on the local holder. - f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - if _, err := f.SetBit(0, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(0, 4000); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(2, 20); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(3, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(120, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(200, 4); err != nil { - t.Fatal(err) - } + hldr0.SetBit("i", "f", 0, 10) + hldr0.SetBit("i", "f", 0, 4000) + hldr0.SetBit("i", "f", 2, 20) + hldr0.SetBit("i", "f", 3, 10) + hldr0.SetBit("i", "f", 120, 10) + hldr0.SetBit("i", "f", 200, 4) - f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1) - if _, err := f.SetBit(9, SliceWidth+5); err != nil { - t.Fatal(err) - } + hldr0.SetBit("i", "f0", 9, SliceWidth+5) - f = hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 2) - if _, err := f.SetBit(10, (2*SliceWidth)+4); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(10, (2*SliceWidth)+5); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(10, (2*SliceWidth)+7); err != nil { - t.Fatal(err) - } + hldr0.SetBit("y", "z", 10, (2*SliceWidth)+4) + hldr0.SetBit("y", "z", 10, (2*SliceWidth)+5) + hldr0.SetBit("y", "z", 10, (2*SliceWidth)+7) // Set highest slice. hldr0.Index("i").SetRemoteMaxSlice(1) @@ -558,29 +508,23 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0} { - f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - a := f.Row(9).Columns() - if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a) - } - if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } - f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) - if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -601,26 +545,24 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0} { - f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) + f := hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) if f != nil { t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f) } - f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) - if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } diff --git a/http/client.go b/http/client.go index b0fee7887..b3ffb20cb 100644 --- a/http/client.go +++ b/http/client.go @@ -1010,13 +1010,18 @@ type BitsByPos []pilosa.Bit func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p BitsByPos) Len() int { return len(p) } func (p BitsByPos) Less(i, j int) bool { - p0, p1 := pilosa.Pos(p[i].RowID, p[i].ColumnID), pilosa.Pos(p[j].RowID, p[j].ColumnID) + p0, p1 := pos(p[i].RowID, p[i].ColumnID), pos(p[j].RowID, p[j].ColumnID) if p0 == p1 { return p[i].Timestamp < p[j].Timestamp } return p0 < p1 } +// pos returns the row position of a row/column pair. +func pos(rowID, columnID uint64) uint64 { + return (rowID * pilosa.SliceWidth) + (columnID % pilosa.SliceWidth) +} + func uriPathToURL(uri *pilosa.URI, path string) url.URL { return url.URL{ Scheme: uri.Scheme(), diff --git a/http/client_test.go b/http/client_test.go index 93b16a177..7920d03cf 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -112,26 +112,26 @@ func TestClient_MultiNode(t *testing.T) { } } - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(100, baseBit0+10) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(22, baseBit0+1, baseBit0+2, baseBit0+10) + hldr[0].MustSetBits("i", "f", 100, baseBit0+10) + hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12) + hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) + hldr[0].MustSetBits("i", "f", 2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) + hldr[0].MustSetBits("i", "f", 3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) + hldr[0].MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2, baseBit0+10) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(1, baseBit1+4) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5) + hldr[1].MustSetBits("i", "f", 99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) + hldr[1].MustSetBits("i", "f", 100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) + hldr[1].MustSetBits("i", "f", 98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6) + hldr[1].MustSetBits("i", "f", 1, baseBit1+4) + hldr[1].MustSetBits("i", "f", 22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(21, baseBit2+10) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(100, baseBit2+10) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(99, baseBit2+10, baseBit2+11, baseBit2+12) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(98, baseBit2+10, baseBit2+11) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(22, baseBit2+10, baseBit2+11, baseBit2+12) + hldr[2].MustSetBits("i", "f", 24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) + hldr[2].MustSetBits("i", "f", 20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13) + hldr[2].MustSetBits("i", "f", 21, baseBit2+10) + hldr[2].MustSetBits("i", "f", 100, baseBit2+10) + hldr[2].MustSetBits("i", "f", 99, baseBit2+10, baseBit2+11, baseBit2+12) + hldr[2].MustSetBits("i", "f", 98, baseBit2+10, baseBit2+11) + hldr[2].MustSetBits("i", "f", 22, baseBit2+10, baseBit2+11, baseBit2+12) // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay @@ -214,8 +214,8 @@ func TestClient_Import(t *testing.T) { defer hldr.Close() // Load bitmap into cache to ensure cache gets updated. - f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - f.Row(0) + hldr.SetBit("i", "f", 1, 0) // set a bit so the view gets created. + hldr.Row("i", "f", 0) s := test.NewServer() defer s.Close() @@ -234,10 +234,10 @@ func TestClient_Import(t *testing.T) { } // Verify data. - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) { t.Fatalf("unexpected columns: %+v", a) } - if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{6}) { + if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{6}) { t.Fatalf("unexpected columns: %+v", a) } } @@ -325,11 +325,11 @@ func TestClient_FragmentBlocks(t *testing.T) { defer hldr.Close() // Set two bits on blocks 0 & 3. - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100) // Set a bit on a different slice. - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1) + hldr.SetBit("i", "f", 0, 1) s := test.NewServer() defer s.Close() diff --git a/http/handler_test.go b/http/handler_test.go index 39941a669..93f9906b2 100644 --- a/http/handler_test.go +++ b/http/handler_test.go @@ -189,13 +189,13 @@ func TestHandler_MaxSlices(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*pilosa.SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*pilosa.SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*pilosa.SliceWidth)+4) + hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1) + hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2) + hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*pilosa.SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*pilosa.SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*pilosa.SliceWidth)+8) + hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1) + hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2) + hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8) h := test.MustNewHandler() h.API.Holder = hldr.Holder diff --git a/stats_test.go b/stats_test.go index 55e90a741..ddf577918 100644 --- a/stats_test.go +++ b/stats_test.go @@ -36,11 +36,11 @@ func TestMultiStatClient_Expvar(t *testing.T) { ms[0] = c hldr.Stats = ms - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).ClearBit(0, 1) + hldr.SetBit("d", "f", 0, 0) + hldr.SetBit("d", "f", 0, 1) + hldr.SetBit("d", "f", 0, SliceWidth) + hldr.SetBit("d", "f", 0, SliceWidth+2) + hldr.ClearBit("d", "f", 0, 1) if pilosa.Expvar.String() != `{"index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) @@ -88,10 +88,10 @@ func TestStatsCount_TopN(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) + hldr.SetBit("d", "f", 0, 0) + hldr.SetBit("d", "f", 0, 1) + hldr.SetBit("d", "f", 0, SliceWidth) + hldr.SetBit("d", "f", 0, SliceWidth+2) // Execute query. called := false @@ -121,8 +121,8 @@ func TestStatsCount_Bitmap(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.SetBit("d", "f", 0, 0) + hldr.SetBit("d", "f", 0, 1) called := false e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) e.Holder.Stats = &MockStats{ @@ -150,8 +150,8 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1) + hldr.SetBit("d", "f", 10, 0) + hldr.SetBit("d", "f", 10, 1) called := false e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) @@ -180,8 +180,8 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1) + hldr.SetBit("d", "f", 10, 0) + hldr.SetBit("d", "f", 10, 1) called := false e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) diff --git a/test/fragment.go b/test/fragment.go index 140d394a7..6faf5d72d 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -26,13 +26,3 @@ type Fragment struct { *pilosa.Fragment RowAttrStore pilosa.AttrStore } - -// MustSetBits sets columns on a row. Panic on error. -// This function does not accept a timestamp or quantum. -func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := f.SetBit(rowID, columnID); err != nil { - panic(err) - } - } -} diff --git a/test/holder.go b/test/holder.go index 3cb0a8791..4484850fd 100644 --- a/test/holder.go +++ b/test/holder.go @@ -89,24 +89,6 @@ func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field { return f } -// MustCreateFragmentIfNotExists returns a given fragment. Panic on error. -func (h *Holder) MustCreateFragmentIfNotExists(index, field, view string, slice uint64) *Fragment { - idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) - if err != nil { - panic(err) - } - v, err := f.CreateViewIfNotExists(view) - if err != nil { - panic(err) - } - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - panic(err) - } - return &Fragment{Fragment: frag} -} - // MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error. func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, slice uint64) *Fragment { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) @@ -124,3 +106,59 @@ func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, } return &Fragment{Fragment: frag} } + +// Row returns a Row for a given field. +func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + if err != nil { + panic(err) + } + row, err := f.Row(rowID) + if err != nil { + panic(err) + } + return row +} + +// ViewRow returns a Row for a given field and view. +func (h *Holder) ViewRow(index, field, view string, rowID uint64) *pilosa.Row { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + if err != nil { + panic(err) + } + row, err := f.ViewRow(view, rowID) + if err != nil { + panic(err) + } + return row +} + +// SetBit clears a bit on the given field. +func (h *Holder) SetBit(index, field string, rowID, columnID uint64) { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + if err != nil { + panic(err) + } + f.SetBit(pilosa.ViewStandard, rowID, columnID, nil) +} + +// ClearBit clears a bit on the given field. +func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + if err != nil { + panic(err) + } + f.ClearBit(pilosa.ViewStandard, rowID, columnID, nil) +} + +// MustSetBits sets columns on a row. Panic on error. +// This function does not accept a timestamp or quantum. +func (h *Holder) MustSetBits(index, field string, rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + h.SetBit(index, field, rowID, columnID) + } +} diff --git a/view.go b/view.go index d76f33d59..a7aaf0cb8 100644 --- a/view.go +++ b/view.go @@ -82,20 +82,8 @@ func NewView(path, index, field, name string, cacheSize uint32) *View { } } -// Name returns the name the view was initialized with. -func (v *View) Name() string { return v.name } - -// Index returns the index name the view was initialized with. -func (v *View) Index() string { return v.index } - -// Field returns the field name the view was initialized with. -func (v *View) Field() string { return v.field } - -// Path returns the path the view was initialized with. -func (v *View) Path() string { return v.path } - -// Open opens and initializes the view. -func (v *View) Open() error { +// open opens and initializes the view. +func (v *View) open() error { // Never keep a cache for field views. if strings.HasPrefix(v.name, viewBSIGroupPrefix) { @@ -116,7 +104,7 @@ func (v *View) Open() error { return nil }(); err != nil { - v.Close() + v.close() return err } @@ -149,19 +137,19 @@ func (v *View) openFragments() error { continue } - frag := v.newFragment(v.FragmentPath(slice), slice) + frag := v.newFragment(v.fragmentPath(slice), slice) if err := frag.Open(); err != nil { - return fmt.Errorf("open fragment: slice=%d, err=%s", frag.Slice(), err) + return fmt.Errorf("open fragment: slice=%d, err=%s", frag.slice, err) } frag.RowAttrStore = v.RowAttrStore - v.fragments[frag.Slice()] = frag + v.fragments[frag.slice] = frag } return nil } -// Close closes the view and its fragments. -func (v *View) Close() error { +// close closes the view and its fragments. +func (v *View) close() error { v.mu.Lock() defer v.mu.Unlock() @@ -176,8 +164,8 @@ func (v *View) Close() error { return nil } -// MaxSlice returns the max slice in the view. -func (v *View) MaxSlice() uint64 { +// calculateMaxSlice returns the max slice in the view. +func (v *View) calculateMaxSlice() uint64 { v.mu.RLock() defer v.mu.RUnlock() @@ -191,8 +179,8 @@ func (v *View) MaxSlice() uint64 { return max } -// FragmentPath returns the path to a fragment in the view. -func (v *View) FragmentPath(slice uint64) string { +// fragmentPath returns the path to a fragment in the view. +func (v *View) fragmentPath(slice uint64) string { return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10)) } @@ -205,8 +193,8 @@ func (v *View) Fragment(slice uint64) *Fragment { func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] } -// Fragments returns a list of all fragments in the view. -func (v *View) Fragments() []*Fragment { +// allFragments returns a list of all fragments in the view. +func (v *View) allFragments() []*Fragment { v.mu.Lock() defer v.mu.Unlock() @@ -217,9 +205,9 @@ func (v *View) Fragments() []*Fragment { return other } -// RecalculateCaches recalculates the cache on every fragment in the view. -func (v *View) RecalculateCaches() { - for _, fragment := range v.Fragments() { +// recalculateCaches recalculates the cache on every fragment in the view. +func (v *View) recalculateCaches() { + for _, fragment := range v.allFragments() { fragment.RecalculateCache() } } @@ -238,7 +226,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { } // Initialize and open fragment. - frag := v.newFragment(v.FragmentPath(slice), slice) + frag := v.newFragment(v.fragmentPath(slice), slice) if err := frag.Open(); err != nil { return nil, errors.Wrap(err, "opening fragment") } @@ -273,8 +261,8 @@ func (v *View) newFragment(path string, slice uint64) *Fragment { return frag } -// DeleteFragment removes the fragment from the view. -func (v *View) DeleteFragment(slice uint64) error { +// deleteFragment removes the fragment from the view. +func (v *View) deleteFragment(slice uint64) error { fragment := v.fragments[slice] if fragment == nil { @@ -289,12 +277,12 @@ func (v *View) DeleteFragment(slice uint64) error { } // Delete fragment file. - if err := os.Remove(fragment.Path()); err != nil { + if err := os.Remove(fragment.path); err != nil { return errors.Wrap(err, "deleting fragment file") } // Delete fragment cache file. - if err := os.Remove(fragment.CachePath()); err != nil { + if err := os.Remove(fragment.cachePath()); err != nil { v.Logger.Printf("no cache file to delete for slice %d", slice) } @@ -303,24 +291,38 @@ func (v *View) DeleteFragment(slice uint64) error { return nil } -// SetBit sets a bit within the view. -func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { - slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - return changed, err +// row returns a row for a slice of the view. +func (v *View) row(rowID uint64) *Row { + row := NewRow() + for _, frag := range v.allFragments() { + fr := frag.row(rowID) + if fr == nil { + continue + } + row.Merge(fr) } - return frag.SetBit(rowID, columnID) + return row + } -// ClearBit clears a bit within the view. -func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) { +// setBit sets a bit within the view. +func (v *View) setBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return changed, err } - return frag.ClearBit(rowID, columnID) + return frag.setBit(rowID, columnID) +} + +// clearBit clears a bit within the view. +func (v *View) clearBit(rowID, columnID uint64) (changed bool, err error) { + slice := columnID / SliceWidth + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + return changed, err + } + return frag.clearBit(rowID, columnID) } // value uses a column of bits to read a multi-bit value. @@ -330,7 +332,7 @@ func (v *View) value(columnID uint64, bitDepth uint) (value uint64, exists bool, if err != nil { return value, exists, err } - return frag.Value(columnID, bitDepth) + return frag.value(columnID, bitDepth) } // setValue uses a column of bits to set a multi-bit value. @@ -340,13 +342,13 @@ func (v *View) setValue(columnID uint64, bitDepth uint, value uint64) (changed b if err != nil { return changed, err } - return frag.SetValue(columnID, bitDepth, value) + return frag.setValue(columnID, bitDepth, value) } // sum returns the sum & count of a field. func (v *View) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { - for _, f := range v.Fragments() { - fsum, fcount, err := f.Sum(filter, bitDepth) + for _, f := range v.allFragments() { + fsum, fcount, err := f.sum(filter, bitDepth) if err != nil { return sum, count, err } @@ -359,8 +361,8 @@ func (v *View) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { // min returns the min and count of a field. func (v *View) min(filter *Row, bitDepth uint) (min, count uint64, err error) { var minHasValue bool - for _, f := range v.Fragments() { - fmin, fcount, err := f.Min(filter, bitDepth) + for _, f := range v.allFragments() { + fmin, fcount, err := f.min(filter, bitDepth) if err != nil { return min, count, err } @@ -386,8 +388,8 @@ func (v *View) min(filter *Row, bitDepth uint) (min, count uint64, err error) { // max returns the max and count of a field. func (v *View) max(filter *Row, bitDepth uint) (max, count uint64, err error) { - for _, f := range v.Fragments() { - fmax, fcount, err := f.Max(filter, bitDepth) + for _, f := range v.allFragments() { + fmax, fcount, err := f.max(filter, bitDepth) if err != nil { return max, count, err } @@ -402,8 +404,8 @@ func (v *View) max(filter *Row, bitDepth uint) (max, count uint64, err error) { // rangeOp returns rows with a field value encoding matching the predicate. func (v *View) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { r := NewRow() - for _, frag := range v.Fragments() { - other, err := frag.RangeOp(op, bitDepth, predicate) + for _, frag := range v.allFragments() { + other, err := frag.rangeOp(op, bitDepth, predicate) if err != nil { return nil, err } @@ -416,8 +418,8 @@ func (v *View) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, err // value between predicateMin and predicateMax. func (v *View) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { r := NewRow() - for _, frag := range v.Fragments() { - other, err := frag.RangeBetween(bitDepth, predicateMin, predicateMax) + for _, frag := range v.allFragments() { + other, err := frag.rangeBetween(bitDepth, predicateMin, predicateMax) if err != nil { return nil, err } diff --git a/view_internal_test.go b/view_internal_test.go new file mode 100644 index 000000000..d0e8bfdd1 --- /dev/null +++ b/view_internal_test.go @@ -0,0 +1,68 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "io/ioutil" + "testing" +) + +// mustOpenView returns a new instance of View with a temporary path. +func mustOpenView(index, field, name string) *View { + path, err := ioutil.TempDir("", "pilosa-view-") + if err != nil { + panic(err) + } + + v := NewView(path, index, field, name, DefaultCacheSize) + if err := v.open(); err != nil { + panic(err) + } + v.RowAttrStore = newMemAttrStore() + return v +} + +// Ensure view can open and retrieve a fragment. +func TestView_DeleteFragment(t *testing.T) { + v := mustOpenView("i", "f", "v") + defer v.close() + + slice := uint64(9) + + // Create fragment. + fragment, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + t.Fatal(err) + } else if fragment == nil { + t.Fatal("expected fragment") + } + + err = v.deleteFragment(slice) + if err != nil { + t.Fatal(err) + } + + if v.Fragment(slice) != nil { + t.Fatal("fragment still exists in view") + } + + // Recreate fragment with same slice, verify that the old fragment was not reused. + fragment2, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + t.Fatal(err) + } else if fragment == fragment2 { + t.Fatal("failed to create new fragment") + } +} diff --git a/view_test.go b/view_test.go deleted file mode 100644 index 217c4bbb4..000000000 --- a/view_test.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/test" -) - -// View is a test wrapper for pilosa.View. -type View struct { - *pilosa.View - RowAttrStore pilosa.AttrStore -} - -// NewView returns a new instance of View with a temporary path. -func NewView(index, field, name string) *View { - path, err := ioutil.TempDir("", "pilosa-view-") - if err != nil { - panic(err) - } - - v := &View{ - View: pilosa.NewView(path, index, field, name, pilosa.DefaultCacheSize), - RowAttrStore: test.MustOpenAttrStore(), - } - v.View.RowAttrStore = v.RowAttrStore - return v -} - -// MustOpenView creates and opens an view at a temporary path. Panic on error. -func MustOpenView(index, field, name string) *View { - v := NewView(index, field, name) - if err := v.Open(); err != nil { - panic(err) - } - return v -} - -// Close closes the view and removes all underlying data. -func (v *View) Close() error { - defer os.Remove(v.Path()) - defer v.RowAttrStore.Close() - return v.View.Close() -} - -// Reopen closes the view and reopens it as a new instance. -func (v *View) Reopen() error { - path := v.Path() - if err := v.View.Close(); err != nil { - return err - } - - v.View = pilosa.NewView(path, v.Index(), v.Field(), v.Name(), pilosa.DefaultCacheSize) - v.View.RowAttrStore = v.RowAttrStore - return v.Open() -} - -// MustSetBits sets columns on a row. Panic on error. -// This function does not accept a timestamp or quantum. -func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := v.SetBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// MustClearColumns clears columns on a row. Panic on error. -func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := v.ClearBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// Ensure view can open and retrieve a fragment. -func TestView_DeleteFragment(t *testing.T) { - v := MustOpenView("i", "f", "v") - defer v.Close() - - slice := uint64(9) - - // Create fragment. - fragment, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - t.Fatal(err) - } else if fragment == nil { - t.Fatal("expected fragment") - } - - err = v.DeleteFragment(slice) - if err != nil { - t.Fatal(err) - } - - if v.Fragment(slice) != nil { - t.Fatal("fragment still exists in view") - } - - // Recreate fragment with same slice, verify that the old fragment was not reused. - fragment2, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - t.Fatal(err) - } else if fragment == fragment2 { - t.Fatal("failed to create new fragment") - } -}