scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-10 04:47:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into http-inversion

Browse source
This commit is contained in:
Cody Soyland 2018-06-12 14:49:54 -05:00
commit 0b16a3afb9
18 changed files with 658 additions and 740 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
api.go
View file

@ -306,7 +306,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
cw := csv.NewWriter(w)
// Iterate over each column.
if err := f.ForEachBit(func(rowID, columnID uint64) error {
if err := f.forEachBit(func(rowID, columnID uint64) error {
return cw.Write([]string{
strconv.FormatUint(rowID, 10),
strconv.FormatUint(columnID, 10),
@ -403,7 +403,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
}
var resp = internal.BlockDataResponse{}
resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block))
resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block))
// Encode response.
buf, err := proto.Marshal(&resp)

2
cluster.go
View file

@ -633,7 +633,7 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost {
for _, field := range idx.Fields() {
for _, view := range field.Views() {
fieldViews.addView(field.Name(), view.Name())
fieldViews.addView(field.Name(), view.name)
}
}

24
executor.go
View file

@ -399,7 +399,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq
return ValCount{}, nil
}
vsum, vcount, err := fragment.Sum(filter, bsig.BitDepth())
vsum, vcount, err := fragment.sum(filter, bsig.BitDepth())
if err != nil {
return ValCount{}, errors.Wrap(err, "computing sum")
}
@ -437,7 +437,7 @@ func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Cal
return ValCount{}, nil
}
fmin, fcount, err := fragment.Min(filter, bsig.BitDepth())
fmin, fcount, err := fragment.min(filter, bsig.BitDepth())
if err != nil {
return ValCount{}, err
}
@ -475,7 +475,7 @@ func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Cal
return ValCount{}, nil
}
fmax, fcount, err := fragment.Max(filter, bsig.BitDepth())
fmax, fcount, err := fragment.max(filter, bsig.BitDepth())
if err != nil {
return ValCount{}, err
}
@ -602,7 +602,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
if tanimotoThreshold > 100 {
return nil, errors.New("Tanimoto Threshold is from 1 to 100 only")
}
return f.Top(TopOptions{
return f.top(TopOptions{
N: int(n),
Src: src,
RowIDs: rowIDs,
@ -664,7 +664,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
if frag == nil {
return NewRow(), nil
}
return frag.Row(rowID), nil
return frag.row(rowID), nil
}
// executeIntersectSlice executes a intersect() call for a local slice.
@ -756,7 +756,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
if f == nil {
continue
}
row = row.Union(f.Row(rowID))
row = row.Union(f.row(rowID))
}
f.Stats.Count("range", 1, 1.0)
return row, nil
@ -808,7 +808,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string,
return NewRow(), nil
}
return frag.NotNull(bsig.BitDepth())
return frag.notNull(bsig.BitDepth())
} else if cond.Op == pql.BETWEEN {
@ -846,10 +846,10 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string,
// If the query is asking for the entire valid range, just return
// the not-null bitmap for the bsiGroup.
if predicates[0] <= bsig.Min && predicates[1] >= bsig.Max {
return frag.NotNull(bsig.BitDepth())
return frag.notNull(bsig.BitDepth())
}
return frag.RangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax)
return frag.rangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax)
} else {
@ -879,16 +879,16 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string,
// LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range.
if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) ||
(cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) {
return frag.NotNull(bsig.BitDepth())
return frag.notNull(bsig.BitDepth())
}
// outOfRange for NEQ should return all not-null.
if outOfRange && cond.Op == pql.NEQ {
return frag.NotNull(bsig.BitDepth())
return frag.notNull(bsig.BitDepth())
}
f.Stats.Count("range:bsigroup", 1, 1.0)
return frag.RangeOp(cond.Op, bsig.BitDepth(), baseValue)
return frag.rangeOp(cond.Op, bsig.BitDepth(), baseValue)
}
}

146
executor_test.go
View file

@ -107,11 +107,11 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
func TestExecutor_Execute_Difference(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4)
hldr.SetBit("i", "general", 10, 1)
hldr.SetBit("i", "general", 10, 2)
hldr.SetBit("i", "general", 10, 3)
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, 4)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
@ -125,7 +125,7 @@ func TestExecutor_Execute_Difference(t *testing.T) {
func TestExecutor_Execute_Empty_Difference(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
hldr.SetBit("i", "general", 10, 1)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil {
@ -137,13 +137,13 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) {
func TestExecutor_Execute_Intersect(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.SetBit("i", "general", 10, 1)
hldr.SetBit("i", "general", 10, SliceWidth+1)
hldr.SetBit("i", "general", 10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
hldr.SetBit("i", "general", 11, 1)
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
@ -168,12 +168,12 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) {
func TestExecutor_Execute_Union(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.SetBit("i", "general", 10, 0)
hldr.SetBit("i", "general", 10, SliceWidth+1)
hldr.SetBit("i", "general", 10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
@ -187,7 +187,7 @@ func TestExecutor_Execute_Union(t *testing.T) {
func TestExecutor_Execute_Empty_Union(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
hldr.SetBit("i", "general", 10, 0)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil {
@ -201,12 +201,12 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) {
func TestExecutor_Execute_Xor(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.SetBit("i", "general", 10, 0)
hldr.SetBit("i", "general", 10, SliceWidth+1)
hldr.SetBit("i", "general", 10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
@ -220,9 +220,9 @@ func TestExecutor_Execute_Xor(t *testing.T) {
func TestExecutor_Execute_Count(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.SetBit("i", "f", 10, 3)
hldr.SetBit("i", "f", 10, SliceWidth+1)
hldr.SetBit("i", "f", 10, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil {
@ -237,9 +237,11 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// set a bit so the view gets created.
hldr.SetBit("i", "f", 1, 0)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if n := f.Row(11).Count(); n != 0 {
if n := hldr.Row("i", "f", 11).Count(); n != 0 {
t.Fatalf("unexpected bitmap count: %d", n)
}
@ -251,7 +253,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
}
}
if n := f.Row(11).Count(); n != 1 {
if n := hldr.Row("i", "f", 11).Count(); n != 1 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil {
@ -427,12 +429,12 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
defer hldr.Close()
// Set columns for rows 0, 10, & 20 across two slices.
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth)
hldr.SetBit("i", "f", 0, 0)
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", 0, 2)
hldr.SetBit("i", "f", 0, SliceWidth)
hldr.SetBit("i", "f", 1, SliceWidth+2)
hldr.SetBit("i", "f", 1, SliceWidth)
// Execute query.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
@ -450,23 +452,23 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth)
hldr.SetBit("i", "f", 0, 0)
hldr.SetBit("i", "f", 0, SliceWidth)
hldr.SetBit("i", "f", 0, 2*SliceWidth)
hldr.SetBit("i", "f", 0, 3*SliceWidth)
hldr.SetBit("i", "f", 0, 4*SliceWidth)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 1)
hldr.SetBit("i", "f", 1, 0)
hldr.SetBit("i", "f", 1, 1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1)
hldr.SetBit("i", "f", 2, SliceWidth)
hldr.SetBit("i", "f", 2, SliceWidth+1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1)
hldr.SetBit("i", "f", 3, 2*SliceWidth)
hldr.SetBit("i", "f", 3, 2*SliceWidth+1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1)
hldr.SetBit("i", "f", 4, 3*SliceWidth)
hldr.SetBit("i", "f", 4, 3*SliceWidth+1)
// Execute query.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
@ -485,19 +487,19 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
defer hldr.Close()
// Set columns for rows 0, 10, & 20 across two slices.
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2)
hldr.SetBit("i", "f", 0, 0)
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", 0, SliceWidth)
hldr.SetBit("i", "f", 10, SliceWidth)
hldr.SetBit("i", "f", 10, SliceWidth+1)
hldr.SetBit("i", "f", 20, SliceWidth)
hldr.SetBit("i", "f", 20, SliceWidth+1)
hldr.SetBit("i", "f", 20, SliceWidth+2)
// Create an intersecting row.
hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth)
hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1)
hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2)
hldr.SetBit("i", "other", 100, SliceWidth)
hldr.SetBit("i", "other", 100, SliceWidth+1)
hldr.SetBit("i", "other", 100, SliceWidth+2)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
@ -521,9 +523,9 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
//
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
hldr.SetBit("i", "f", 0, 0)
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", 10, SliceWidth)
if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
t.Fatal(err)
@ -544,9 +546,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
//
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
hldr.SetBit("i", "f", 0, 0)
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", 10, SliceWidth)
if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
t.Fatal(err)
@ -989,7 +991,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.API.Holder = hldr.Holder
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
hldr.SetBit("i", "f", 10, SliceWidth+1)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil {
@ -1023,8 +1025,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.API.Holder = hldr.Holder
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2)
hldr.SetBit("i", "f", 10, (2*SliceWidth)+1)
hldr.SetBit("i", "f", 10, (2*SliceWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil {
@ -1078,7 +1080,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
}
// Verify that one column is set on both node's holder.
if n := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 {
if n := hldr.Row("i", "f", 10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
if !remoteCalled {
@ -1132,7 +1134,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
}
// Verify that one column is set on both node's holder.
if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 {
if n := hldr.ViewRow("i", "f", "standard_2016", 10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
if !remoteCalled {
@ -1192,8 +1194,8 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.API.Holder = hldr.Holder
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2)
hldr.SetBit("i", "f", 30, (2*SliceWidth)+1)
hldr.SetBit("i", "f", 30, (4*SliceWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=3)`), nil, nil); err != nil {

58
field.go
View file

@ -137,7 +137,7 @@ func (f *Field) MaxSlice() uint64 {
var max uint64
for _, view := range f.views {
if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max {
if viewMaxSlice := view.calculateMaxSlice(); viewMaxSlice > max {
max = viewMaxSlice
}
}
@ -249,11 +249,11 @@ func (f *Field) openViews() error {
name := filepath.Base(fi.Name())
view := f.newView(f.ViewPath(name), name)
if err := view.Open(); err != nil {
return fmt.Errorf("opening view: view=%s, err=%s", view.Name(), err)
if err := view.open(); err != nil {
return fmt.Errorf("opening view: view=%s, err=%s", view.name, err)
}
view.RowAttrStore = f.rowAttrStore
f.views[view.Name()] = view
f.views[view.name] = view
}
return nil
@ -369,7 +369,7 @@ func (f *Field) Close() error {
// Close all views.
for _, view := range f.views {
if err := view.Close(); err != nil {
if err := view.close(); err != nil {
return err
}
}
@ -447,9 +447,9 @@ func (f *Field) deleteBSIGroupAndView(name string) error {
if view := f.views[viewName]; view != nil {
delete(f.views, viewName)
if err := view.Close(); err != nil {
if err := view.close(); err != nil {
return errors.Wrap(err, "closing view")
} else if err := os.RemoveAll(view.Path()); err != nil {
} else if err := os.RemoveAll(view.path); err != nil {
return errors.Wrap(err, "deleting directory")
}
}
@ -538,7 +538,7 @@ func (f *Field) viewNames() []string {
// RecalculateCaches recalculates caches on every view in the field.
func (f *Field) RecalculateCaches() {
for _, view := range f.Views() {
view.RecalculateCaches()
view.recalculateCaches()
}
}
@ -579,11 +579,11 @@ func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) {
view := f.newView(f.ViewPath(name), name)
if err := view.Open(); err != nil {
if err := view.open(); err != nil {
return nil, false, errors.Wrap(err, "opening view")
}
view.RowAttrStore = f.rowAttrStore
f.views[view.Name()] = view
f.views[view.name] = view
return view, true, nil
}
@ -606,12 +606,12 @@ func (f *Field) DeleteView(name string) error {
}
// Close data files before deletion.
if err := view.Close(); err != nil {
if err := view.close(); err != nil {
return errors.Wrap(err, "closing view")
}
// Delete view directory.
if err := os.RemoveAll(view.Path()); err != nil {
if err := os.RemoveAll(view.path); err != nil {
return errors.Wrap(err, "deleting directory")
}
@ -620,6 +620,28 @@ func (f *Field) DeleteView(name string) error {
return nil
}
// Row returns a row of the standard view.
func (f *Field) Row(rowID uint64) (*Row, error) {
if f.Type() != FieldTypeSet {
return nil, errors.Errorf("row method unsupported for field type: %s", f.Type())
}
view := f.View(ViewStandard)
if view == nil {
return nil, ErrInvalidView
}
return view.row(rowID), nil
}
// ViewRow returns a row for a view and slice.
// TODO: unexport this with views (it's only used in tests).
func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) {
view := f.View(viewName)
if view == nil {
return nil, ErrInvalidView
}
return view.row(rowID), nil
}
// SetBit sets a bit on a view within the field.
func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) {
// Validate view name.
@ -634,7 +656,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed
}
// Set non-time bit.
if v, err := view.SetBit(rowID, colID); err != nil {
if v, err := view.setBit(rowID, colID); err != nil {
return changed, errors.Wrap(err, "setting on view")
} else if v {
changed = v
@ -652,7 +674,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed
return changed, errors.Wrapf(err, "creating view %s", subname)
}
if c, err := view.SetBit(rowID, colID); err != nil {
if c, err := view.setBit(rowID, colID); err != nil {
return changed, errors.Wrapf(err, "setting on view %s", subname)
} else if c {
changed = true
@ -676,7 +698,7 @@ func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (change
}
// Clear non-time bit.
if v, err := view.ClearBit(rowID, colID); err != nil {
if v, err := view.clearBit(rowID, colID); err != nil {
return changed, errors.Wrap(err, "clearing on view")
} else if v {
changed = v
@ -694,7 +716,7 @@ func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (change
return changed, errors.Wrapf(err, "creating view %s", subname)
}
if c, err := view.ClearBit(rowID, colID); err != nil {
if c, err := view.clearBit(rowID, colID); err != nil {
return changed, errors.Wrapf(err, "clearing on view %s", subname)
} else if c {
changed = true
@ -905,7 +927,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
return errors.Wrap(err, "creating view")
}
if err := frag.Import(data.RowIDs, data.ColumnIDs); err != nil {
if err := frag.bulkImport(data.RowIDs, data.ColumnIDs); err != nil {
return err
}
}
@ -962,7 +984,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error {
baseValues[i] = uint64(value - bsig.Min)
}
if err := frag.ImportValue(data.ColumnIDs, baseValues, bsig.BitDepth()); err != nil {
if err := frag.importValue(data.ColumnIDs, baseValues, bsig.BitDepth()); err != nil {
return err
}
}

244
fragment.go
View file

@ -130,27 +130,8 @@ func NewFragment(path, index, field, view string, slice uint64) *Fragment {
}
}
// Path returns the path the fragment was initialized with.
func (f *Fragment) Path() string { return f.path }
// CachePath returns the path to the fragment's cache data.
func (f *Fragment) CachePath() string { return f.path + CacheExt }
// Index returns the index that the fragment was initialized with.
func (f *Fragment) Index() string { return f.index }
// Field returns the field the fragment was initialized with.
func (f *Fragment) Field() string { return f.field }
// View returns the view the fragment was initialized with.
func (f *Fragment) View() string { return f.view }
// Slice returns the slice the fragment was initialized with.
func (f *Fragment) Slice() uint64 { return f.slice }
// Cache returns the fragment's cache.
// This is not safe for concurrent use.
func (f *Fragment) Cache() Cache { return f.cache }
// cachePath returns the path to the fragment's cache data.
func (f *Fragment) cachePath() string { return f.path + CacheExt }
// Open opens the underlying storage.
func (f *Fragment) Open() error {
@ -261,7 +242,7 @@ func (f *Fragment) openCache() error {
}
// Read cache data from disk.
path := f.CachePath()
path := f.cachePath()
buf, err := ioutil.ReadFile(path)
if os.IsNotExist(err) {
return nil
@ -342,14 +323,14 @@ func (f *Fragment) closeStorage() error {
return nil
}
// Row returns a row by ID.
func (f *Fragment) Row(rowID uint64) *Row {
// row returns a row by ID.
func (f *Fragment) row(rowID uint64) *Row {
f.mu.Lock()
defer f.mu.Unlock()
return f.row(rowID, true, true)
return f.unprotectedRow(rowID, true, true)
}
func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *Row {
func (f *Fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCache bool) *Row {
if checkRowCache {
r, ok := f.rowCache.Fetch(rowID)
if ok && r != nil {
@ -380,15 +361,15 @@ func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *R
return row
}
// SetBit sets a bit for a given column & row within the fragment.
// setBit sets a bit for a given column & row within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) {
func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.setBit(rowID, columnID)
return f.unprotectedSetBit(rowID, columnID)
}
func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
func (f *Fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err error) {
changed = false
// Determine the position of the bit in the storage.
pos, err := f.pos(rowID, columnID)
@ -415,7 +396,7 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
}
// Get the row from row cache or fragment.storage.
row := f.row(rowID, true, true)
row := f.unprotectedRow(rowID, true, true)
row.SetBit(columnID)
// Update the cache.
@ -432,15 +413,15 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
return changed, nil
}
// ClearBit clears a bit for a given column & row within the fragment.
// clearBit clears a bit for a given column & row within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) {
func (f *Fragment) clearBit(rowID, columnID uint64) (bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.clearBit(rowID, columnID)
return f.unprotectedClearBit(rowID, columnID)
}
func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) {
func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, err error) {
changed = false
// Determine the position of the bit in the storage.
pos, err := f.pos(rowID, columnID)
@ -467,7 +448,7 @@ func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) {
}
// Get the row from cache or fragment.storage.
row := f.row(rowID, true, true)
row := f.unprotectedRow(rowID, true, true)
row.ClearBit(columnID)
// Update the cache.
@ -486,8 +467,8 @@ func (f *Fragment) bit(rowID, columnID uint64) (bool, error) {
return f.storage.Contains(pos), nil
}
// Value uses a column of bits to read a multi-bit value.
func (f *Fragment) Value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
// value uses a column of bits to read a multi-bit value.
func (f *Fragment) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
@ -510,20 +491,20 @@ func (f *Fragment) Value(columnID uint64, bitDepth uint) (value uint64, exists b
return value, true, nil
}
// SetValue uses a column of bits to set a multi-bit value.
func (f *Fragment) SetValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
// setValue uses a column of bits to set a multi-bit value.
func (f *Fragment) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
for i := uint(0); i < bitDepth; i++ {
if value&(1<<i) != 0 {
if c, err := f.setBit(uint64(i), columnID); err != nil {
if c, err := f.unprotectedSetBit(uint64(i), columnID); err != nil {
return changed, err
} else if c {
changed = true
}
} else {
if c, err := f.clearBit(uint64(i), columnID); err != nil {
if c, err := f.unprotectedClearBit(uint64(i), columnID); err != nil {
return changed, err
} else if c {
changed = true
@ -532,7 +513,7 @@ func (f *Fragment) SetValue(columnID uint64, bitDepth uint, value uint64) (chang
}
// Mark value as set.
if c, err := f.setBit(uint64(bitDepth), columnID); err != nil {
if c, err := f.unprotectedSetBit(uint64(bitDepth), columnID); err != nil {
return changed, errors.Wrap(err, "marking not-null")
} else if c {
changed = true
@ -582,11 +563,11 @@ func (f *Fragment) importSetValue(columnID uint64, bitDepth uint, value uint64)
return changed, nil
}
// Sum returns the sum of a given bsiGroup as well as the number of columns involved.
// sum returns the sum of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) Sum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
func (f *Fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
// Compute count based on the existence row.
row := f.Row(uint64(bitDepth))
row := f.row(uint64(bitDepth))
if filter != nil {
count = row.IntersectionCount(filter)
} else {
@ -601,7 +582,7 @@ func (f *Fragment) Sum(filter *Row, bitDepth uint) (sum, count uint64, err error
// 10*(2^0) + 4*(2^1) + 3*(2^2) = 30
//
for i := uint(0); i < bitDepth; i++ {
row := f.Row(uint64(i))
row := f.row(uint64(i))
cnt := uint64(0)
if filter != nil {
cnt = row.IntersectionCount(filter)
@ -614,11 +595,11 @@ func (f *Fragment) Sum(filter *Row, bitDepth uint) (sum, count uint64, err error
return sum, count, nil
}
// Min returns the min of a given bsiGroup as well as the number of columns involved.
// min returns the min of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) Min(filter *Row, bitDepth uint) (min, count uint64, err error) {
func (f *Fragment) min(filter *Row, bitDepth uint) (min, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
consider := f.row(uint64(bitDepth))
if filter != nil {
consider = consider.Intersect(filter)
}
@ -630,7 +611,7 @@ func (f *Fragment) Min(filter *Row, bitDepth uint) (min, count uint64, err error
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitDepth-1) to 0
row := f.Row(uint64(ii))
row := f.row(uint64(ii))
x := consider.Difference(row)
count = x.Count()
@ -647,11 +628,11 @@ func (f *Fragment) Min(filter *Row, bitDepth uint) (min, count uint64, err error
return min, count, nil
}
// Max returns the max of a given bsiGroup as well as the number of columns involved.
// max returns the max of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) Max(filter *Row, bitDepth uint) (max, count uint64, err error) {
func (f *Fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
consider := f.row(uint64(bitDepth))
if filter != nil {
consider = consider.Intersect(filter)
}
@ -663,7 +644,7 @@ func (f *Fragment) Max(filter *Row, bitDepth uint) (max, count uint64, err error
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitDepth-1) to 0
row := f.Row(uint64(ii))
row := f.row(uint64(ii))
x := row.Intersect(consider)
count = x.Count()
@ -678,8 +659,8 @@ func (f *Fragment) Max(filter *Row, bitDepth uint) (max, count uint64, err error
return max, count, nil
}
// RangeOp returns bitmaps with a bsiGroup value encoding matching the predicate.
func (f *Fragment) RangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) {
// rangeOp returns bitmaps with a bsiGroup value encoding matching the predicate.
func (f *Fragment) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) {
switch op {
case pql.EQ:
return f.rangeEQ(bitDepth, predicate)
@ -696,11 +677,11 @@ func (f *Fragment) RangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row,
func (f *Fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) {
// Start with set of columns with values set.
b := f.Row(uint64(bitDepth))
b := f.row(uint64(bitDepth))
// Filter any bits that don't match the current bit value.
for i := int(bitDepth - 1); i >= 0; i-- {
row := f.Row(uint64(i))
row := f.row(uint64(i))
bit := (predicate >> uint(i)) & 1
if bit == 1 {
@ -715,7 +696,7 @@ func (f *Fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) {
func (f *Fragment) rangeNEQ(bitDepth uint, predicate uint64) (*Row, error) {
// Start with set of columns with values set.
b := f.Row(uint64(bitDepth))
b := f.row(uint64(bitDepth))
// Get the equal bitmap.
eq, err := f.rangeEQ(bitDepth, predicate)
@ -733,12 +714,12 @@ func (f *Fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool)
keep := NewRow()
// Start with set of columns with values set.
b := f.Row(uint64(bitDepth))
b := f.row(uint64(bitDepth))
// Filter any bits that don't match the current bit value.
leadingZeros := true
for i := int(bitDepth - 1); i >= 0; i-- {
row := f.Row(uint64(i))
row := f.row(uint64(i))
bit := (predicate >> uint(i)) & 1
// Remove any columns with higher bits set.
@ -778,12 +759,12 @@ func (f *Fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool)
}
func (f *Fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
b := f.Row(uint64(bitDepth))
b := f.row(uint64(bitDepth))
keep := NewRow()
// Filter any bits that don't match the current bit value.
for i := int(bitDepth - 1); i >= 0; i-- {
row := f.Row(uint64(i))
row := f.row(uint64(i))
bit := (predicate >> uint(i)) & 1
// Handle last bit differently.
@ -812,20 +793,20 @@ func (f *Fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool)
return b, nil
}
// NotNull returns the not-null row (stored at bitDepth).
func (f *Fragment) NotNull(bitDepth uint) (*Row, error) {
return f.Row(uint64(bitDepth)), nil
// notNull returns the not-null row (stored at bitDepth).
func (f *Fragment) notNull(bitDepth uint) (*Row, error) {
return f.row(uint64(bitDepth)), nil
}
// RangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax.
func (f *Fragment) RangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
b := f.Row(uint64(bitDepth))
// rangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax.
func (f *Fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
b := f.row(uint64(bitDepth))
keep1 := NewRow() // GTE
keep2 := NewRow() // LTE
// Filter any bits that don't match the current bit value.
for i := int(bitDepth - 1); i >= 0; i-- {
row := f.Row(uint64(i))
row := f.row(uint64(i))
bit1 := (predicateMin >> uint(i)) & 1
bit2 := (predicateMax >> uint(i)) & 1
@ -864,12 +845,12 @@ func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) {
if columnID < minColumnID || columnID >= minColumnID+SliceWidth {
return 0, errors.New("column out of bounds")
}
return Pos(rowID, columnID), nil
return pos(rowID, columnID), nil
}
// ForEachBit executes fn for every bit set in the fragment.
// forEachBit executes fn for every bit set in the fragment.
// Errors returned from fn are passed through.
func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error {
func (f *Fragment) forEachBit(fn func(rowID, columnID uint64) error) error {
f.mu.Lock()
defer f.mu.Unlock()
@ -886,10 +867,10 @@ func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error {
return err
}
// Top returns the top rows from the fragment.
// top returns the top rows from the fragment.
// If opt.Src is specified then only rows which intersect src are returned.
// If opt.FilterValues exist then the row attribute specified by field is matched.
func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
func (f *Fragment) top(opt TopOptions) ([]Pair, error) {
// Retrieve pairs. If no row ids specified then return from cache.
pairs := f.topBitmapPairs(opt.RowIDs)
@ -960,7 +941,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// Calculate count and append.
count := cnt
if opt.Src != nil {
count = opt.Src.IntersectionCount(f.Row(rowID))
count = opt.Src.IntersectionCount(f.row(rowID))
}
if count == 0 {
continue
@ -1004,7 +985,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// Calculate the intersecting column count and skip if it's below our
// last row in our current result set.
count := opt.Src.IntersectionCount(f.Row(rowID))
count := opt.Src.IntersectionCount(f.row(rowID))
if count < threshold {
continue
}
@ -1048,7 +1029,7 @@ func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair {
continue
}
row := f.Row(rowID)
row := f.row(rowID)
if row.Count() > 0 {
// Otherwise load from storage.
pairs = append(pairs, BitmapPair{
@ -1089,13 +1070,6 @@ func (f *Fragment) Checksum() []byte {
return h.Sum(nil)
}
// BlockN returns the number of blocks in the fragment.
func (f *Fragment) BlockN() int {
f.mu.Lock()
defer f.mu.Unlock()
return int(f.storage.Max() / (HashBlockSize * SliceWidth))
}
// InvalidateChecksums clears all cached block checksums.
func (f *Fragment) InvalidateChecksums() {
f.mu.Lock()
@ -1184,8 +1158,8 @@ func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i
}
}
// BlockData returns bits in a block as row & column ID pairs.
func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) {
// blockData returns bits in a block as row & column ID pairs.
func (f *Fragment) blockData(id int) (rowIDs, columnIDs []uint64) {
f.mu.Lock()
defer f.mu.Unlock()
@ -1196,17 +1170,17 @@ func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) {
return
}
// MergeBlock compares the block's bits and computes a diff with another set of block bits.
// mergeBlock compares the block's bits and computes a diff with another set of block bits.
// The state of a bit is determined by consensus from all blocks being considered.
//
// For example, if 3 blocks are compared and two have a set bit and one has a
// cleared bit then the bit is considered cleared. The function returns the
// diff per incoming block so that all can be in sync.
func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, err error) {
func (f *Fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, err error) {
// Ensure that all pair sets are of equal length.
for i := range data {
if len(data[i].RowIDs) != len(data[i].ColumnIDs) {
return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].RowIDs), len(data[i].ColumnIDs))
if len(data[i].rowIDs) != len(data[i].columnIDs) {
return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].rowIDs), len(data[i].columnIDs))
}
}
@ -1214,8 +1188,8 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
defer f.mu.Unlock()
// Track sets and clears for all blocks (including local).
sets = make([]PairSet, len(data)+1)
clears = make([]PairSet, len(data)+1)
sets = make([]pairSet, len(data)+1)
clears = make([]pairSet, len(data)+1)
// Limit upper row/column pair.
maxRowID := uint64(id+1) * HashBlockSize
@ -1231,7 +1205,7 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
// Append buffered iterators for each incoming block.
for i := range data {
var itr Iterator = NewSliceIterator(data[i].RowIDs, data[i].ColumnIDs)
var itr Iterator = NewSliceIterator(data[i].rowIDs, data[i].columnIDs)
itr = NewLimitIterator(itr, maxRowID, maxColumnID)
itrs = append(itrs, NewBufIterator(itr))
}
@ -1296,25 +1270,25 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
// Append to either the set or clear diff.
if newValue {
sets[i].RowIDs = append(sets[i].RowIDs, min.rowID)
sets[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID)
sets[i].rowIDs = append(sets[i].rowIDs, min.rowID)
sets[i].columnIDs = append(sets[i].columnIDs, min.columnID)
} else {
clears[i].RowIDs = append(sets[i].RowIDs, min.rowID)
clears[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID)
clears[i].rowIDs = append(sets[i].rowIDs, min.rowID)
clears[i].columnIDs = append(sets[i].columnIDs, min.columnID)
}
}
}
// Set local bits.
for i := range sets[0].ColumnIDs {
if _, err := f.setBit(sets[0].RowIDs[i], (f.Slice()*SliceWidth)+sets[0].ColumnIDs[i]); err != nil {
for i := range sets[0].columnIDs {
if _, err := f.unprotectedSetBit(sets[0].rowIDs[i], (f.slice*SliceWidth)+sets[0].columnIDs[i]); err != nil {
return nil, nil, errors.Wrap(err, "setting")
}
}
// Clear local bits.
for i := range clears[0].ColumnIDs {
if _, err := f.clearBit(clears[0].RowIDs[i], (f.Slice()*SliceWidth)+clears[0].ColumnIDs[i]); err != nil {
for i := range clears[0].columnIDs {
if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.slice*SliceWidth)+clears[0].columnIDs[i]); err != nil {
return nil, nil, errors.Wrap(err, "clearing")
}
}
@ -1322,9 +1296,9 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
return sets[1:], clears[1:], nil
}
// Import bulk imports a set of bits and then snapshots the storage.
// bulkImport bulk imports a set of bits and then snapshots the storage.
// This does not affect the fragment's cache.
func (f *Fragment) Import(rowIDs, columnIDs []uint64) error {
func (f *Fragment) bulkImport(rowIDs, columnIDs []uint64) error {
f.mu.Lock()
defer f.mu.Unlock()
// Verify that there are an equal number of row ids and column ids.
@ -1373,7 +1347,7 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error {
// Import should ALWAYS have row() load a new row from fragment.storage
// because the row that's in rowCache hasn't been updated with
// this import's data.
f.cache.BulkAdd(rowID, f.row(rowID, false, false).Count())
f.cache.BulkAdd(rowID, f.unprotectedRow(rowID, false, false).Count())
}
f.cache.Invalidate()
@ -1392,8 +1366,8 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error {
return nil
}
// ImportValue bulk imports a set of range-encoded values.
func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error {
// importValue bulk imports a set of range-encoded values.
func (f *Fragment) importValue(columnIDs, values []uint64, bitDepth uint) error {
f.mu.Lock()
defer f.mu.Unlock()
// Verify that there are an equal number of column ids and values.
@ -1529,7 +1503,7 @@ func (f *Fragment) flushCache() error {
}
// Write to disk.
if err := ioutil.WriteFile(f.CachePath(), buf, 0666); err != nil {
if err := ioutil.WriteFile(f.cachePath(), buf, 0666); err != nil {
return errors.Wrap(err, "writing")
}
@ -1603,7 +1577,7 @@ func (f *Fragment) writeCacheToArchive(tw *tar.Writer) error {
defer f.mu.Unlock()
// Read cache into buffer.
buf, err := ioutil.ReadFile(f.CachePath())
buf, err := ioutil.ReadFile(f.cachePath())
if os.IsNotExist(err) {
return nil
} else if err != nil {
@ -1697,7 +1671,7 @@ func (f *Fragment) readCacheFromArchive(r io.Reader) error {
buf, err := ioutil.ReadAll(r)
if err != nil {
return errors.Wrap(err, "reading")
} else if err := ioutil.WriteFile(f.CachePath(), buf, 0666); err != nil {
} else if err := ioutil.WriteFile(f.cachePath(), buf, 0666); err != nil {
return errors.Wrap(err, "writing")
}
@ -1762,11 +1736,11 @@ func (s *FragmentSyncer) isClosing() bool {
}
}
// SyncFragment compares checksums for the local and remote fragments and
// syncFragment compares checksums for the local and remote fragments and
// then merges any blocks which have differences.
func (s *FragmentSyncer) SyncFragment() error {
func (s *FragmentSyncer) syncFragment() error {
// Determine replica set.
nodes := s.Cluster.SliceNodes(s.Fragment.Index(), s.Fragment.Slice())
nodes := s.Cluster.SliceNodes(s.Fragment.index, s.Fragment.slice)
if len(nodes) == 1 {
return nil
}
@ -1782,7 +1756,7 @@ func (s *FragmentSyncer) SyncFragment() error {
}
// Retrieve remote blocks.
blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.Index(), s.Fragment.Field(), s.Fragment.Slice())
blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.index, s.Fragment.field, s.Fragment.slice)
if err != nil && err != ErrFragmentNotFound {
return errors.Wrap(err, "getting blocks")
}
@ -1845,9 +1819,9 @@ func (s *FragmentSyncer) syncBlock(id int) error {
f := s.Fragment
// Read pairs from each remote block.
var pairSets []PairSet
var uris []*URI
for _, node := range s.Cluster.SliceNodes(f.Index(), f.Slice()) {
var pairSets []pairSet
for _, node := range s.Cluster.SliceNodes(f.index, f.slice) {
if s.Node.ID == node.ID {
continue
}
@ -1861,14 +1835,14 @@ func (s *FragmentSyncer) syncBlock(id int) error {
uris = append(uris, uri)
// Only sync the standard block.
rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.Index(), f.Field(), f.Slice(), id)
rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.index, f.field, f.slice, id)
if err != nil {
return errors.Wrap(err, "getting block")
}
pairSets = append(pairSets, PairSet{
ColumnIDs: columnIDs,
RowIDs: rowIDs,
pairSets = append(pairSets, pairSet{
columnIDs: columnIDs,
rowIDs: rowIDs,
})
}
@ -1878,7 +1852,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
}
// Merge blocks together.
sets, clears, err := f.MergeBlock(id, pairSets)
sets, clears, err := f.mergeBlock(id, pairSets)
if err != nil {
return errors.Wrap(err, "merging")
}
@ -1889,12 +1863,12 @@ func (s *FragmentSyncer) syncBlock(id int) error {
count := 0
// Ignore if there are no differences.
if len(set.ColumnIDs) == 0 && len(clear.ColumnIDs) == 0 {
if len(set.columnIDs) == 0 && len(clear.columnIDs) == 0 {
continue
}
// Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest.
total := len(set.ColumnIDs) + len(clear.ColumnIDs)
total := len(set.columnIDs) + len(clear.columnIDs)
maxWrites := s.Cluster.MaxWritesPerRequest
if maxWrites <= 0 {
maxWrites = 5000
@ -1902,12 +1876,12 @@ func (s *FragmentSyncer) syncBlock(id int) error {
buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites))))
// Only sync the standard block.
for j := 0; j < len(set.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(field=%q, row=%d, col=%d)\n", f.Field(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
for j := 0; j < len(set.columnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(field=%q, row=%d, col=%d)\n", f.field, set.rowIDs[j], (f.slice*SliceWidth)+set.columnIDs[j])
count++
}
for j := 0; j < len(clear.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(field=%q, row=%d, col=%d)\n", f.Field(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
for j := 0; j < len(clear.columnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(field=%q, row=%d, col=%d)\n", f.field, clear.rowIDs[j], (f.slice*SliceWidth)+clear.columnIDs[j])
count++
}
@ -1923,7 +1897,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
Query: buffers[k].String(),
Remote: true,
}
_, err := s.Cluster.InternalClient.QueryNode(context.Background(), uris[i], f.Index(), queryRequest)
_, err := s.Cluster.InternalClient.QueryNode(context.Background(), uris[i], f.index, queryRequest)
if err != nil {
return errors.Wrap(err, "executing")
}
@ -1941,10 +1915,10 @@ func madvise(b []byte, advice int) (err error) {
return
}
// PairSet is a list of equal length row and column id lists.
type PairSet struct {
RowIDs []uint64
ColumnIDs []uint64
// pairSet is a list of equal length row and column id lists.
type pairSet struct {
rowIDs []uint64
columnIDs []uint64
}
// byteSlicesEqual returns true if all slices are equal.
@ -1961,7 +1935,7 @@ func byteSlicesEqual(a [][]byte) bool {
return true
}
// Pos returns the row position of a row/column pair.
func Pos(rowID, columnID uint64) uint64 {
// pos returns the row position of a row/column pair.
func pos(rowID, columnID uint64) uint64 {
return (rowID * SliceWidth) + (columnID % SliceWidth)
}

256
fragment_internal_test.go
View file

@ -40,27 +40,27 @@ func TestFragment_SetBit(t *testing.T) {
defer f.Close()
// Set bits on the fragment.
if _, err := f.SetBit(120, 1); err != nil {
if _, err := f.setBit(120, 1); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(120, 6); err != nil {
} else if _, err := f.setBit(120, 6); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(121, 0); err != nil {
} else if _, err := f.setBit(121, 0); err != nil {
t.Fatal(err)
}
// Verify counts on rows.
if n := f.Row(120).Count(); n != 2 {
if n := f.row(120).Count(); n != 2 {
t.Fatalf("unexpected count: %d", n)
} else if n := f.Row(121).Count(); n != 1 {
} else if n := f.row(121).Count(); n != 1 {
t.Fatalf("unexpected count: %d", n)
}
// Close and reopen the fragment & verify the data.
if err := f.reopen(); err != nil {
t.Fatal(err)
} else if n := f.Row(120).Count(); n != 2 {
} else if n := f.row(120).Count(); n != 2 {
t.Fatalf("unexpected count (reopen): %d", n)
} else if n := f.Row(121).Count(); n != 1 {
} else if n := f.row(121).Count(); n != 1 {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
@ -71,23 +71,23 @@ func TestFragment_ClearBit(t *testing.T) {
defer f.Close()
// Set and then clear bits on the fragment.
if _, err := f.SetBit(1000, 1); err != nil {
if _, err := f.setBit(1000, 1); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(1000, 2); err != nil {
} else if _, err := f.setBit(1000, 2); err != nil {
t.Fatal(err)
} else if _, err := f.ClearBit(1000, 1); err != nil {
} else if _, err := f.clearBit(1000, 1); err != nil {
t.Fatal(err)
}
// Verify count on row.
if n := f.Row(1000).Count(); n != 1 {
if n := f.row(1000).Count(); n != 1 {
t.Fatalf("unexpected count: %d", n)
}
// Close and reopen the fragment & verify the data.
if err := f.reopen(); err != nil {
t.Fatal(err)
} else if n := f.Row(1000).Count(); n != 1 {
} else if n := f.row(1000).Count(); n != 1 {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
@ -99,14 +99,14 @@ func TestFragment_SetValue(t *testing.T) {
defer f.Close()
// Set value.
if changed, err := f.SetValue(100, 16, 3829); err != nil {
if changed, err := f.setValue(100, 16, 3829); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Read value.
if value, exists, err := f.Value(100, 16); err != nil {
if value, exists, err := f.value(100, 16); err != nil {
t.Fatal(err)
} else if value != 3829 {
t.Fatalf("unexpected value: %d", value)
@ -115,7 +115,7 @@ func TestFragment_SetValue(t *testing.T) {
}
// Setting value should return no change.
if changed, err := f.SetValue(100, 16, 3829); err != nil {
if changed, err := f.setValue(100, 16, 3829); err != nil {
t.Fatal(err)
} else if changed {
t.Fatal("expected no change")
@ -127,21 +127,21 @@ func TestFragment_SetValue(t *testing.T) {
defer f.Close()
// Set value.
if changed, err := f.SetValue(100, 16, 3829); err != nil {
if changed, err := f.setValue(100, 16, 3829); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Overwriting value should overwrite all bits.
if changed, err := f.SetValue(100, 16, 2028); err != nil {
if changed, err := f.setValue(100, 16, 2028); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Read value.
if value, exists, err := f.Value(100, 16); err != nil {
if value, exists, err := f.value(100, 16); err != nil {
t.Fatal(err)
} else if value != 2028 {
t.Fatalf("unexpected value: %d", value)
@ -155,14 +155,14 @@ func TestFragment_SetValue(t *testing.T) {
defer f.Close()
// Set value.
if changed, err := f.SetValue(100, 10, 20); err != nil {
if changed, err := f.setValue(100, 10, 20); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Non-existent value.
if value, exists, err := f.Value(100, 11); err != nil {
if value, exists, err := f.value(100, 11); err != nil {
t.Fatal(err)
} else if value != 0 {
t.Fatalf("unexpected value: %d", value)
@ -191,14 +191,14 @@ func TestFragment_SetValue(t *testing.T) {
m[columnID] = int64(value)
if _, err := f.SetValue(columnID, bitDepth, value); err != nil {
if _, err := f.setValue(columnID, bitDepth, value); err != nil {
t.Fatal(err)
}
}
// Ensure values are set.
for columnID, value := range m {
v, exists, err := f.Value(columnID, bitDepth)
v, exists, err := f.value(columnID, bitDepth)
if err != nil {
t.Fatal(err)
} else if value != int64(v) {
@ -223,18 +223,18 @@ func TestFragment_Sum(t *testing.T) {
defer f.Close()
// Set values.
if _, err := f.SetValue(1000, bitDepth, 382); err != nil {
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.setValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(3000, bitDepth, 2818); err != nil {
} else if _, err := f.setValue(3000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(4000, bitDepth, 300); err != nil {
} else if _, err := f.setValue(4000, bitDepth, 300); err != nil {
t.Fatal(err)
}
t.Run("NoFilter", func(t *testing.T) {
if sum, n, err := f.Sum(nil, bitDepth); err != nil {
if sum, n, err := f.sum(nil, bitDepth); err != nil {
t.Fatal(err)
} else if n != 4 {
t.Fatalf("unexpected count: %d", n)
@ -244,7 +244,7 @@ func TestFragment_Sum(t *testing.T) {
})
t.Run("WithFilter", func(t *testing.T) {
if sum, n, err := f.Sum(NewRow(2000, 4000, 5000), bitDepth); err != nil {
if sum, n, err := f.sum(NewRow(2000, 4000, 5000), bitDepth); err != nil {
t.Fatal(err)
} else if n != 2 {
t.Fatalf("unexpected count: %d", n)
@ -262,19 +262,19 @@ func TestFragment_MinMax(t *testing.T) {
defer f.Close()
// Set values.
if _, err := f.SetValue(1000, bitDepth, 382); err != nil {
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.setValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(3000, bitDepth, 2818); err != nil {
} else if _, err := f.setValue(3000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(4000, bitDepth, 300); err != nil {
} else if _, err := f.setValue(4000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(5000, bitDepth, 2818); err != nil {
} else if _, err := f.setValue(5000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(6000, bitDepth, 2817); err != nil {
} else if _, err := f.setValue(6000, bitDepth, 2817); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(7000, bitDepth, 0); err != nil {
} else if _, err := f.setValue(7000, bitDepth, 0); err != nil {
t.Fatal(err)
}
@ -292,7 +292,7 @@ func TestFragment_MinMax(t *testing.T) {
{filter: NewRow(7000), exp: 0, cnt: 1},
}
for i, test := range tests {
if min, cnt, err := f.Min(test.filter, bitDepth); err != nil {
if min, cnt, err := f.min(test.filter, bitDepth); err != nil {
t.Fatal(err)
} else if min != test.exp {
t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min)
@ -316,7 +316,7 @@ func TestFragment_MinMax(t *testing.T) {
{filter: NewRow(7000), exp: 0, cnt: 1},
}
for i, test := range tests {
if max, cnt, err := f.Max(test.filter, bitDepth); err != nil {
if max, cnt, err := f.max(test.filter, bitDepth); err != nil {
t.Fatal(err)
} else if max != test.exp {
t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max)
@ -336,18 +336,18 @@ func TestFragment_Range(t *testing.T) {
defer f.Close()
// Set values.
if _, err := f.SetValue(1000, bitDepth, 382); err != nil {
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.setValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(3000, bitDepth, 2818); err != nil {
} else if _, err := f.setValue(3000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(4000, bitDepth, 300); err != nil {
} else if _, err := f.setValue(4000, bitDepth, 300); err != nil {
t.Fatal(err)
}
// Query for equality.
if b, err := f.RangeOp(pql.EQ, bitDepth, 300); err != nil {
if b, err := f.rangeOp(pql.EQ, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
@ -359,18 +359,18 @@ func TestFragment_Range(t *testing.T) {
defer f.Close()
// Set values.
if _, err := f.SetValue(1000, bitDepth, 382); err != nil {
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.setValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(3000, bitDepth, 2818); err != nil {
} else if _, err := f.setValue(3000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(4000, bitDepth, 300); err != nil {
} else if _, err := f.setValue(4000, bitDepth, 300); err != nil {
t.Fatal(err)
}
// Query for inequality.
if b, err := f.RangeOp(pql.NEQ, bitDepth, 300); err != nil {
if b, err := f.rangeOp(pql.NEQ, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
@ -382,43 +382,43 @@ func TestFragment_Range(t *testing.T) {
defer f.Close()
// Set values.
if _, err := f.SetValue(1000, bitDepth, 382); err != nil {
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.setValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(3000, bitDepth, 2817); err != nil {
} else if _, err := f.setValue(3000, bitDepth, 2817); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(4000, bitDepth, 301); err != nil {
} else if _, err := f.setValue(4000, bitDepth, 301); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(5000, bitDepth, 1); err != nil {
} else if _, err := f.setValue(5000, bitDepth, 1); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(6000, bitDepth, 0); err != nil {
} else if _, err := f.setValue(6000, bitDepth, 0); err != nil {
t.Fatal(err)
}
// Query for values less than (ending with set column).
if b, err := f.RangeOp(pql.LT, bitDepth, 301); err != nil {
if b, err := f.rangeOp(pql.LT, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for values less than (ending with unset column).
if b, err := f.RangeOp(pql.LT, bitDepth, 300); err != nil {
if b, err := f.rangeOp(pql.LT, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for values less than or equal to (ending with set column).
if b, err := f.RangeOp(pql.LTE, bitDepth, 301); err != nil {
if b, err := f.rangeOp(pql.LTE, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for values less than or equal to (ending with unset column).
if b, err := f.RangeOp(pql.LTE, bitDepth, 300); err != nil {
if b, err := f.rangeOp(pql.LTE, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
@ -430,43 +430,43 @@ func TestFragment_Range(t *testing.T) {
defer f.Close()
// Set values.
if _, err := f.SetValue(1000, bitDepth, 382); err != nil {
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.setValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(3000, bitDepth, 2817); err != nil {
} else if _, err := f.setValue(3000, bitDepth, 2817); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(4000, bitDepth, 301); err != nil {
} else if _, err := f.setValue(4000, bitDepth, 301); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(5000, bitDepth, 1); err != nil {
} else if _, err := f.setValue(5000, bitDepth, 1); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(6000, bitDepth, 0); err != nil {
} else if _, err := f.setValue(6000, bitDepth, 0); err != nil {
t.Fatal(err)
}
// Query for values greater than (ending with unset bit).
if b, err := f.RangeOp(pql.GT, bitDepth, 300); err != nil {
if b, err := f.rangeOp(pql.GT, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for values greater than (ending with set bit).
if b, err := f.RangeOp(pql.GT, bitDepth, 301); err != nil {
if b, err := f.rangeOp(pql.GT, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for values greater than or equal to (ending with unset bit).
if b, err := f.RangeOp(pql.GTE, bitDepth, 300); err != nil {
if b, err := f.rangeOp(pql.GTE, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for values greater than or equal to (ending with set bit).
if b, err := f.RangeOp(pql.GTE, bitDepth, 301); err != nil {
if b, err := f.rangeOp(pql.GTE, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
@ -478,43 +478,43 @@ func TestFragment_Range(t *testing.T) {
defer f.Close()
// Set values.
if _, err := f.SetValue(1000, bitDepth, 382); err != nil {
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.setValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(3000, bitDepth, 2817); err != nil {
} else if _, err := f.setValue(3000, bitDepth, 2817); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(4000, bitDepth, 301); err != nil {
} else if _, err := f.setValue(4000, bitDepth, 301); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(5000, bitDepth, 1); err != nil {
} else if _, err := f.setValue(5000, bitDepth, 1); err != nil {
t.Fatal(err)
} else if _, err := f.SetValue(6000, bitDepth, 0); err != nil {
} else if _, err := f.setValue(6000, bitDepth, 0); err != nil {
t.Fatal(err)
}
// Query for values greater than (ending with unset column).
if b, err := f.RangeBetween(bitDepth, 300, 2817); err != nil {
if b, err := f.rangeBetween(bitDepth, 300, 2817); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for values greater than (ending with set column).
if b, err := f.RangeBetween(bitDepth, 301, 2817); err != nil {
if b, err := f.rangeBetween(bitDepth, 301, 2817); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for values greater than or equal to (ending with unset column).
if b, err := f.RangeBetween(bitDepth, 301, 2816); err != nil {
if b, err := f.rangeBetween(bitDepth, 301, 2816); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for values greater than or equal to (ending with set column).
if b, err := f.RangeBetween(bitDepth, 300, 2816); err != nil {
if b, err := f.rangeBetween(bitDepth, 300, 2816); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
@ -528,25 +528,25 @@ func TestFragment_Snapshot(t *testing.T) {
defer f.Close()
// Set and then clear bits on the fragment.
if _, err := f.SetBit(1000, 1); err != nil {
if _, err := f.setBit(1000, 1); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(1000, 2); err != nil {
} else if _, err := f.setBit(1000, 2); err != nil {
t.Fatal(err)
} else if _, err := f.ClearBit(1000, 1); err != nil {
} else if _, err := f.clearBit(1000, 1); err != nil {
t.Fatal(err)
}
// Snapshot bitmap and verify data.
if err := f.Snapshot(); err != nil {
t.Fatal(err)
} else if n := f.Row(1000).Count(); n != 1 {
} else if n := f.row(1000).Count(); n != 1 {
t.Fatalf("unexpected count: %d", n)
}
// Close and reopen the fragment & verify the data.
if err := f.reopen(); err != nil {
t.Fatal(err)
} else if n := f.Row(1000).Count(); n != 1 {
} else if n := f.row(1000).Count(); n != 1 {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
@ -557,17 +557,17 @@ func TestFragment_ForEachBit(t *testing.T) {
defer f.Close()
// Set bits on the fragment.
if _, err := f.SetBit(100, 20); err != nil {
if _, err := f.setBit(100, 20); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(2, 38); err != nil {
} else if _, err := f.setBit(2, 38); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(2, 37); err != nil {
} else if _, err := f.setBit(2, 37); err != nil {
t.Fatal(err)
}
// Iterate over bits.
var result [][2]uint64
if err := f.ForEachBit(func(rowID, columnID uint64) error {
if err := f.forEachBit(func(rowID, columnID uint64) error {
result = append(result, [2]uint64{rowID, columnID})
return nil
}); err != nil {
@ -591,7 +591,7 @@ func TestFragment_Top(t *testing.T) {
f.RecalculateCache()
// Retrieve top rows.
if pairs, err := f.Top(TopOptions{N: 2}); err != nil {
if pairs, err := f.top(TopOptions{N: 2}); err != nil {
t.Fatal(err)
} else if len(pairs) != 2 {
t.Fatalf("unexpected count: %d", len(pairs))
@ -617,7 +617,7 @@ func TestFragment_Top_Filter(t *testing.T) {
f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": int64(20)})
// Retrieve top rows.
if pairs, err := f.Top(TopOptions{
if pairs, err := f.top(TopOptions{
N: 2,
FilterName: "x",
FilterValues: []interface{}{int64(10), int64(15), int64(20)},
@ -648,7 +648,7 @@ func TestFragment_TopN_Intersect(t *testing.T) {
f.RecalculateCache()
// Retrieve top rows.
if pairs, err := f.Top(TopOptions{N: 3, Src: src}); err != nil {
if pairs, err := f.top(TopOptions{N: 3, Src: src}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(pairs, []Pair{
{ID: 101, Count: 3},
@ -683,7 +683,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
f.RecalculateCache()
// Retrieve top rows.
if pairs, err := f.Top(TopOptions{N: 10, Src: src}); err != nil {
if pairs, err := f.top(TopOptions{N: 10, Src: src}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(pairs, []Pair{
{ID: 999, Count: 19},
@ -712,7 +712,7 @@ func TestFragment_TopN_IDs(t *testing.T) {
f.mustSetBits(102, 8, 9, 10, 11, 12)
// Retrieve top rows.
if pairs, err := f.Top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil {
if pairs, err := f.top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(pairs, []Pair{
{ID: 101, Count: 4},
@ -733,7 +733,7 @@ func TestFragment_TopN_NopCache(t *testing.T) {
f.mustSetBits(102, 8, 9, 10, 11, 12)
// Retrieve top rows.
if pairs, err := f.Top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil {
if pairs, err := f.top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(pairs, []Pair{}) {
t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs))
@ -792,7 +792,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
}
// Retrieve top rows.
if pairs, err := f.Top(TopOptions{N: 5}); err != nil {
if pairs, err := f.top(TopOptions{N: 5}); err != nil {
t.Fatal(err)
} else if len(pairs) > int(cacheSize) {
t.Fatalf("TopN count cannot exceed cache size: %d", cacheSize)
@ -810,9 +810,9 @@ func TestFragment_Checksum(t *testing.T) {
// Retrieve checksum and set bits.
orig := f.Checksum()
if _, err := f.SetBit(1, 200); err != nil {
if _, err := f.setBit(1, 200); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(HashBlockSize*2, 200); err != nil {
} else if _, err := f.setBit(HashBlockSize*2, 200); err != nil {
t.Fatal(err)
}
@ -831,7 +831,7 @@ func TestFragment_Blocks(t *testing.T) {
var prev []FragmentBlock
// Set first bit.
if _, err := f.SetBit(0, 0); err != nil {
if _, err := f.setBit(0, 0); err != nil {
t.Fatal(err)
}
blocks := f.Blocks()
@ -841,7 +841,7 @@ func TestFragment_Blocks(t *testing.T) {
prev = blocks
// Set bit on different row.
if _, err := f.SetBit(20, 0); err != nil {
if _, err := f.setBit(20, 0); err != nil {
t.Fatal(err)
}
blocks = f.Blocks()
@ -851,7 +851,7 @@ func TestFragment_Blocks(t *testing.T) {
prev = blocks
// Set bit on different column.
if _, err := f.SetBit(20, 100); err != nil {
if _, err := f.setBit(20, 100); err != nil {
t.Fatal(err)
}
blocks = f.Blocks()
@ -866,7 +866,7 @@ func TestFragment_Blocks_Empty(t *testing.T) {
defer f.Close()
// Set bits on a different block.
if _, err := f.SetBit(100, 1); err != nil {
if _, err := f.setBit(100, 1); err != nil {
t.Fatal(err)
}
@ -885,14 +885,14 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
// Set bits on the fragment.
for i := uint64(0); i < 1000; i++ {
if _, err := f.SetBit(i, 0); err != nil {
if _, err := f.setBit(i, 0); err != nil {
t.Fatal(err)
}
}
// Verify correct cache type and size.
if cache, ok := f.Cache().(*LRUCache); !ok {
t.Fatalf("unexpected cache: %T", f.Cache())
if cache, ok := f.cache.(*LRUCache); !ok {
t.Fatalf("unexpected cache: %T", f.cache)
} else if cache.Len() != 1000 {
t.Fatalf("unexpected cache len: %d", cache.Len())
}
@ -903,8 +903,8 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
}
// Re-verify correct cache type and size.
if cache, ok := f.Cache().(*LRUCache); !ok {
t.Fatalf("unexpected cache: %T", f.Cache())
if cache, ok := f.cache.(*LRUCache); !ok {
t.Fatalf("unexpected cache: %T", f.cache)
} else if cache.Len() != 1000 {
t.Fatalf("unexpected cache len: %d", cache.Len())
}
@ -935,14 +935,14 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
// Set bits on the fragment.
for i := uint64(0); i < 1000; i++ {
if _, err := f.SetBit(i, 0); err != nil {
if _, err := f.setBit(i, 0); err != nil {
t.Fatal(err)
}
}
// Verify correct cache type and size.
if cache, ok := f.Cache().(*RankCache); !ok {
t.Fatalf("unexpected cache: %T", f.Cache())
if cache, ok := f.cache.(*RankCache); !ok {
t.Fatalf("unexpected cache: %T", f.cache)
} else if cache.Len() != 1000 {
t.Fatalf("unexpected cache len: %d", cache.Len())
}
@ -956,8 +956,8 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
f = index.Field("f").View(ViewStandard).Fragment(0)
// Re-verify correct cache type and size.
if cache, ok := f.Cache().(*RankCache); !ok {
t.Fatalf("unexpected cache: %T", f.Cache())
if cache, ok := f.cache.(*RankCache); !ok {
t.Fatalf("unexpected cache: %T", f.cache)
} else if cache.Len() != 1000 {
t.Fatalf("unexpected cache len: %d", cache.Len())
}
@ -969,16 +969,16 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
defer f0.Close()
// Set and then clear bits on the fragment.
if _, err := f0.SetBit(1000, 1); err != nil {
if _, err := f0.setBit(1000, 1); err != nil {
t.Fatal(err)
} else if _, err := f0.SetBit(1000, 2); err != nil {
} else if _, err := f0.setBit(1000, 2); err != nil {
t.Fatal(err)
} else if _, err := f0.ClearBit(1000, 1); err != nil {
} else if _, err := f0.clearBit(1000, 1); err != nil {
t.Fatal(err)
}
// Verify cache is populated.
if n := f0.Cache().Len(); n != 1 {
if n := f0.cache.Len(); n != 1 {
t.Fatalf("unexpected cache size: %d", n)
}
@ -998,21 +998,21 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
}
// Verify cache is in other fragment.
if n := f1.Cache().Len(); n != 1 {
if n := f1.cache.Len(); n != 1 {
t.Fatalf("unexpected cache size: %d", n)
}
// Verify data in other fragment.
if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) {
if a := f1.row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected columns: %+v", a)
}
// Close and reopen the fragment & verify the data.
if err := f1.reopen(); err != nil {
t.Fatal(err)
} else if n := f1.Cache().Len(); n != 1 {
} else if n := f1.cache.Len(); n != 1 {
t.Fatalf("unexpected cache size (reopen): %d", n)
} else if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) {
} else if a := f1.row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected columns (reopen): %+v", a)
}
}
@ -1045,12 +1045,12 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
// Generate some intersecting data.
for i := 0; i < 10000; i += 2 {
if _, err := f.SetBit(1, uint64(i)); err != nil {
if _, err := f.setBit(1, uint64(i)); err != nil {
b.Fatal(err)
}
}
for i := 0; i < 10000; i += 3 {
if _, err := f.SetBit(2, uint64(i)); err != nil {
if _, err := f.setBit(2, uint64(i)); err != nil {
b.Fatal(err)
}
}
@ -1063,7 +1063,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
// Start benchmark
b.ResetTimer()
for i := 0; i < b.N; i++ {
if n := f.Row(1).IntersectionCount(f.Row(2)); n == 0 {
if n := f.row(1).IntersectionCount(f.row(2)); n == 0 {
b.Fatalf("unexpected count: %d", n)
}
}
@ -1081,7 +1081,7 @@ func TestFragment_Tanimoto(t *testing.T) {
f.mustSetBits(102, 1, 2, 10, 12)
f.RecalculateCache()
if pairs, err := f.Top(TopOptions{TanimotoThreshold: 50, Src: src}); err != nil {
if pairs, err := f.top(TopOptions{TanimotoThreshold: 50, Src: src}); err != nil {
t.Fatal(err)
} else if len(pairs) != 2 {
t.Fatalf("unexpected count: %d", len(pairs))
@ -1104,7 +1104,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) {
f.mustSetBits(102, 1, 2, 10, 12)
f.RecalculateCache()
if pairs, err := f.Top(TopOptions{TanimotoThreshold: 0, Src: src}); err != nil {
if pairs, err := f.top(TopOptions{TanimotoThreshold: 0, Src: src}); err != nil {
t.Fatal(err)
} else if len(pairs) != 3 {
t.Fatalf("unexpected count: %d", len(pairs))
@ -1123,7 +1123,7 @@ func TestFragment_Snapshot_Run(t *testing.T) {
// Set bits on the fragment.
for i := uint64(1); i < 3; i++ {
if _, err := f.SetBit(1000, i); err != nil {
if _, err := f.setBit(1000, i); err != nil {
t.Fatal(err)
}
}
@ -1131,14 +1131,14 @@ func TestFragment_Snapshot_Run(t *testing.T) {
// Snapshot bitmap and verify data.
if err := f.Snapshot(); err != nil {
t.Fatal(err)
} else if n := f.Row(1000).Count(); n != 2 {
} else if n := f.row(1000).Count(); n != 2 {
t.Fatalf("unexpected count: %d", n)
}
// Close and reopen the fragment & verify the data.
if err := f.reopen(); err != nil {
t.Fatal(err)
} else if n := f.Row(1000).Count(); n != 2 {
} else if n := f.row(1000).Count(); n != 2 {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
@ -1187,7 +1187,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
val += 2
i++
}
if err := f.Import(rows, cols); err != nil {
if err := f.bulkImport(rows, cols); err != nil {
b.Fatalf("Error Building Sample: %s", err)
}
if row > max {
@ -1228,7 +1228,7 @@ func BenchmarkFragment_Import(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if err := f.Import(rows, cols); err != nil {
if err := f.bulkImport(rows, cols); err != nil {
b.Fatalf("Error Building Sample: %s", err)
}
}
@ -1273,7 +1273,7 @@ func (f *Fragment) reopen() error {
// This function does not accept a timestamp or quantum.
func (f *Fragment) mustSetBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.SetBit(rowID, columnID); err != nil {
if _, err := f.setBit(rowID, columnID); err != nil {
panic(err)
}
}

14
holder.go
View file

@ -217,7 +217,7 @@ func (h *Holder) Schema() []*IndexInfo {
for _, field := range index.Fields() {
fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
for _, view := range field.Views() {
fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()})
fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
}
sort.Sort(viewInfoSlice(fi.Views))
di.Fields = append(di.Fields, fi)
@ -437,7 +437,7 @@ func (h *Holder) flushCaches() {
for _, index := range h.Indexes() {
for _, field := range index.Fields() {
for _, view := range field.Views() {
for _, fragment := range view.Fragments() {
for _, fragment := range view.allFragments() {
select {
case <-h.closing:
return
@ -445,7 +445,7 @@ func (h *Holder) flushCaches() {
}
if err := fragment.FlushCache(); err != nil {
h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath())
h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.cachePath())
}
}
}
@ -761,7 +761,7 @@ func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) err
Closing: s.Closing,
RemoteClient: s.RemoteClient,
}
if err := fs.SyncFragment(); err != nil {
if err := fs.syncFragment(); err != nil {
return errors.Wrap(err, "syncing fragment")
}
@ -804,14 +804,14 @@ func (c *HolderCleaner) CleanHolder() error {
// Get the fragments registered in memory.
for _, field := range index.Fields() {
for _, view := range field.Views() {
for _, fragment := range view.Fragments() {
fragSlice := fragment.Slice()
for _, fragment := range view.allFragments() {
fragSlice := fragment.slice
// Ignore fragments that should be present.
if uint64InSlice(fragSlice, containedSlices) {
continue
}
// Delete fragment.
if err := view.DeleteFragment(fragSlice); err != nil {
if err := view.deleteFragment(fragSlice); err != nil {
return errors.Wrap(err, "deleting fragment")
}
}

160
holder_test.go
View file

@ -210,9 +210,7 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if _, err := view.SetBit(0, 0); err != nil {
} else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -233,9 +231,7 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if _, err := view.SetBit(0, 0); err != nil {
} else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -261,7 +257,7 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if _, err := view.SetBit(0, 0); err != nil {
} else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
} else if err := view.Fragment(0).FlushCache(); err != nil {
t.Fatal(err)
@ -331,14 +327,8 @@ func TestHolder_DeleteIndex(t *testing.T) {
defer hldr.Close()
// Write bits to separate indexes.
f0 := hldr.MustCreateFragmentIfNotExists("i0", "f", pilosa.ViewStandard, 0)
if _, err := f0.SetBit(100, 200); err != nil {
t.Fatal(err)
}
f1 := hldr.MustCreateFragmentIfNotExists("i1", "f", pilosa.ViewStandard, 0)
if _, err := f1.SetBit(100, 200); err != nil {
t.Fatal(err)
}
hldr.SetBit("i0", "f", 100, 200)
hldr.SetBit("i1", "f", 100, 200)
// Ensure i0 exists.
if _, err := os.Stat(hldr.IndexPath("i0")); err != nil {
@ -404,42 +394,24 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
}
// Set data on the local holder.
f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if _, err := f.SetBit(0, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(2, 20); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(120, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(200, 4); err != nil {
t.Fatal(err)
}
hldr0.SetBit("i", "f", 0, 10)
hldr0.SetBit("i", "f", 2, 20)
hldr0.SetBit("i", "f", 120, 10)
hldr0.SetBit("i", "f", 200, 4)
f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1)
if _, err := f.SetBit(9, SliceWidth+5); err != nil {
t.Fatal(err)
}
hldr0.SetBit("i", "f0", 9, SliceWidth+5)
hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0)
// Set a bit to create the fragment.
hldr0.SetBit("y", "z", 0, 0)
// Set data on the remote holder.
f = hldr1.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if _, err := f.SetBit(0, 4000); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(3, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(120, 10); err != nil {
t.Fatal(err)
}
hldr1.SetBit("i", "f", 0, 4000)
hldr1.SetBit("i", "f", 3, 10)
hldr1.SetBit("i", "f", 120, 10)
f = hldr1.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 3)
if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (3*SliceWidth)+7); err != nil {
t.Fatal(err)
}
hldr1.SetBit("y", "z", 10, (3*SliceWidth)+4)
hldr1.SetBit("y", "z", 10, (3*SliceWidth)+5)
hldr1.SetBit("y", "z", 10, (3*SliceWidth)+7)
// Set highest slice.
hldr0.Index("i").SetRemoteMaxSlice(1)
@ -460,29 +432,23 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Columns()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a)
}
if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
@ -509,34 +475,18 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
}
// Set data on the local holder.
f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if _, err := f.SetBit(0, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 4000); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(2, 20); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(3, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(120, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(200, 4); err != nil {
t.Fatal(err)
}
hldr0.SetBit("i", "f", 0, 10)
hldr0.SetBit("i", "f", 0, 4000)
hldr0.SetBit("i", "f", 2, 20)
hldr0.SetBit("i", "f", 3, 10)
hldr0.SetBit("i", "f", 120, 10)
hldr0.SetBit("i", "f", 200, 4)
f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1)
if _, err := f.SetBit(9, SliceWidth+5); err != nil {
t.Fatal(err)
}
hldr0.SetBit("i", "f0", 9, SliceWidth+5)
f = hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 2)
if _, err := f.SetBit(10, (2*SliceWidth)+4); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (2*SliceWidth)+5); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (2*SliceWidth)+7); err != nil {
t.Fatal(err)
}
hldr0.SetBit("y", "z", 10, (2*SliceWidth)+4)
hldr0.SetBit("y", "z", 10, (2*SliceWidth)+5)
hldr0.SetBit("y", "z", 10, (2*SliceWidth)+7)
// Set highest slice.
hldr0.Index("i").SetRemoteMaxSlice(1)
@ -558,29 +508,23 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Columns()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a)
}
if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
@ -601,26 +545,24 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
f := hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
if f != nil {
t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}

7
http/client.go
View file

@ -1010,13 +1010,18 @@ type BitsByPos []pilosa.Bit
func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p BitsByPos) Len() int { return len(p) }
func (p BitsByPos) Less(i, j int) bool {
p0, p1 := pilosa.Pos(p[i].RowID, p[i].ColumnID), pilosa.Pos(p[j].RowID, p[j].ColumnID)
p0, p1 := pos(p[i].RowID, p[i].ColumnID), pos(p[j].RowID, p[j].ColumnID)
if p0 == p1 {
return p[i].Timestamp < p[j].Timestamp
}
return p0 < p1
}
// pos returns the row position of a row/column pair.
func pos(rowID, columnID uint64) uint64 {
return (rowID * pilosa.SliceWidth) + (columnID % pilosa.SliceWidth)
}
func uriPathToURL(uri *pilosa.URI, path string) url.URL {
return url.URL{
Scheme: uri.Scheme(),

50
http/client_test.go
View file

@ -112,26 +112,26 @@ func TestClient_MultiNode(t *testing.T) {
}
}
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(100, baseBit0+10)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(22, baseBit0+1, baseBit0+2, baseBit0+10)
hldr[0].MustSetBits("i", "f", 100, baseBit0+10)
hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12)
hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15)
hldr[0].MustSetBits("i", "f", 2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4)
hldr[0].MustSetBits("i", "f", 3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5)
hldr[0].MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2, baseBit0+10)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(1, baseBit1+4)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5)
hldr[1].MustSetBits("i", "f", 99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4)
hldr[1].MustSetBits("i", "f", 100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10)
hldr[1].MustSetBits("i", "f", 98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6)
hldr[1].MustSetBits("i", "f", 1, baseBit1+4)
hldr[1].MustSetBits("i", "f", 22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(21, baseBit2+10)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(100, baseBit2+10)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(99, baseBit2+10, baseBit2+11, baseBit2+12)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(98, baseBit2+10, baseBit2+11)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(22, baseBit2+10, baseBit2+11, baseBit2+12)
hldr[2].MustSetBits("i", "f", 24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14)
hldr[2].MustSetBits("i", "f", 20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13)
hldr[2].MustSetBits("i", "f", 21, baseBit2+10)
hldr[2].MustSetBits("i", "f", 100, baseBit2+10)
hldr[2].MustSetBits("i", "f", 99, baseBit2+10, baseBit2+11, baseBit2+12)
hldr[2].MustSetBits("i", "f", 98, baseBit2+10, baseBit2+11)
hldr[2].MustSetBits("i", "f", 22, baseBit2+10, baseBit2+11, baseBit2+12)
// Rebuild the RankCache.
// We have to do this to avoid the 10-second cache invalidation delay
@ -214,8 +214,8 @@ func TestClient_Import(t *testing.T) {
defer hldr.Close()
// Load bitmap into cache to ensure cache gets updated.
f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
f.Row(0)
hldr.SetBit("i", "f", 1, 0) // set a bit so the view gets created.
hldr.Row("i", "f", 0)
s := test.NewServer()
defer s.Close()
@ -234,10 +234,10 @@ func TestClient_Import(t *testing.T) {
}
// Verify data.
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) {
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) {
t.Fatalf("unexpected columns: %+v", a)
}
if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{6}) {
if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{6}) {
t.Fatalf("unexpected columns: %+v", a)
}
}
@ -325,11 +325,11 @@ func TestClient_FragmentBlocks(t *testing.T) {
defer hldr.Close()
// Set two bits on blocks 0 & 3.
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100)
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100)
// Set a bit on a different slice.
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1)
hldr.SetBit("i", "f", 0, 1)
s := test.NewServer()
defer s.Close()

12
http/handler_test.go
View file

@ -189,13 +189,13 @@ func TestHandler_MaxSlices(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*pilosa.SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*pilosa.SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*pilosa.SliceWidth)+4)
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1)
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2)
hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*pilosa.SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*pilosa.SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*pilosa.SliceWidth)+8)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8)
h := test.MustNewHandler()
h.API.Holder = hldr.Holder

30
stats_test.go
View file

@ -36,11 +36,11 @@ func TestMultiStatClient_Expvar(t *testing.T) {
ms[0] = c
hldr.Stats = ms
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).ClearBit(0, 1)
hldr.SetBit("d", "f", 0, 0)
hldr.SetBit("d", "f", 0, 1)
hldr.SetBit("d", "f", 0, SliceWidth)
hldr.SetBit("d", "f", 0, SliceWidth+2)
hldr.ClearBit("d", "f", 0, 1)
if pilosa.Expvar.String() != `{"index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
@ -88,10 +88,10 @@ func TestStatsCount_TopN(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
hldr.SetBit("d", "f", 0, 0)
hldr.SetBit("d", "f", 0, 1)
hldr.SetBit("d", "f", 0, SliceWidth)
hldr.SetBit("d", "f", 0, SliceWidth+2)
// Execute query.
called := false
@ -121,8 +121,8 @@ func TestStatsCount_Bitmap(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.SetBit("d", "f", 0, 0)
hldr.SetBit("d", "f", 0, 1)
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e.Holder.Stats = &MockStats{
@ -150,8 +150,8 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1)
hldr.SetBit("d", "f", 10, 0)
hldr.SetBit("d", "f", 10, 1)
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
@ -180,8 +180,8 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1)
hldr.SetBit("d", "f", 10, 0)
hldr.SetBit("d", "f", 10, 1)
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))

10
test/fragment.go
View file

@ -26,13 +26,3 @@ type Fragment struct {
*pilosa.Fragment
RowAttrStore pilosa.AttrStore
}
// MustSetBits sets columns on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.SetBit(rowID, columnID); err != nil {
panic(err)
}
}
}

74
test/holder.go
View file

@ -89,24 +89,6 @@ func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field {
return f
}
// MustCreateFragmentIfNotExists returns a given fragment. Panic on error.
func (h *Holder) MustCreateFragmentIfNotExists(index, field, view string, slice uint64) *Fragment {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
v, err := f.CreateViewIfNotExists(view)
if err != nil {
panic(err)
}
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
panic(err)
}
return &Fragment{Fragment: frag}
}
// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error.
func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, slice uint64) *Fragment {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
@ -124,3 +106,59 @@ func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string,
}
return &Fragment{Fragment: frag}
}
// Row returns a Row for a given field.
func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
row, err := f.Row(rowID)
if err != nil {
panic(err)
}
return row
}
// ViewRow returns a Row for a given field and view.
func (h *Holder) ViewRow(index, field, view string, rowID uint64) *pilosa.Row {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
row, err := f.ViewRow(view, rowID)
if err != nil {
panic(err)
}
return row
}
// SetBit clears a bit on the given field.
func (h *Holder) SetBit(index, field string, rowID, columnID uint64) {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
f.SetBit(pilosa.ViewStandard, rowID, columnID, nil)
}
// ClearBit clears a bit on the given field.
func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
f.ClearBit(pilosa.ViewStandard, rowID, columnID, nil)
}
// MustSetBits sets columns on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (h *Holder) MustSetBits(index, field string, rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
h.SetBit(index, field, rowID, columnID)
}
}

114
view.go
View file

@ -82,20 +82,8 @@ func NewView(path, index, field, name string, cacheSize uint32) *View {
}
}
// Name returns the name the view was initialized with.
func (v *View) Name() string { return v.name }
// Index returns the index name the view was initialized with.
func (v *View) Index() string { return v.index }
// Field returns the field name the view was initialized with.
func (v *View) Field() string { return v.field }
// Path returns the path the view was initialized with.
func (v *View) Path() string { return v.path }
// Open opens and initializes the view.
func (v *View) Open() error {
// open opens and initializes the view.
func (v *View) open() error {
// Never keep a cache for field views.
if strings.HasPrefix(v.name, viewBSIGroupPrefix) {
@ -116,7 +104,7 @@ func (v *View) Open() error {
return nil
}(); err != nil {
v.Close()
v.close()
return err
}
@ -149,19 +137,19 @@ func (v *View) openFragments() error {
continue
}
frag := v.newFragment(v.FragmentPath(slice), slice)
frag := v.newFragment(v.fragmentPath(slice), slice)
if err := frag.Open(); err != nil {
return fmt.Errorf("open fragment: slice=%d, err=%s", frag.Slice(), err)
return fmt.Errorf("open fragment: slice=%d, err=%s", frag.slice, err)
}
frag.RowAttrStore = v.RowAttrStore
v.fragments[frag.Slice()] = frag
v.fragments[frag.slice] = frag
}
return nil
}
// Close closes the view and its fragments.
func (v *View) Close() error {
// close closes the view and its fragments.
func (v *View) close() error {
v.mu.Lock()
defer v.mu.Unlock()
@ -176,8 +164,8 @@ func (v *View) Close() error {
return nil
}
// MaxSlice returns the max slice in the view.
func (v *View) MaxSlice() uint64 {
// calculateMaxSlice returns the max slice in the view.
func (v *View) calculateMaxSlice() uint64 {
v.mu.RLock()
defer v.mu.RUnlock()
@ -191,8 +179,8 @@ func (v *View) MaxSlice() uint64 {
return max
}
// FragmentPath returns the path to a fragment in the view.
func (v *View) FragmentPath(slice uint64) string {
// fragmentPath returns the path to a fragment in the view.
func (v *View) fragmentPath(slice uint64) string {
return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10))
}
@ -205,8 +193,8 @@ func (v *View) Fragment(slice uint64) *Fragment {
func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] }
// Fragments returns a list of all fragments in the view.
func (v *View) Fragments() []*Fragment {
// allFragments returns a list of all fragments in the view.
func (v *View) allFragments() []*Fragment {
v.mu.Lock()
defer v.mu.Unlock()
@ -217,9 +205,9 @@ func (v *View) Fragments() []*Fragment {
return other
}
// RecalculateCaches recalculates the cache on every fragment in the view.
func (v *View) RecalculateCaches() {
for _, fragment := range v.Fragments() {
// recalculateCaches recalculates the cache on every fragment in the view.
func (v *View) recalculateCaches() {
for _, fragment := range v.allFragments() {
fragment.RecalculateCache()
}
}
@ -238,7 +226,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
}
// Initialize and open fragment.
frag := v.newFragment(v.FragmentPath(slice), slice)
frag := v.newFragment(v.fragmentPath(slice), slice)
if err := frag.Open(); err != nil {
return nil, errors.Wrap(err, "opening fragment")
}
@ -273,8 +261,8 @@ func (v *View) newFragment(path string, slice uint64) *Fragment {
return frag
}
// DeleteFragment removes the fragment from the view.
func (v *View) DeleteFragment(slice uint64) error {
// deleteFragment removes the fragment from the view.
func (v *View) deleteFragment(slice uint64) error {
fragment := v.fragments[slice]
if fragment == nil {
@ -289,12 +277,12 @@ func (v *View) DeleteFragment(slice uint64) error {
}
// Delete fragment file.
if err := os.Remove(fragment.Path()); err != nil {
if err := os.Remove(fragment.path); err != nil {
return errors.Wrap(err, "deleting fragment file")
}
// Delete fragment cache file.
if err := os.Remove(fragment.CachePath()); err != nil {
if err := os.Remove(fragment.cachePath()); err != nil {
v.Logger.Printf("no cache file to delete for slice %d", slice)
}
@ -303,24 +291,38 @@ func (v *View) DeleteFragment(slice uint64) error {
return nil
}
// SetBit sets a bit within the view.
func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return changed, err
// row returns a row for a slice of the view.
func (v *View) row(rowID uint64) *Row {
row := NewRow()
for _, frag := range v.allFragments() {
fr := frag.row(rowID)
if fr == nil {
continue
}
row.Merge(fr)
}
return frag.SetBit(rowID, columnID)
return row
}
// ClearBit clears a bit within the view.
func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) {
// setBit sets a bit within the view.
func (v *View) setBit(rowID, columnID uint64) (changed bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return changed, err
}
return frag.ClearBit(rowID, columnID)
return frag.setBit(rowID, columnID)
}
// clearBit clears a bit within the view.
func (v *View) clearBit(rowID, columnID uint64) (changed bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return changed, err
}
return frag.clearBit(rowID, columnID)
}
// value uses a column of bits to read a multi-bit value.
@ -330,7 +332,7 @@ func (v *View) value(columnID uint64, bitDepth uint) (value uint64, exists bool,
if err != nil {
return value, exists, err
}
return frag.Value(columnID, bitDepth)
return frag.value(columnID, bitDepth)
}
// setValue uses a column of bits to set a multi-bit value.
@ -340,13 +342,13 @@ func (v *View) setValue(columnID uint64, bitDepth uint, value uint64) (changed b
if err != nil {
return changed, err
}
return frag.SetValue(columnID, bitDepth, value)
return frag.setValue(columnID, bitDepth, value)
}
// sum returns the sum & count of a field.
func (v *View) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
for _, f := range v.Fragments() {
fsum, fcount, err := f.Sum(filter, bitDepth)
for _, f := range v.allFragments() {
fsum, fcount, err := f.sum(filter, bitDepth)
if err != nil {
return sum, count, err
}
@ -359,8 +361,8 @@ func (v *View) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
// min returns the min and count of a field.
func (v *View) min(filter *Row, bitDepth uint) (min, count uint64, err error) {
var minHasValue bool
for _, f := range v.Fragments() {
fmin, fcount, err := f.Min(filter, bitDepth)
for _, f := range v.allFragments() {
fmin, fcount, err := f.min(filter, bitDepth)
if err != nil {
return min, count, err
}
@ -386,8 +388,8 @@ func (v *View) min(filter *Row, bitDepth uint) (min, count uint64, err error) {
// max returns the max and count of a field.
func (v *View) max(filter *Row, bitDepth uint) (max, count uint64, err error) {
for _, f := range v.Fragments() {
fmax, fcount, err := f.Max(filter, bitDepth)
for _, f := range v.allFragments() {
fmax, fcount, err := f.max(filter, bitDepth)
if err != nil {
return max, count, err
}
@ -402,8 +404,8 @@ func (v *View) max(filter *Row, bitDepth uint) (max, count uint64, err error) {
// rangeOp returns rows with a field value encoding matching the predicate.
func (v *View) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) {
r := NewRow()
for _, frag := range v.Fragments() {
other, err := frag.RangeOp(op, bitDepth, predicate)
for _, frag := range v.allFragments() {
other, err := frag.rangeOp(op, bitDepth, predicate)
if err != nil {
return nil, err
}
@ -416,8 +418,8 @@ func (v *View) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, err
// value between predicateMin and predicateMax.
func (v *View) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
r := NewRow()
for _, frag := range v.Fragments() {
other, err := frag.RangeBetween(bitDepth, predicateMin, predicateMax)
for _, frag := range v.allFragments() {
other, err := frag.rangeBetween(bitDepth, predicateMin, predicateMax)
if err != nil {
return nil, err
}

68
view_internal_test.go Normal file
View file

@ -0,0 +1,68 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"io/ioutil"
"testing"
)
// mustOpenView returns a new instance of View with a temporary path.
func mustOpenView(index, field, name string) *View {
path, err := ioutil.TempDir("", "pilosa-view-")
if err != nil {
panic(err)
}
v := NewView(path, index, field, name, DefaultCacheSize)
if err := v.open(); err != nil {
panic(err)
}
v.RowAttrStore = newMemAttrStore()
return v
}
// Ensure view can open and retrieve a fragment.
func TestView_DeleteFragment(t *testing.T) {
v := mustOpenView("i", "f", "v")
defer v.close()
slice := uint64(9)
// Create fragment.
fragment, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
t.Fatal(err)
} else if fragment == nil {
t.Fatal("expected fragment")
}
err = v.deleteFragment(slice)
if err != nil {
t.Fatal(err)
}
if v.Fragment(slice) != nil {
t.Fatal("fragment still exists in view")
}
// Recreate fragment with same slice, verify that the old fragment was not reused.
fragment2, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
t.Fatal(err)
} else if fragment == fragment2 {
t.Fatal("failed to create new fragment")
}
}

125
view_test.go
View file

@ -1,125 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
"io/ioutil"
"os"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
// View is a test wrapper for pilosa.View.
type View struct {
*pilosa.View
RowAttrStore pilosa.AttrStore
}
// NewView returns a new instance of View with a temporary path.
func NewView(index, field, name string) *View {
path, err := ioutil.TempDir("", "pilosa-view-")
if err != nil {
panic(err)
}
v := &View{
View: pilosa.NewView(path, index, field, name, pilosa.DefaultCacheSize),
RowAttrStore: test.MustOpenAttrStore(),
}
v.View.RowAttrStore = v.RowAttrStore
return v
}
// MustOpenView creates and opens an view at a temporary path. Panic on error.
func MustOpenView(index, field, name string) *View {
v := NewView(index, field, name)
if err := v.Open(); err != nil {
panic(err)
}
return v
}
// Close closes the view and removes all underlying data.
func (v *View) Close() error {
defer os.Remove(v.Path())
defer v.RowAttrStore.Close()
return v.View.Close()
}
// Reopen closes the view and reopens it as a new instance.
func (v *View) Reopen() error {
path := v.Path()
if err := v.View.Close(); err != nil {
return err
}
v.View = pilosa.NewView(path, v.Index(), v.Field(), v.Name(), pilosa.DefaultCacheSize)
v.View.RowAttrStore = v.RowAttrStore
return v.Open()
}
// MustSetBits sets columns on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := v.SetBit(rowID, columnID); err != nil {
panic(err)
}
}
}
// MustClearColumns clears columns on a row. Panic on error.
func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := v.ClearBit(rowID, columnID); err != nil {
panic(err)
}
}
}
// Ensure view can open and retrieve a fragment.
func TestView_DeleteFragment(t *testing.T) {
v := MustOpenView("i", "f", "v")
defer v.Close()
slice := uint64(9)
// Create fragment.
fragment, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
t.Fatal(err)
} else if fragment == nil {
t.Fatal("expected fragment")
}
err = v.DeleteFragment(slice)
if err != nil {
t.Fatal(err)
}
if v.Fragment(slice) != nil {
t.Fatal("fragment still exists in view")
}
// Recreate fragment with same slice, verify that the old fragment was not reused.
fragment2, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
t.Fatal(err)
} else if fragment == fragment2 {
t.Fatal("failed to create new fragment")
}
}