diff --git a/rbf/array.go b/rbf/array.go index a7753db90..7c4aa3252 100644 --- a/rbf/array.go +++ b/rbf/array.go @@ -1,17 +1,3 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package rbf import ( @@ -30,6 +16,12 @@ func fromArray16(a []uint16) []byte { return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*2 : len(a)*2] } +func cloneArray16(a []uint16) []uint16 { + other := make([]uint16, len(a)) + copy(other, a) + return other +} + // arrayIndex returns the insertion index of v in a. Returns true if exact match. func arrayIndex(a []uint16, v uint16) (int, bool) { return search(len(a), func(i int) int { @@ -67,3 +59,9 @@ func toInterval16(a []byte) []roaring.Interval16 { func fromInterval16(a []roaring.Interval16) []byte { return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4] } + +func cloneInterval16(a []roaring.Interval16) []roaring.Interval16 { + other := make([]roaring.Interval16, len(a)) + copy(other, a) + return other +} diff --git a/rbf/cursor.go b/rbf/cursor.go index 8d2479164..56822341e 100644 --- a/rbf/cursor.go +++ b/rbf/cursor.go @@ -1,20 +1,7 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package rbf import ( + "encoding/binary" "fmt" "io" "math/bits" @@ -25,7 +12,7 @@ import ( ) const ( - bitmapN = (1 << 16) / 64 + BitmapN = (1 << 16) / 64 ) type Cursor struct { @@ -83,10 +70,10 @@ func runAdd(runs []roaring.Interval16, v uint16) ([]roaring.Interval16, bool) { } return runs, true } -func checkRun(runs []roaring.Interval16, key uint64) leafCell { +func checkRun(runs []roaring.Interval16, key uint64) leafArgs { if len(runs) >= RLEMaxSize { //convertToBitmap - bitmap := make([]uint64, bitmapN) + bitmap := make([]uint64, BitmapN) for _, iv := range runs { w1, w2 := iv.Start/64, iv.Last/64 b1, b2 := iv.Start&63, iv.Last&63 @@ -122,9 +109,9 @@ func checkRun(runs []roaring.Interval16, key uint64) leafCell { n += popcount(v) } - return leafCell{Key: key, N: int(n), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)} + return leafArgs{Key: key, N: int(n), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)} } - return leafCell{Key: key, N: len(runs), Type: ContainerTypeRLE, Data: fromInterval16(runs)} + return leafArgs{Key: key, N: len(runs), Type: ContainerTypeRLE, Data: fromInterval16(runs)} } // Add sets a bit on the underlying bitmap. @@ -135,7 +122,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) { if exact, err := c.Seek(hi); err != nil { return false, err } else if !exact { - return true, c.putLeafCell(leafCell{Key: hi, Type: ContainerTypeArray, N: 1, Data: fromArray16([]uint16{lo})}) + return true, c.putLeafCell(leafArgs{Key: hi, Type: ContainerTypeArray, N: 1, Data: fromArray16([]uint16{lo})}) } // If the container exists and bit is not set then update the page. @@ -154,7 +141,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) { copy(other, a[:i]) other[i] = lo copy(other[i+1:], a[i:]) - return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), Data: fromArray16(other)}) + return true, c.putLeafCell(leafArgs{Key: cell.Key, Type: ContainerTypeArray, N: len(other), Data: fromArray16(other)}) case ContainerTypeRLE: runs := toInterval16(cell.Data) @@ -168,14 +155,19 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) { return false, nil case ContainerTypeBitmap: // Exit if bit set in bitmap container. - a := cloneArray64(toArray64(cell.Data)) + pgno, bm, err := cell.GetBitmap(c.tx) + if err != nil { + return false, err + } + + a := cloneArray64(bm) if a[lo/64]&(1< runs[i].Start { + last := runs[i].Last + runs[i].Last = lo - 1 + runs = append(runs, roaring.Interval16{}) + copy(runs[i+2:], runs[i+1:]) + runs[i+1] = roaring.Interval16{Start: lo + 1, Last: last} + } + if len(runs) == 0 { + return true, c.deleteLeafCell(cell.Key) + } + return true, c.putLeafCell(leafArgs{Key: cell.Key, Type: ContainerTypeRLE, N: len(runs), Data: fromInterval16(runs)}) case ContainerTypeBitmap: - // Exit if bit not set in bitmap container. - a := cloneArray64(toArray64(cell.Data)) + pgno, bm, err := cell.GetBitmap(c.tx) + if err != nil { + return false, err + } + a := cloneArray64(bm) if a[lo/64]&(1<= len(cells) || c.Key() != cell.Key { + //new cell + if cell.Type == ContainerTypeBitmap { + //allocated bitmap() + bitmapPgno, _ := c.tx.allocate() + cell.Data = fromPgno(bitmapPgno) + } + + // Shift cells over if this is an insertion. cells = append(cells, leafCell{}) copy(cells[elem.index+1:], cells[elem.index:]) + + } else if in.Type == ContainerTypeBitmap { + ecell := cells[elem.index] + if ecell.Type != ContainerTypeBitmap { + bitmapPgno, _ := c.tx.allocate() + cell.Data = fromPgno(bitmapPgno) + } else { + cell.Data = ecell.Data //fill in the old pgno + } } + + if in.Type == ContainerTypeArray && in.N > ArrayMaxSize { + //convert to bitmap + in.Type = ContainerTypeBitmap + a := make([]uint64, PageSize/8) + for _, v := range toArray16(in.Data) { + a[v/64] |= 1 << uint64(v%64) + } + in.Data = fromArray64(a) + cell.Type = ContainerTypeBitmap + bitmapPgno, _ := c.tx.allocate() + cell.Data = fromPgno(bitmapPgno) + } + cells[elem.index] = cell // Split into multiple pages if page size is exceeded. @@ -291,23 +347,16 @@ func (c *Cursor) putLeafCell(cell leafCell) (err error) { if leafCellsPageSize(cells) >= PageSize { groups = splitLeafCells(cells) } + // Write each group to a separate page. - var hasBitmap bool - - for _, group := range groups { - if len(group) == 1 && (group[0].Type == ContainerTypeBitmap || group[0].N > ArrayMaxSize) && (group[0].Type != ContainerTypeRLE) { - hasBitmap = true - } - } - + newRoot := (len(groups) > 1) && (c.stack.index == 0) var parents []branchCell origPgno := elem.pgno - - newRoot := (len(groups) > 1 || hasBitmap) && c.stack.index == 0 + // newRoot if split occured and bottom of the stack for i, group := range groups { // First page should overwrite the original. // Subsequent pages should allocate new pages. - parent := branchCell{Key: group[0].Key} + parent := branchCell{Key: group[0].Key} //<<< this is the key spot for making sure that key is correct if i == 0 && !newRoot { parent.Pgno = origPgno } else { @@ -316,33 +365,26 @@ func (c *Cursor) putLeafCell(cell leafCell) (err error) { } } - // If cell exceeds threshold then write out bitmap page. - // Otherwise encode leaf page normally. + // if the cell is a bitmap write out its page + if in.Type == ContainerTypeBitmap { + var bm [PageSize]byte + copy(bm[:], fromArray64(toArray64(in.Data))) + c.tx.writeBitmapPage(toPgno(cell.Data), bm[:]) + } var buf [PageSize]byte - if len(group) == 1 && (group[0].Type == ContainerTypeBitmap || group[0].N > ArrayMaxSize) && (group[0].Type != ContainerTypeRLE) { + // Write cells to page. + writePageNo(buf[:], parent.Pgno) + writeFlags(buf[:], PageTypeLeaf) + writeCellN(buf[:], len(group)) - hasBitmap = true - parent.Flags |= ContainerTypeBitmap - copy(buf[:], fromArray64(cell.Bitmap())) + offset := dataOffset(len(group)) + for j, cell := range group { + writeLeafCell(buf[:], j, offset, cell) + offset += align8(cell.Size()) + } - if err := c.tx.writeBitmapPage(parent.Pgno, buf[:]); err != nil { - return err - } - } else { - // Write cells to page. - writePageNo(buf[:], parent.Pgno) - writeFlags(buf[:], PageTypeLeaf) - writeCellN(buf[:], len(group)) - - offset := dataOffset(len(group)) - for j, cell := range group { - writeLeafCell(buf[:], j, offset, cell) - offset += align8(cell.Size()) - } - - if err := c.tx.writePage(buf[:]); err != nil { - return err - } + if err := c.tx.writePage(buf[:]); err != nil { + return err } parents = append(parents, parent) @@ -350,9 +392,8 @@ func (c *Cursor) putLeafCell(cell leafCell) (err error) { // TODO(BBJ): Update page in buffer & cursor stack. - // If this is not a split and we have no bitmap containers, then exit now. - // Bitmap containers require a parent and the parent's flag must be set. - if len(groups) == 1 && !hasBitmap { + // If this is not a split then exit now. + if len(groups) == 1 { return nil } @@ -369,9 +410,15 @@ func (c *Cursor) putLeafCell(cell leafCell) (err error) { // deleteLeafCell removes a cell from the currently positioned page & index. func (c *Cursor) deleteLeafCell(key uint64) (err error) { + cells := readLeafCells(c.leafPage, c.leafCells[:]) elem := &c.stack.elems[c.stack.index] - cells := readLeafCells(c.leafPage, elem.isBitmap, c.leafCells[:]) oldPageKey := cells[0].Key + cell := c.cell() + if cell.Type == ContainerTypeBitmap { + if err := c.tx.deallocate(toPgno(cell.Data)); err != nil { + return err + } + } // If no more cells exist and we have a parent, remove from parent. if c.stack.index > 0 && len(cells) == 1 { @@ -570,7 +617,7 @@ func (c *Cursor) deleteBranchCell(stackIndex int, key uint64) (err error) { return err } - if stackIndex > 0 && oldPageKey != cells[0].Key { + if stackIndex > 0 && len(cells) > 0 && oldPageKey != cells[0].Key { return c.updateBranchCell(stackIndex-1, cells[0].Key) } return nil @@ -606,6 +653,9 @@ func splitLeafCells(cells []leafCell) [][]leafCell { // half a page then create a new group of cells. if cellN != 0 && (dataOffset(cellN+1)+dataSize+sz) > (PageSize*60)/100 { slices, dataSize = append(slices, nil), 0 + } else if cellN != 0 && cell.Type == ContainerTypeArray && cell.N > ArrayMaxSize { + slices, dataSize = append(slices, nil), 0 + sz = PageSize } // Append to current slice & increase total cell data size. @@ -644,18 +694,12 @@ func splitBranchCells(cells []branchCell) [][]branchCell { // Key returns the key that the cursor is currently positioned over. func (c *Cursor) Key() uint64 { elem := &c.stack.elems[c.stack.index] - if elem.isBitmap { - return elem.key - } offset := readCellOffset(c.leafPage, elem.index) return *(*uint64)(unsafe.Pointer(&c.leafPage[offset])) } func (c *Cursor) cell() leafCell { elem := &c.stack.elems[c.stack.index] - if elem.isBitmap { - return leafCell{Type: ContainerTypeBitmap, Key: elem.key, Data: c.leafPage[:]} - } return readLeafCell(c.leafPage[:], elem.index) } @@ -677,21 +721,10 @@ func (c *Cursor) First() error { // Read cell pgno into the next stack level. cell := readBranchCell(buf, elem.index) - isBitmap := cell.Flags&ContainerTypeBitmap != 0 c.stack.elems[c.stack.index+1] = stackElem{ - pgno: cell.Pgno, - key: cell.Key, - isBitmap: isBitmap, - } - - // If cell points at a bitmap page then increment stack but exit immediately. - if isBitmap { - c.stack.index++ - if c.leafPage, err = c.tx.readPage(cell.Pgno); err != nil { - return err - } - return nil + pgno: cell.Pgno, + key: cell.Key, } case PageTypeLeaf: @@ -726,21 +759,9 @@ func (c *Cursor) Last() error { // Read cell pgno into the next stack level. cell := readBranchCell(buf, elem.index) - isBitmap := cell.Flags&ContainerTypeBitmap != 0 - c.stack.elems[c.stack.index+1] = stackElem{ - pgno: cell.Pgno, - key: cell.Key, - isBitmap: isBitmap, - } - - // If cell points at a bitmap page then increment stack but exit immediately. - if isBitmap { - c.stack.index++ - if c.leafPage, err = c.tx.readPage(cell.Pgno); err != nil { - return err - } - return nil + pgno: cell.Pgno, + key: cell.Key, } case PageTypeLeaf: @@ -780,6 +801,7 @@ func (c *Cursor) Seek(key uint64) (exact bool, err error) { } return 1 }) + //if not found (ok) the cell if !ok && index > 0 { index-- } @@ -788,21 +810,10 @@ func (c *Cursor) Seek(key uint64) (exact bool, err error) { // Read cell pgno into the next stack level. cell := readBranchCell(buf, elem.index) - isBitmap := cell.Flags&ContainerTypeBitmap != 0 c.stack.elems[c.stack.index+1] = stackElem{ - pgno: cell.Pgno, - key: cell.Key, - isBitmap: isBitmap, - } - - // If cell points at a bitmap page then increment stack but exit immediately. - if isBitmap { - c.stack.index++ - if c.leafPage, err = c.tx.readPage(cell.Pgno); err != nil { - return false, err - } - return ok, nil + pgno: cell.Pgno, + key: cell.Key, } case PageTypeLeaf: @@ -833,7 +844,7 @@ func (c *Cursor) Next() error { } // Move forward to the next leaf element if available. - if elem := &c.stack.elems[c.stack.index]; !elem.isBitmap && elem.index < readCellN(c.leafPage)-1 { + if elem := &c.stack.elems[c.stack.index]; elem.index < readCellN(c.leafPage)-1 { elem.index++ return nil } @@ -848,7 +859,7 @@ func (c *Cursor) Prev() error { } // Move forward to the next leaf element if available. - if elem := &c.stack.elems[c.stack.index]; !elem.isBitmap && elem.index > 0 { + if elem := &c.stack.elems[c.stack.index]; elem.index > 0 { elem.index-- return nil } @@ -880,21 +891,10 @@ func (c *Cursor) Prev() error { switch typ := readFlags(buf); typ { case PageTypeBranch: cell := readBranchCell(buf, elem.index) - isBitmap := cell.Flags&ContainerTypeBitmap != 0 c.stack.elems[c.stack.index+1] = stackElem{ - pgno: cell.Pgno, - key: cell.Key, - isBitmap: isBitmap, - } - - // If cell points at a bitmap page then increment stack but exit immediately. - if isBitmap { - c.stack.index++ - if c.leafPage, err = c.tx.readPage(cell.Pgno); err != nil { - return err - } - return nil + pgno: cell.Pgno, + key: cell.Key, } case PageTypeLeaf: @@ -936,7 +936,8 @@ func (c *Cursor) Union(rowID uint64, row []uint64) error { case ContainerTypeRLE: panic("TODO(BBJ): rbf.Bitmap.Union() RLE support") case ContainerTypeBitmap: - for i, v := range toArray64(cell.Data) { + _, bm, _ := cell.GetBitmap(c.tx) + for i, v := range bm { row[(offset/64)+uint64(i)] |= v } default: @@ -974,13 +975,17 @@ func (c *Cursor) Intersect(rowID uint64, row []uint64) error { switch cell.Type { case ContainerTypeArray: - for i, v := range cell.Bitmap() { + for i, v := range cell.Bitmap(c.tx) { row[(offset/64)+uint64(i)] &= v } case ContainerTypeRLE: panic("TODO(BBJ): rbf.Bitmap.Intersect() RLE support") case ContainerTypeBitmap: - for i, v := range toArray64(cell.Data) { + _, bm, err := cell.GetBitmap(c.tx) + if err != nil { + return err + } + for i, v := range bm { row[(offset/64)+uint64(i)] &= v } default: @@ -1003,21 +1008,15 @@ func (c *Cursor) Intersect(rowID uint64, row []uint64) error { // Values returns the values for the container the cursor is currently pointing to. func (c *Cursor) Values() []uint16 { elem := &c.stack.elems[c.stack.index] - var cell leafCell - if elem.isBitmap { - cell = leafCell{Type: ContainerTypeBitmap, Key: elem.key, Data: c.leafPage} - } else { - cell = readLeafCell(c.leafPage[:], elem.index) - } - return cell.Values() + cell := readLeafCell(c.leafPage[:], elem.index) + return cell.Values(c.tx) } // stackElem represents a single element on the cursor stack. type stackElem struct { - pgno uint32 // current page number - index int // cell index - key uint64 // element key - isBitmap bool // if true, entire page is a bitmap + pgno uint32 // current page number + index int // cell index + key uint64 // element key } func (c *Cursor) goNextPage() error { @@ -1048,23 +1047,10 @@ func (c *Cursor) goNextPage() error { switch typ := readFlags(buf); typ { case PageTypeBranch: cell := readBranchCell(buf, elem.index) - isBitmap := cell.Flags&ContainerTypeBitmap != 0 - c.stack.elems[c.stack.index+1] = stackElem{ - pgno: cell.Pgno, - key: cell.Key, - isBitmap: isBitmap, + pgno: cell.Pgno, + key: cell.Key, } - - // If cell points at a bitmap page then increment stack but exit immediately. - if isBitmap { - c.stack.index++ - if c.leafPage, err = c.tx.readPage(cell.Pgno); err != nil { - return err - } - return nil - } - case PageTypeLeaf: elem.index = 0 c.leafPage = buf @@ -1075,8 +1061,7 @@ func (c *Cursor) goNextPage() error { } } -func ConvertToLeaf(key uint64, c *roaring.Container) (result leafCell) { - //TODO(twg) clean up roaring constant import export +func ConvertToLeafArgs(key uint64, c *roaring.Container) (result leafArgs) { result.Key = key result.N = int(c.N()) result.Type = ContainerTypeNone @@ -1110,7 +1095,6 @@ func ConvertToLeaf(key uint64, c *roaring.Container) (result leafCell) { result.Type = ContainerTypeRLE result.Data = fromInterval16(r) return - } return } @@ -1123,16 +1107,18 @@ func (c *Cursor) merge(key uint64, data *roaring.Container) (bool, error) { d := toArray16(cell.Data) container = roaring.NewContainerArray(d) case ContainerTypeBitmap: - d := toArray64(cell.Data) + _, d, err := cell.GetBitmap(c.tx) + if err != nil { + return false, err + } container = roaring.NewContainerBitmap(cell.N, d) case ContainerTypeRLE: d := toInterval16(cell.Data) container = roaring.NewContainerRun(d) } - res := roaring.Union(data, container) if res.N() != data.N() { - leaf := ConvertToLeaf(key, res) + leaf := ConvertToLeafArgs(key, res) err := c.putLeafCell(leaf) return true, err } @@ -1144,7 +1130,7 @@ func (c *Cursor) AddRoaring(bm *roaring.Bitmap) (changed bool, err error) { itr, _ := bm.Containers.Iterator(0) for itr.Next() { hi, cont := itr.Value() - leaf := ConvertToLeaf(hi, cont) + leaf := ConvertToLeafArgs(hi, cont) if leaf.N == 0 { continue } @@ -1160,7 +1146,6 @@ func (c *Cursor) AddRoaring(bm *roaring.Bitmap) (changed bool, err error) { changed = true continue } - // If the container exists and bit is not set then update the page. u, err := c.merge(hi, cont) if err != nil { @@ -1176,3 +1161,59 @@ func (c *Cursor) AddRoaring(bm *roaring.Bitmap) (changed bool, err error) { func popcount(x uint64) uint64 { return uint64(bits.OnesCount64(x)) } + +func (c *Cursor) RemoveRoaring(bm *roaring.Bitmap) (changed bool, err error) { + itr, _ := bm.Containers.Iterator(0) + for itr.Next() { + hi, cont := itr.Value() + if cont.N() == 0 { + continue + } + // Move cursor to the key of the container. + // Insert new container if it doesn't exist. + if exact, err := c.Seek(hi); err != nil { + return false, err + } else if exact { + f, err := c.difference(hi, cont) + if err != nil { + return f, err + } + if f { + changed = true + } + } + } + return +} + +func (c *Cursor) difference(key uint64, data *roaring.Container) (bool, error) { + cell := c.cell() + var container *roaring.Container + switch cell.Type { + case ContainerTypeArray: + d := toArray16(cell.Data) + container = roaring.NewContainerArray(d) + case ContainerTypeBitmap: + _, d, err := cell.GetBitmap(c.tx) + if err != nil { + return false, err + } + container = roaring.NewContainerBitmap(cell.N, d) + case ContainerTypeRLE: + d := toInterval16(cell.Data) + container = roaring.NewContainerRun(d) + } + + res := roaring.Difference(container, data) + if res == nil { + return true, c.deleteLeafCell(cell.Key) + } + + if res.N() != container.N() { + leaf := ConvertToLeafArgs(key, res) + err := c.putLeafCell(leaf) + return true, err + } + + return false, nil +} diff --git a/rbf/cursor_test.go b/rbf/cursor_test.go index d918169ac..b6b8330d9 100644 --- a/rbf/cursor_test.go +++ b/rbf/cursor_test.go @@ -19,6 +19,7 @@ import ( "math/rand" "reflect" "sort" + "strings" "testing" "github.com/pilosa/pilosa/v2/rbf" @@ -762,6 +763,7 @@ func TestCursor_RLEConversion(t *testing.T) { } else if err := c.First(); err != nil { t.Fatal(err) } + if c.CurrentPageType() != rbf.ContainerTypeRLE { t.Fatalf("Should Be RLE but is: %v\n", c.CurrentPageType()) } @@ -772,6 +774,14 @@ func TestCursor_RLEConversion(t *testing.T) { if !exists { t.Fatalf("Should Contain %v", 0x7) } + exists, err = c.Contains(0x6) + if err != nil { + t.Fatalf("ERR:%v", err) + } + if exists { + t.Fatalf("Should Not Contain %v", 0x6) + } + //add a few bits to create another run _, err = tx.Add("x", func() []uint64 { @@ -797,3 +807,244 @@ func TestCursor_RLEConversion(t *testing.T) { } } + +type EasyWalker struct { + tx *rbf.Tx + path strings.Builder +} + +func (e *EasyWalker) Visitor(pgno uint32, records []*rbf.RootRecord) { + for _, record := range records { + e.VisitRoot(record.Pgno, record.Name) + rbf.Page(e.tx, record.Pgno, e) + } +} +func (e *EasyWalker) VisitRoot(pgno uint32, name string) { + e.path.WriteString("R") +} +func (e *EasyWalker) VisitBranch(pgno uint32) { + e.path.WriteString("B") + +} +func (e *EasyWalker) VisitLeaf(pgno uint32) { + e.path.WriteString("L") + +} +func (e *EasyWalker) VisitBitmap(pgno uint32) { +} +func (e *EasyWalker) String() string { + return e.path.String() +} + +func TestCursor_UpdateBranchCells(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer MustRollback(t, tx) + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + c, err := tx.Cursor("x") + if err != nil { + t.Fatal(err) + } + if err != nil { + t.Fatal(err) + } + changed, err := c.Add(1) + if changed { + + if err := c.First(); err != nil { + t.Fatal(err) + } + if got, want := c.Values(), []uint16{uint16(1)}; !reflect.DeepEqual(got, want) { + t.Fatal(err) + } + } else { + t.Fatal("Expected Add Change") + } + changed, err = c.Remove(1) + if changed { + c.First() + if got, want := c.Values(), []uint16{}; !reflect.DeepEqual(got, want) { + t.Fatal(err) + } + } else { + t.Fatal("Expected Remove Change") + } + + rb := func() *roaring.Bitmap { + bm := roaring.NewBitmap() + bm.Put(0, roaring.NewContainerRun([]roaring.Interval16{{Start: 1, Last: 2}})) + return bm + }() + _, err = tx.AddRoaring("x", rb) + if err != nil { + t.Fatal(err) + } + changed, err = c.Remove(2) + if changed { + c.First() + if got, want := c.Values(), []uint16{1}; !reflect.DeepEqual(got, want) { + t.Fatal(err) + } + } else { + t.Fatal("Expected Remove Change") + } + + changed, err = c.Remove(1) + if changed { + c.First() + if got, want := c.Values(), []uint16{}; !reflect.DeepEqual(got, want) { + t.Fatal(err) + } + } else { + t.Fatal("Expected Remove Change") + } + +} + +func TestCursor_SplitBranchCells(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer MustRollback(t, tx) + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + rb := func(key uint64) *roaring.Bitmap { + bm := roaring.NewBitmap() + bits := make([]uint64, rbf.BitmapN) + n := 0 + for i := range bits { + bits[i] = ^uint64(0) + n += 64 + } + bm.Put(key, roaring.NewContainerBitmap(n, bits)) + return bm + } + //634 == offset, 24== size of leafcell with bitmap + // measured should split at 634+(24*314) + numContainers := 314 + for i := 0; i < numContainers; i++ { //need to calculate how many will force a split + b := rb(uint64(i)) + tx.AddRoaring("x", b) + } + before := &EasyWalker{tx: tx} + rbf.Page(tx, 0, before) + if before.String() != "RL" { + + t.Fatalf("Expecting RL (one branch) got %v", before.String()) + + } + // adding one more container should split it + tx.AddRoaring("x", rb(uint64(numContainers))) + after := &EasyWalker{tx: tx} + rbf.Page(tx, 0, after) + if after.String() != "RBLL" { + + t.Fatalf("Expecting RBLL (a branch split) got %v", after.String()) + + } + +} + +func TestCursor_RemoveCells(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer MustRollback(t, tx) + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + cur, _ := tx.Cursor("x") + rb := func(key uint64) *roaring.Bitmap { + bm := roaring.NewBitmap() + bits := make([]uint64, rbf.BitmapN) + n := 0 + for i := range bits { + bits[i] = ^uint64(0) + n += 64 + } + bm.Put(key, roaring.NewContainerBitmap(n, bits)) + return bm + } + numContainers := 455 //enough containers to cause a split + for i := 0; i < numContainers; i++ { + b := rb(uint64(i)) + tx.AddRoaring("x", b) + } + + for i := numContainers; i >= 1; i-- { + cur.RemoveRoaring(rb(uint64(i))) + } + + cur.RemoveRoaring(rb(uint64(0))) + + //f, err := os.OpenFile("before.dot", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 066) +} + +//These aren't test i'm just using to generate graphs to look at structure +func TestCursor_PlayContainer(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer MustRollback(t, tx) + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + many := func(c *rbf.Cursor, start, count uint64) { + for i := start; i < start+count; i++ { + c.Add(i) + } + } + cur, _ := tx.Cursor("x") + offset := uint64(0) + many(cur, 0, rbf.ArrayMaxSize+offset) + many(cur, 65536, rbf.ArrayMaxSize+offset) + /* + many(cur, 2*65536, rbf.ArrayMaxSize+offset) + many(cur, 3*65536, rbf.ArrayMaxSize) //+offset) + many(cur, 4*65536, rbf.ArrayMaxSize) //+offset) + many(cur, 5*65536, rbf.ArrayMaxSize) //+offset) + many(cur, 6*65536, 10) //+offset) + many(cur, 7*65536, 10) //+offset) + many(cur, 8*65536, rbf.ArrayMaxSize+10) //+offset) + many(cur, 9*65536, rbf.ArrayMaxSize+10) //+offset) + */ + + cur.First() + cur.Dump("fun.dot") +} + +func TestCursor_OneBitmap(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer MustRollback(t, tx) + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + rb := func(key uint64) *roaring.Bitmap { + bm := roaring.NewBitmap() + bits := make([]uint64, rbf.BitmapN) + n := 0 + for i := range bits { + bits[i] = ^uint64(0) + n += 64 + } + bm.Put(key, roaring.NewContainerBitmap(n, bits)) + return bm + } + numContainers := 4 + for i := 0; i < numContainers; i++ { //need to calculate how many will force a split + b := rb(uint64(i)) // measured at i=454 seems reasonable should occur at Len(branchcells)+header >8192 + tx.AddRoaring("x", b) + } + cur, err := tx.Cursor("x") + if err != nil { + panic(err) + } + cur.First() + cur.Dump("fun.dot") +} diff --git a/rbf/cursorx.go b/rbf/cursorx.go index 35d5818c2..366d44b00 100644 --- a/rbf/cursorx.go +++ b/rbf/cursorx.go @@ -27,9 +27,7 @@ import ( // but for now i'll do it func (c *Cursor) Rows() ([]uint64, error) { shardVsContainerExponent := uint(4) //needs constant exported from roaring package - if err := c.First(); err != nil { - return nil, err - } + c.First() rows := make([]uint64, 0) var err error var lastRow uint64 = math.MaxUint64 @@ -48,7 +46,6 @@ func (c *Cursor) Rows() ([]uint64, error) { } return rows, err } - func (tx *Tx) FieldViews() []string { r, _ := tx.rootRecords() res := make([]string, len(r)) @@ -57,23 +54,17 @@ func (tx *Tx) FieldViews() []string { } return res } - -func (c *Cursor) DumpKeys() error { - if err := c.First(); err != nil { - return err - } +func (c *Cursor) DumpKeys() { + c.First() for { err := c.Next() if err == io.EOF { - return nil - } else if err != nil { - return err + break } cell := c.cell() fmt.Println("key", cell.Key) } } - func (c *Cursor) DumpStack() { fmt.Println("STACK") for i := c.stack.index; i >= 0; i-- { @@ -81,18 +72,18 @@ func (c *Cursor) DumpStack() { } fmt.Println() } - -func (c *Cursor) Dump() { - bufStdout := bufio.NewWriter(os.Stdout) - defer bufStdout.Flush() +func (c *Cursor) Dump(name string) { + writer, _ := os.Create(name) + defer writer.Close() + bufStdout := bufio.NewWriter(writer) fmt.Fprintf(bufStdout, "digraph RBF{\n") fmt.Fprintf(bufStdout, "rankdir=\"LR\"\n") fmt.Fprintf(bufStdout, "node [shape=record height=.1]\n") dumpdot(c.tx, 0, " ", bufStdout) fmt.Fprintf(bufStdout, "\n}") + bufStdout.Flush() } - func (c *Cursor) Row(rowID uint64) (*roaring.Bitmap, error) { base := rowID * ShardWidth @@ -108,9 +99,7 @@ func (c *Cursor) Row(rowID uint64) (*roaring.Bitmap, error) { elem := &c.stack.elems[c.stack.index] n := readCellN(c.leafPage) if elem.index >= n { - if err := c.goNextPage(); err != nil { - return nil, err - } + c.goNextPage() } } other := roaring.NewSliceBitmap() @@ -149,3 +138,41 @@ func toContainer(l leafCell) *roaring.Container { } return nil } + +type Walker interface { + Visitor(pgno uint32, records []*RootRecord) + VisitRoot(pgno uint32, name string) + VisitBranch(pgno uint32) + VisitLeaf(pgno uint32) + VisitBitmap(pgno uint32) +} + +func Page(tx *Tx, pgno uint32, walker Walker) { + page, err := tx.readPage(pgno) + if err != nil { + panic(err) + } + + if IsMetaPage(page) { + Walk(tx, readMetaRootRecordPageNo(page), walker.Visitor) + return + } + + // Handle + switch typ := readFlags(page); typ { + case PageTypeBranch: + walker.VisitBranch(pgno) + for i, n := 0, readCellN(page); i < n; i++ { + cell := readBranchCell(page, i) + if cell.Flags&ContainerTypeBitmap == 0 { // leaf/branch child page + Page(tx, cell.Pgno, walker) + } else { + walker.VisitBitmap(cell.Pgno) + } + } + case PageTypeLeaf: + walker.VisitLeaf(pgno) + default: + panic(err) + } +} diff --git a/rbf/dot.go b/rbf/dot.go index 38b386234..adbb96dcf 100644 --- a/rbf/dot.go +++ b/rbf/dot.go @@ -1,17 +1,4 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - +// Package rbf implements the roaring b-tree file format. package rbf import ( @@ -32,7 +19,8 @@ func dotCell(b []byte, parent string, writer io.Writer) { switch { case flags&PageTypeLeaf != 0: fmt.Fprintf(writer, "cell%d [ shape=none label=<\n", pgno) - fmt.Fprintf(writer, "\n") + fmt.Fprintf(writer, "\n", pgno) + links := make([]string, 0) for i := 0; i < cellN; i++ { cell := readLeafCell(b, i) switch cell.Type { @@ -41,12 +29,19 @@ func dotCell(b []byte, parent string, writer io.Writer) { fmt.Fprintf(writer, "\n", i, cell.Key, cell.N) case ContainerTypeRLE: fmt.Fprintf(writer, "\n", i, cell.Key, cell.N) + case ContainerTypeBitmap: + bpn := toPgno(cell.Data) + fmt.Fprintf(writer, "\n", bpn, i, cell.Key, cell.N) + links = append(links, fmt.Sprintf("bitmap%d[label=\"bitmap (%d)\"]\n cell%d:%d -> bitmap%d\n", bpn, bpn, pgno, i, bpn)) default: - fmt.Fprintf(writer, "\n", i, cell.Key, cell.Type, cell.N) + fmt.Fprintf(writer, "\n", i, cell.Key, cell.Type, cell.N) } } fmt.Fprintf(writer, "
CELL
CELL (%d)
[%d]: key=%d type=array n=%d
[%d]: key=%d type=rle n=%d
[%d]: key=%d type=bitmap n=%d
[%d]: key=%d type=unknown<%d> n=%d
[%d]: key=%d type=unknown<%d> n=%d
>]\n") fmt.Fprintf(writer, "%s -> cell%d\n", parent, pgno) + for _, link := range links { + fmt.Fprintf(writer, "%s", link) + } default: //should not happen fmt.Fprintf(writer, "==!PAGE %d flags=%d\n", pgno, flags) @@ -76,7 +71,7 @@ func dumpdot(tx *Tx, pgno uint32, parent string, writer io.Writer) { } } - rrdump(tx, readMetaRootRecordPageNo(page), visitor) + Walk(tx, readMetaRootRecordPageNo(page), visitor) return } @@ -92,14 +87,12 @@ func dumpdot(tx *Tx, pgno uint32, parent string, writer io.Writer) { dumpdot(tx, cell.Pgno, p, writer) } else { b := fmt.Sprintf("bm%d", cell.Pgno) - fmt.Fprintf(writer, "%s[label=\"BITMAP(%d)\"]\n %s -> %s\n", b, cell.Pgno, p, b) + fmt.Fprintf(writer, "%s[label=\"BITMAP(%d) key=%d \"]\n %s -> %s\n", b, cell.Pgno, cell.Key, p, b) } } case PageTypeLeaf: p := fmt.Sprintf("leaf%d", pgno) fmt.Fprintf(writer, "%s[label=\"LEAF(%d)| n=%d\"]\n%s->%s\n", p, pgno, readCellN(page), parent, p) dotCell(page, p, writer) - default: - panic(err) } } diff --git a/rbf/internal_test.go b/rbf/internal_test.go index 985642c0c..9711384cc 100644 --- a/rbf/internal_test.go +++ b/rbf/internal_test.go @@ -21,6 +21,6 @@ func TestUsed(t *testing.T) { t.Skip("This function is always skipped") dump(nil) hexdump(nil) - pagedump(nil, "", nil) + pagedumpi(nil, "", nil) treedump(nil, 0, "", nil) } diff --git a/rbf/rbf.go b/rbf/rbf.go index 1b5f4e8d6..5e5707d3a 100644 --- a/rbf/rbf.go +++ b/rbf/rbf.go @@ -1,17 +1,3 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - // Package rbf implements the roaring b-tree file format. package rbf @@ -113,13 +99,12 @@ func writeMetaRootRecordPageNo(page []byte, pgno uint32) { binary.BigEndian.PutU func readMetaFreelistPageNo(page []byte) uint32 { return binary.BigEndian.Uint32(page[24:]) } func writeMetaFreelistPageNo(page []byte, pgno uint32) { binary.BigEndian.PutUint32(page[24:], pgno) } -// func readMetaChecksum(page []byte) uint32 { -// return binary.BigEndian.Uint32(page[PageSize-4 : PageSize]) -// } - -// func writeMetaChecksum(page []byte, chksum uint32) { -// binary.BigEndian.PutUint32(page[PageSize-4:PageSize], chksum) -// } +func readMetaChecksum(page []byte) uint32 { + return binary.BigEndian.Uint32(page[PageSize-4 : PageSize]) +} +func writeMetaChecksum(page []byte, chksum uint32) { + binary.BigEndian.PutUint32(page[PageSize-4:PageSize], chksum) +} // Root record page helpers @@ -164,7 +149,6 @@ func readCellN(page []byte) int { return int(binary.BigEndian.Uint16(page[8: func writeCellN(page []byte, v int) { binary.BigEndian.PutUint16(page[8:10], uint16(v)) } func readCellOffset(page []byte, i int) int { - assert(i < readCellN(page)) return int(binary.BigEndian.Uint16(page[10+(i*2):])) } @@ -177,8 +161,11 @@ func dataOffset(n int) int { } func IsBitmapHeader(page []byte) bool { + if readFlags(page) != PageTypeBitmapHeader { + return false + } // TODO(BBJ): Verify checksum. - return readFlags(page) == PageTypeBitmapHeader + return true } type RootRecord struct { @@ -262,17 +249,23 @@ type leafCell struct { N int Data []byte } +type leafArgs leafCell // Size returns the size of the leaf cell, in bytes. func (c *leafCell) Size() int { - if c.Type == ContainerTypeBitmap { - return PageSize - } return leafCellHeaderSize + len(c.Data) } +func (c *leafCell) GetBitmap(tx *Tx) (pgno uint32, bm []uint64, err error) { + pgno = toPgno(c.Data) + page, err := tx.readPage(pgno) + if err != nil { + return 0, nil, err + } + return pgno, toArray64(page), err +} // Bitmap returns a bitmap representation of the cell data. -func (c *leafCell) Bitmap() []uint64 { +func (c *leafCell) Bitmap(tx *Tx) []uint64 { switch c.Type { case ContainerTypeArray: buf := make([]uint64, PageSize/8) @@ -300,14 +293,15 @@ func (c *leafCell) Bitmap() []uint64 { } return buf case ContainerTypeBitmap: - return toArray64(c.Data) + _, bm, _ := c.GetBitmap(tx) + return bm default: panic(fmt.Sprintf("invalid container type: %d", c.Type)) } } // Values returns a slice of 16-bit values from a container. -func (c *leafCell) Values() []uint16 { +func (c *leafCell) Values(tx *Tx) []uint16 { switch c.Type { case ContainerTypeArray: return toArray16(c.Data) @@ -324,8 +318,9 @@ func (c *leafCell) Values() []uint16 { a = a[:n] return a case ContainerTypeBitmap: - a := make([]uint16, 0, ArrayMaxSize) - for i, v := range toArray64(c.Data) { + a := make([]uint16, 0, BitmapN*64) + _, bm, _ := c.GetBitmap(tx) + for i, v := range bm { for j := uint(0); j < 64; j++ { if v&(1<= v })) if i < int32(len(a)) { @@ -3176,7 +3176,7 @@ func binSearchRuns(v uint16, a []Interval16) (int32, bool) { // runContains determines if v is in the container assuming c is a run // container. func (c *Container) runContains(v uint16) bool { - _, found := binSearchRuns(v, c.runs()) + _, found := BinSearchRuns(v, c.runs()) return found } @@ -3239,7 +3239,7 @@ func (c *Container) bitmapRemove(v uint16) (*Container, bool) { // runRemove removes v from a run container, and returns true if v was removed. func (c *Container) runRemove(v uint16) (*Container, bool) { runs := c.runs() - i, contains := binSearchRuns(v, runs) + i, contains := BinSearchRuns(v, runs) if !contains { return c, false } @@ -6783,3 +6783,6 @@ func Optimize(c *Container) { func Union(a, b *Container) *Container { return union(a, b) } +func Difference(a, b *Container) *Container { + return difference(a, b) +}