scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fixed things broken in hash tables

Browse source
This commit is contained in:
pokeeffe-molecula 2023-04-05 10:46:09 -05:00
parent 9d294c4f39
commit 2d3f09c92b
5 changed files with 201 additions and 124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

149
bufferpool/page.go
View file

@ -2,7 +2,6 @@ package bufferpool
import (
"encoding/binary"
"errors"
"fmt"
"sync"
)
@ -39,8 +38,8 @@ const PAGE_TYPE_HASH_TABLE = 12
// | <start of slot array 1..slotCount> |
// |----------------------------------------------------|
// | 52 | slotcount | slot entry is int16 |
// | | * slotwidth | values (payloadOffset, |
// | | * #slots | payloadLength) |
// | | * slotwidth | values (payloadOffset) |
// | | * #slots | |
// |----------------------------------------------------|
// | <free space> |
// |----------------------------------------------------|
@ -53,6 +52,12 @@ const PAGE_TYPE_HASH_TABLE = 12
// keyBytes
// ptrValue (int64) (page number)
// == hash table payload chunk ==
// keyLength (int16)
// keyBytes
// rowPayloadChunkLen (int16)
// rowPayloadChunkBytes
// == leaf payload chunk ==
// keyLength (int16)
// keyBytes
@ -87,15 +92,16 @@ const PAGE_SLOTS_START_OFFSET = 52 // offset 52
// PAGE_SLOT_LENGTH is the size of the page slot offset value.
const PAGE_SLOT_LENGTH = 2
// 1k is the max key size for now
// MAX_KEY_ON_PAGE_SIZE is 1k; the max key size for now
// we can make this bigger later with overflow
const MAX_KEY_ON_PAGE_SIZE = 1024
// 768 bytes is the max payload on a page before we overflow
// this gives us 1792 bytes for key and payload
// available space on a page after header is 8144 ish
// this gives us room for 4 key/value @ 2036 bytes per page, so
// there is a little bit of wiggle room
// MAX_PAYLOAD_CHUNK_SIZE_ON_PAGE bytes is the max payload on a
// page before we overflow. The default of 768 (assumming max key size)
// gives us 1792 bytes for key and payload.
// Thus available space on a page after header is 8144 (ish)
// This gives us room for 4 key/value @ 2036 bytes per page, plus
// a little bit of wiggle room.
const MAX_PAYLOAD_CHUNK_SIZE_ON_PAGE = 768
type PageLatchState int
@ -239,49 +245,6 @@ func (p *Page) WritePageSlot(slot int16, value PageSlot) {
binary.BigEndian.PutUint16(p.data[offset:], uint16(value.PayloadOffset))
}
// TODO(pok) deprecate this once we get b+tree working, this
// is just used in hash table right now
func (p *Page) PutKeyValueInPageSlot(slotNumber int16, keyBytes []byte, payloadBytes []byte) error {
freeSpaceOffset := p.ReadFreeSpaceOffset()
keyLength := len(keyBytes)
payloadChunkLength := len(payloadBytes)
// check for overflow
if payloadChunkLength > MAX_PAYLOAD_CHUNK_SIZE_ON_PAGE {
return errors.New("overflow")
}
totalPayloadSize := p.ComputeLeafPayloadTotalLength(keyLength, payloadChunkLength)
// check to make sure we don't blow free space
if p.FreeSpaceOnPage() < int16(totalPayloadSize) {
return errors.New("page is full")
}
// compute the new free space offset
freeSpaceOffset -= int16(totalPayloadSize)
offset := freeSpaceOffset
// write the header
offset = p.WriteLeafPagePayloadHeader(offset, int16(keyLength), keyBytes, 0, 0, int32(payloadChunkLength))
// write the payload
p.WriteLeafPagePayloadBytes(offset, int16(payloadChunkLength), payloadBytes)
// update the free space offset
p.WriteFreeSpaceOffset(int16(freeSpaceOffset))
// make a slot
slot := PageSlot{
PayloadOffset: freeSpaceOffset,
}
// write the slot
p.WritePageSlot(slotNumber, slot)
return nil
}
func (p *Page) ComputeInternalPayloadTotalLength(keyLength int) int32 {
l := /*keyLength*/ 2 + keyLength + /*ptrValue*/ 8
return int32(l)
@ -304,6 +267,11 @@ func (p *Page) ComputeLeafPayloadTotalLength(keyLength int, payloadLength int) i
return int32(l)
}
func (p *Page) ComputeHashPayloadTotalLength(keyLength int, payloadLength int) int32 {
l := /*keyLength*/ 2 + keyLength + /*payLoadLength*/ 2 + payloadLength
return int32(l)
}
func (p *Page) WriteLeafPagePayloadHeader(offset int16, keyLen int16, keyBytes []byte, flags int8, overflowPtr int64, payloadTotalLen int32) int16 {
binary.BigEndian.PutUint16(p.data[offset:], uint16(keyLen))
offset += 2
@ -340,6 +308,21 @@ func (p *Page) ReadLeafPagePayloadBytes(offset int16) (int16, []byte) {
return chunkLen, chunkBytes
}
func (p *Page) WriteHashPagePayloadBytes(offset int16, keyLen int16, keyBytes []byte, payloadChunkLength int16, payloadChunkBytes []byte) {
// write key len
binary.BigEndian.PutUint16(p.data[offset:], uint16(keyLen))
offset += 2
// write key
copy(p.data[offset:], keyBytes)
offset += keyLen
// chunk length
binary.BigEndian.PutUint16(p.data[offset:], uint16(payloadChunkLength))
offset += 2
// now copy the payload bytes
copy(p.data[offset:], payloadChunkBytes)
p.isDirty = true
}
func (p *Page) WriteInternalPageChunk(offset int16, chunk InternalPageChunk) {
binary.BigEndian.PutUint16(p.data[offset:], uint16(chunk.KeyLength))
offset += 2
@ -500,16 +483,6 @@ func (l *LeafPayload) ValueLength(page *Page) int32 {
return valueLen
}
// only call this if you are sure this there is no overflow
func (l *LeafPayload) ValueAsBytes(page *Page) []byte {
offset := l.valueOffset(page)
valueLen := int32(binary.BigEndian.Uint32(page.data[offset:]))
offset += 4
result := make([]byte, valueLen)
copy(result, page.data[offset:int32(offset)+valueLen])
return result
}
func (l *LeafPayload) GetPayloadReader(page *Page) LeafPagePayLoadReader {
offset := l.BaseOffset
keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
@ -549,6 +522,42 @@ func (l *LeafPayload) IsVisibleToTID(page *Page, tid int64) bool {
return true
}
type HashPayload struct {
BaseOffset int16
}
func (l *HashPayload) valueOffset(page *Page) int16 {
offset := l.BaseOffset
keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
offset += 2 + keyLen
return offset
}
func (l *HashPayload) ValueLength(page *Page) int16 {
offset := l.valueOffset(page)
valueLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
return valueLen
}
func (l *HashPayload) GetPayloadReader(page *Page) HashPagePayLoadReader {
offset := l.BaseOffset
keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
offset += 2
keyBytes := make([]byte, keyLen)
copy(keyBytes, page.data[offset:offset+keyLen])
offset += keyLen
valueLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
offset += 2
valueBytes := make([]byte, valueLen)
copy(valueBytes, page.data[offset:int16(offset)+valueLen])
return HashPagePayLoadReader{
KeyLength: keyLen,
KeyBytes: keyBytes,
PayloadChunkLength: valueLen,
PayloadChunkBytes: valueBytes,
}
}
type PageSlot struct {
PayloadOffset int16
}
@ -565,6 +574,22 @@ func (s *PageSlot) LeafPayload(page *Page) LeafPayload {
return LeafPayload{BaseOffset: s.PayloadOffset}
}
func (s *PageSlot) HashPayload(page *Page) HashPayload {
return HashPayload{BaseOffset: s.PayloadOffset}
}
type HashPagePayLoadReader struct {
KeyLength int16
KeyBytes []byte
PayloadChunkLength int16
PayloadChunkBytes []byte
}
func (pc *HashPagePayLoadReader) Length() int32 {
l := /*KeyLength*/ 2 + pc.KeyLength + /*PayLoadChunkLength*/ 2 + pc.PayloadChunkLength
return int32(l)
}
type LeafPagePayLoadReader struct {
KeyLength int16
KeyBytes []byte

124
extendiblehash/extendiblehash.go
View file

@ -2,6 +2,7 @@ package extendiblehash
import (
"bytes"
"errors"
"fmt"
"github.com/featurebasedb/featurebase/v3/bufferpool"
@ -18,7 +19,7 @@ type ExtendibleHashTable struct {
// NewExtendibleHashTable creates a new ExtendibleHashTable
func NewExtendibleHashTable(keyLength int, valueLength int, bufferPool *bufferpool.BufferPool) (*ExtendibleHashTable, error) {
bytesPerKV := keyLength + valueLength + bufferpool.PAGE_SLOT_LENGTH
bytesPerKV := /* keyLength */ 2 + keyLength + /* valueLength */ 2 + valueLength + bufferpool.PAGE_SLOT_LENGTH
keysPerPage := int(bufferpool.PAGE_SIZE-bufferpool.PAGE_SLOTS_START_OFFSET) / bytesPerKV
//create the root page
@ -54,8 +55,9 @@ func (e *ExtendibleHashTable) Get(key []byte) ([]byte, bool, error) {
index, found := e.findKey(page, key)
if found {
slot := page.ReadPageSlot(int16(index))
lpl := slot.LeafPayload(page)
return lpl.ValueAsBytes(page), true, nil
lpl := slot.HashPayload(page)
rdr := lpl.GetPayloadReader(page)
return rdr.PayloadChunkBytes, true, nil
}
return []byte{}, false, nil
}
@ -162,21 +164,21 @@ func (e *ExtendibleHashTable) splitOnKey(page *bufferpool.Page, key []byte) erro
if slot == nil {
break
}
pl := slot.KeyPayload(page)
lpl := slot.LeafPayload(page)
keyBytes := pl.KeyBytes(page)
lpl := slot.HashPayload(page)
rdr := lpl.GetPayloadReader(page)
keyBytes := rdr.KeyBytes
k := string(keyBytes)
h := Key(k).Hash()
if h&hiBit > 0 {
sc := p1.ReadSlotCount()
p1.PutKeyValueInPageSlot(sc, keyBytes, lpl.ValueAsBytes(page))
e.putKeyValueInPageSlot(p1, sc, keyBytes, rdr.PayloadChunkBytes)
// update the slot count
p1.WriteSlotCount(int16(sc + 1))
} else {
sc := p0.ReadSlotCount()
p0.PutKeyValueInPageSlot(sc, keyBytes, lpl.ValueAsBytes(page))
e.putKeyValueInPageSlot(p0, sc, keyBytes, rdr.PayloadChunkBytes)
// update the slot count
p0.WriteSlotCount(int16(sc + 1))
}
@ -196,44 +198,54 @@ func (e *ExtendibleHashTable) splitOnKey(page *bufferpool.Page, key []byte) erro
}
func (e *ExtendibleHashTable) cleanPage(page *bufferpool.Page) error {
scratch := e.bufferPool.ScratchPage()
// copy page number
scratch.WritePageNumber(page.ID().Page)
// set the page type
scratch.WritePageType(bufferpool.PAGE_TYPE_HASH_TABLE)
// copy local depth
scratch.WriteLocalDepth(page.ReadLocalDepth())
// 1. make a scratch page
// 2. iterate the slots on this page and copy data
// 3. put the scratch page back over this one
// copy slots from page to scratch
si := bufferpool.NewPageSlotIterator(page, 0)
for {
slot := si.Next()
if slot == nil {
break
}
pl := slot.KeyPayload(page)
lpl := slot.LeafPayload(page)
scratch.PutKeyValueInPageSlot(si.Cursor(), pl.KeyBytes(page), lpl.ValueAsBytes(page))
scratchPage := e.bufferPool.ScratchPage()
scratchPage.WritePageType(bufferpool.PAGE_TYPE_BTREE_LEAF)
scratchPage.WritePageNumber(page.ID().Page)
freeSpaceOffset := scratchPage.ReadFreeSpaceOffset()
slotCount := int(page.ReadSlotCount())
for i := 0; i < slotCount; i++ {
// read the slot from the page
s := page.ReadPageSlot(int16(i))
// read the chunk from the page
lbl := s.HashPayload(page)
c := lbl.GetPayloadReader(page)
// mod the freespace offset based on the size of the chunk
freeSpaceOffset -= int16(c.Length())
// update the values in the slot
s.PayloadOffset = freeSpaceOffset
// write the chunk
scratchPage.WriteHashPagePayloadBytes(freeSpaceOffset, c.KeyLength, c.KeyBytes, c.PayloadChunkLength, c.PayloadChunkBytes)
//write the slot
scratchPage.WritePageSlot(int16(i), s)
// update the free space offset
scratchPage.WriteFreeSpaceOffset(int16(freeSpaceOffset))
}
// set the new slotcounts
scratchPage.WriteSlotCount(int16(slotCount))
// update the slot count
scratch.WriteSlotCount(page.ReadSlotCount())
scratchPage.CopyPageTo(page)
// write scratch back to page
scratch.CopyPageTo(page)
return nil
}
func (e *ExtendibleHashTable) keyValueWillFit(page *bufferpool.Page, key, value []byte) bool {
// will this k/v fit on the page?
slotLen := 4 // we need 2 len words for the slot
chunkLen := 6 + len(key) + len(value) // int16 len + int32 len + len of respective []byte
payloadLen := page.ComputeHashPayloadTotalLength(len(key), len(value))
fs := page.FreeSpaceOnPage()
return fs > (int16(slotLen) + int16(chunkLen))
return fs > int16(payloadLen)
}
func (e *ExtendibleHashTable) putKeyValue(page *bufferpool.Page, key, value []byte) error {
if !e.keyValueWillFit(page, key, value) {
// try to garbage collect the page first
err := e.cleanPage(page)
@ -248,20 +260,17 @@ func (e *ExtendibleHashTable) putKeyValue(page *bufferpool.Page, key, value []by
slotCount := int(page.ReadSlotCount())
if found {
// we found the key, so we will update the value
err := page.PutKeyValueInPageSlot(int16(newIndex), []byte(key), []byte(value))
err := e.putKeyValueInPageSlot(page, int16(newIndex), []byte(key), []byte(value))
if err != nil {
return err
}
} else {
// TODO(pok) should check in WriteSlot() to see if we are out space too...
// TODO(pok) we should move all the slots in one fell swoop, because,... performance
// move all the slots after where we are going to insert
// move the slots over if needed
for j := slotCount; j > newIndex; j-- {
sl := page.ReadPageSlot(int16(j - 1))
page.WritePageSlot(int16(j), sl)
}
err := page.PutKeyValueInPageSlot(int16(newIndex), []byte(key), []byte(value))
err := e.putKeyValueInPageSlot(page, int16(newIndex), []byte(key), []byte(value))
if err != nil {
return err
}
@ -270,3 +279,40 @@ func (e *ExtendibleHashTable) putKeyValue(page *bufferpool.Page, key, value []by
}
return nil
}
func (e *ExtendibleHashTable) putKeyValueInPageSlot(page *bufferpool.Page, slotNumber int16, keyBytes []byte, payloadBytes []byte) error {
freeSpaceOffset := page.ReadFreeSpaceOffset()
keyLength := len(keyBytes)
payloadChunkLength := len(payloadBytes)
// check for overflow
if payloadChunkLength > bufferpool.MAX_PAYLOAD_CHUNK_SIZE_ON_PAGE {
return errors.New("overflow")
}
totalPayloadSize := page.ComputeHashPayloadTotalLength(keyLength, payloadChunkLength)
// check to make sure we don't blow free space
if page.FreeSpaceOnPage() < int16(totalPayloadSize) {
return errors.New("page is full")
}
// compute the new free space offset
freeSpaceOffset -= int16(totalPayloadSize)
// write the payload
page.WriteHashPagePayloadBytes(freeSpaceOffset, int16(keyLength), keyBytes, int16(payloadChunkLength), payloadBytes)
// update the free space offset
page.WriteFreeSpaceOffset(int16(freeSpaceOffset))
// make a slot
slot := bufferpool.PageSlot{
PayloadOffset: freeSpaceOffset,
}
// write the slot
page.WritePageSlot(slotNumber, slot)
return nil
}

36
extendiblehash/extendiblehash_test.go
View file

@ -114,7 +114,7 @@ func TestHashTable_Get(t *testing.T) {
key := "478"
value := "Hi"
page.PutKeyValueInPageSlot(0, []byte(key), []byte(value))
d.putKeyValueInPageSlot(page, 0, []byte(key), []byte(value))
page.WriteSlotCount(int16(1))
result, _, err := d.Get([]byte(key))
@ -150,7 +150,7 @@ func TestHashTable_Put(t *testing.T) {
t.Fatal(err)
}
defer d.bufferPool.UnpinPage(page.ID())
err = addToPage(page, 5)
err = addToPage(d, page, 5)
if err != nil {
t.Fatal(err)
}
@ -176,7 +176,7 @@ func TestHashTable_Put_ShouldIncreaseSize_WhenTableIsFull(t *testing.T) {
t.Fatal(err)
}
defer d.bufferPool.UnpinPage(page.ID())
err = addToPage(page, 227) // keys per page with key 12, value 20
err = addToPage(d, page, 227) // keys per page with key 12, value 20
if err != nil {
t.Fatal(err)
}
@ -203,7 +203,7 @@ func TestHashTable_PutShouldIncrementLD_WhenPageIsFull(t *testing.T) {
t.Fatal(err)
}
defer d.bufferPool.UnpinPage(page.ID())
err = addToPage(page, 227) // keys per page with key 12, value 20
err = addToPage(d, page, 227) // keys per page with key 12, value 20
if err != nil {
t.Fatal(err)
}
@ -241,6 +241,10 @@ func TestHashTable_Put_INT(t *testing.T) {
}
}
// for _, x := range d.directory {
// fmt.Printf("{ObjectID: 0, Shard: 0, Page: %d},\n", x.Page)
// }
assert.Equal(t, []bufferpool.PageID{
{ObjectID: 0, Shard: 0, Page: 0},
{ObjectID: 0, Shard: 0, Page: 1},
@ -250,26 +254,26 @@ func TestHashTable_Put_INT(t *testing.T) {
{ObjectID: 0, Shard: 0, Page: 7},
{ObjectID: 0, Shard: 0, Page: 6},
{ObjectID: 0, Shard: 0, Page: 5},
{ObjectID: 0, Shard: 0, Page: 13},
{ObjectID: 0, Shard: 0, Page: 12},
{ObjectID: 0, Shard: 0, Page: 15},
{ObjectID: 0, Shard: 0, Page: 11},
{ObjectID: 0, Shard: 0, Page: 13},
{ObjectID: 0, Shard: 0, Page: 9},
{ObjectID: 0, Shard: 0, Page: 8},
{ObjectID: 0, Shard: 0, Page: 14},
{ObjectID: 0, Shard: 0, Page: 11},
{ObjectID: 0, Shard: 0, Page: 10},
{ObjectID: 0, Shard: 0, Page: 12},
{ObjectID: 0, Shard: 0, Page: 28},
{ObjectID: 0, Shard: 0, Page: 25},
{ObjectID: 0, Shard: 0, Page: 21},
{ObjectID: 0, Shard: 0, Page: 18},
{ObjectID: 0, Shard: 0, Page: 4},
{ObjectID: 0, Shard: 0, Page: 24},
{ObjectID: 0, Shard: 0, Page: 22},
{ObjectID: 0, Shard: 0, Page: 19},
{ObjectID: 0, Shard: 0, Page: 30},
{ObjectID: 0, Shard: 0, Page: 20},
{ObjectID: 0, Shard: 0, Page: 29},
{ObjectID: 0, Shard: 0, Page: 19},
{ObjectID: 0, Shard: 0, Page: 18},
{ObjectID: 0, Shard: 0, Page: 27},
{ObjectID: 0, Shard: 0, Page: 24},
{ObjectID: 0, Shard: 0, Page: 25},
{ObjectID: 0, Shard: 0, Page: 21},
{ObjectID: 0, Shard: 0, Page: 23},
{ObjectID: 0, Shard: 0, Page: 22},
{ObjectID: 0, Shard: 0, Page: 17},
{ObjectID: 0, Shard: 0, Page: 14},
{ObjectID: 0, Shard: 0, Page: 16},
@ -347,11 +351,11 @@ func BenchmarkHashTable_Put_Many_Keys(b *testing.B) {
}
}
func addToPage(page *bufferpool.Page, numberOfRecords int) error {
func addToPage(table *ExtendibleHashTable, page *bufferpool.Page, numberOfRecords int) error {
for i := 0; i < numberOfRecords; i++ {
//fmt.Printf("writing record %d\n", i+1)
itoa := strconv.Itoa(i)
err := page.PutKeyValueInPageSlot(int16(i), []byte("key"+itoa), []byte("value foo bar"))
err := table.putKeyValueInPageSlot(page, int16(i), []byte("key"+itoa), []byte("value foo bar"))
if err != nil {
return err
}

12
tstore/btree.go
View file

@ -561,6 +561,7 @@ func (b *BTree) compactLeafPage(node *BTreeNode) error {
scratchPage := b.bufferpool.ScratchPage()
scratchPage.WritePageType(bufferpool.PAGE_TYPE_BTREE_LEAF)
scratchPage.WritePageNumber(node.page.ID().Page)
freeSpaceOffset := scratchPage.ReadFreeSpaceOffset()
slotCount := int(node.page.ReadSlotCount())
@ -608,6 +609,7 @@ func (b *BTree) compactInternalPage(node *BTreeNode) error {
scratchPage := b.bufferpool.ScratchPage()
scratchPage.WritePageType(bufferpool.PAGE_TYPE_BTREE_INTERNAL)
scratchPage.WritePageNumber(node.page.ID().Page)
freeSpaceOffset := scratchPage.ReadFreeSpaceOffset()
slotCount := int(node.page.ReadSlotCount())
@ -742,16 +744,16 @@ func (b *BTree) writeLeafEntryInSlot(node *BTreeNode, slotNumber int16, keyBytes
lowWater = hiWater
// set the payload chunk length to the free space on the page
// less the 2 byte chunk length
payloadChunkLength := overflowFreeSpace - 2
if payloadChunkLength > bytesRemaining {
payloadChunkLength = bytesRemaining
overflowChunkLength := overflowFreeSpace - 2
if overflowChunkLength > bytesRemaining {
overflowChunkLength = bytesRemaining
}
hiWater += payloadChunkLength
hiWater += overflowChunkLength
overflowPage.page.WriteNextPointer(bufferpool.PageID{ObjectID: b.objectID, Shard: b.shard, Page: nextOverflowPtr})
overflowPage.page.WriteLeafPagePayloadBytes(bufferpool.PAGE_SLOTS_START_OFFSET, int16(payloadChunkLength), payloadBytes[lowWater:hiWater])
bytesRemaining -= payloadChunkLength
bytesRemaining -= overflowChunkLength
overflowPage = nextOverflowPage
}

4
tstore/btree_test.go
View file

@ -137,8 +137,8 @@ func TestAddItemsToBTreeAndValidate_VeryWide(t *testing.T) {
for j, i := range inserts {
rr[0] = int64(i)
for j := 0; j < numCols; j++ {
rr[j+1] = fmt.Sprintf("%04d", j)
for jn := 0; jn < numCols; jn++ {
rr[jn+1] = fmt.Sprintf("%04d", jn)
}
tup := &BTreeTuple{