diff --git a/api.go b/api.go index a46d8da36..fb764b676 100644 --- a/api.go +++ b/api.go @@ -15,6 +15,7 @@ import ( "math" "net/url" "os" + "path/filepath" "runtime" "sort" "strconv" @@ -29,11 +30,13 @@ import ( "github.com/featurebasedb/featurebase/v3/disco" "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/rbf" + "github.com/featurebasedb/featurebase/v3/tstore" + "github.com/featurebasedb/featurebase/v3/wireprotocol" "github.com/prometheus/client_golang/prometheus" - //"github.com/featurebasedb/featurebase/v3/pg" "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" + "github.com/featurebasedb/featurebase/v3/sql3/parser" planner_types "github.com/featurebasedb/featurebase/v3/sql3/planner/types" "github.com/featurebasedb/featurebase/v3/tracing" "github.com/pkg/errors" @@ -263,9 +266,13 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index return nil, errors.Wrap(err, "validating api method") } + // get the next indexID number + indexID := int32(len(api.holder.indexes) + 1) + // Populate the create index message. ts := timestamp() cim := &CreateIndexMessage{ + IndexID: indexID, Index: indexName, CreatedAt: ts, Owner: requestUserID, @@ -1727,6 +1734,14 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard } } + // handle the tuple data + if len(req.Tuples) > 0 { + err := api.importTuples(ctx, tx, shard, indexName, req.Tuples) + if err != nil { + return err + } + } + if api.isComputeNode && !req.SuppressLog { partition := disco.ShardToShardPartition(indexName, shard, disco.DefaultPartitionN) msg := &computer.ImportRoaringShardMessage{ @@ -1734,6 +1749,7 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard Partition: partition, Shard: shard, Views: make([]computer.RoaringUpdate, len(req.Views)), + Tuples: req.Tuples, } for i, view := range req.Views { msg.Views[i] = computer.RoaringUpdate{ @@ -1766,6 +1782,122 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard return nil } +func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error { + // TODO wouldn't hurt to have consolidated logic somewhere for validating view names. + switch fieldType { + case FieldTypeSet, FieldTypeTime: + if viewUpdate.View == "" { + viewUpdate.View = "standard" + } + // add 'standard_' if we just have a time... this is how IDK works by default + if fieldType == FieldTypeTime && !strings.HasPrefix(viewUpdate.View, viewStandard) { + viewUpdate.View = fmt.Sprintf("%s_%s", viewStandard, viewUpdate.View) + } + case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp: + if viewUpdate.View == "" { + viewUpdate.View = "bsig_" + viewUpdate.Field + } else if viewUpdate.View != "bsig_"+viewUpdate.Field { + return NewBadRequestError(errors.Errorf("invalid view name (%s) for field %s of type %s", viewUpdate.View, viewUpdate.Field, fieldType)) + } + } + return nil +} + +func (api *API) importTuples(ctx context.Context, tx Tx, shard uint64, tableName string, tupleData []byte) error { + + // get the table + index, err := api.Index(ctx, tableName) + if err != nil { + return err + } + + // if the index id is 0 then we can't do an insert into the t-store + if index.ID == 0 { + return errors.Errorf("cannot insert into table '%s' because it does not have a non-zero object id", tableName) + } + basePath := index.TStorePath() + + // open or create btreefile for this shard + dataFile := filepath.Join(basePath, fmt.Sprintf("ts-shard.%04d", shard)) + api.holder.tstoredisk.CreateOrOpenShard(index.ID, int32(shard), dataFile) + + // get the schema from the FeatureBase table in the form of a planner_types.Schema + // we just want the t-store types + + fieldList := make([]*Field, 0) + for _, f := range index.fields { + if strings.EqualFold(f.options.Type, FieldTypeVarchar) { + fieldList = append(fieldList, f) + } + } + + sort.Slice(fieldList, func(i, j int) bool { + return fieldList[i].CreatedAt() < fieldList[j].CreatedAt() + }) + + indexSchema := make(planner_types.Schema, len(fieldList)) + for i, f := range fieldList { + indexSchema[i] = &planner_types.PlannerColumn{ + ColumnName: f.name, + Type: parser.NewDataTypeVarchar(f.options.Length), + } + } + + // create the b-tree we're going to use + b, err := tstore.NewBTree(tstore.KEY_SIZE_INT64, index.ID, int32(shard), indexSchema, api.holder.tstorepool) + if err != nil { + return err + } + + // start to read the data for the import + rdr := bytes.NewReader(tupleData) + _, err = wireprotocol.ExpectToken(rdr, wireprotocol.TOKEN_SCHEMA_INFO) + if err != nil { + return err + } + + // get the row schema from the import data + rowSchema, err := wireprotocol.ReadSchema(rdr) + if err != nil { + return err + } + + // read rows until we get to the end + tk, err := wireprotocol.ReadToken(rdr) + if err != nil { + return err + } + for tk == wireprotocol.TOKEN_ROW { + rr, err := wireprotocol.ReadRow(rdr, rowSchema) + if err != nil { + return err + } + + // make sure the key is at offset 0 + s := rowSchema[0] + if !strings.EqualFold(s.ColumnName, "_id") { + return errors.Errorf("unexpected key column position ") + } + + err = b.Insert(&tstore.BTreeTuple{ + TupleSchema: rowSchema, + Tuple: rr, + }) + if err != nil { + return err + } + + tk, err = wireprotocol.ReadToken(rdr) + if err != nil { + return err + } + } + if tk != wireprotocol.TOKEN_DONE { + return errors.Errorf("unexpected token '%d'", tk) + } + return nil +} + // ImportValue is a wrapper around the common code in ImportValueWithTx, which // currently just translates req.Clear into a clear ImportOption. func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error { diff --git a/batch/batch.go b/batch/batch.go index 0b5205a72..211c190d7 100644 --- a/batch/batch.go +++ b/batch/batch.go @@ -16,6 +16,9 @@ import ( "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" + "github.com/featurebasedb/featurebase/v3/wireprotocol" "github.com/pkg/errors" ) @@ -130,6 +133,9 @@ type Batch struct { // values holds the values for each record of an int field values map[string][]int64 + // values holds the values for each record of an varchar field + tupleValues map[string][]interface{} + // boolValues is a map[fieldName][idsIndex]bool, which holds the values for // each record of a bool field. It is a map of maps in order to accomodate // nil values (they just aren't recorded in the map[int]). @@ -184,6 +190,8 @@ type Batch struct { frags fragments clearFrags fragments + tuplefrags tuplefragments + useShardTransactionalEndpoint bool } @@ -257,6 +265,7 @@ func NewBatch(importer featurebase.Importer, size int, tbl *dax.Table, fields [] headerMap := make(map[string]*featurebase.FieldInfo, len(fields)) rowIDs := make(map[int][]uint64, len(fields)) values := make(map[string][]int64) + tupleValues := make(map[string][]interface{}) boolValues := make(map[string]map[int]bool) boolNulls := make(map[string][]uint64) tt := make(map[int]map[string][]int, len(fields)) @@ -293,6 +302,8 @@ func NewBatch(importer featurebase.Importer, size int, tbl *dax.Table, fields [] rowIDs[i] = make([]uint64, 0, size) case featurebase.FieldTypeBool: boolValues[field.Name] = make(map[int]bool) + case featurebase.FieldTypeVarchar: + tupleValues[field.Name] = make([]interface{}, 0, size) default: return nil, errors.Errorf("field type '%s' is not currently supported through Batch", typ) } @@ -309,6 +320,7 @@ func NewBatch(importer featurebase.Importer, size int, tbl *dax.Table, fields [] clearRowIDs: make(map[int]map[int]uint64), rowIDSets: make(map[string][][]uint64), values: values, + tupleValues: tupleValues, boolValues: boolValues, boolNulls: boolNulls, nullIndices: make(map[string][]uint64), @@ -325,6 +337,7 @@ func NewBatch(importer featurebase.Importer, size int, tbl *dax.Table, fields [] frags: make(fragments), clearFrags: make(fragments), + tuplefrags: make(tuplefragments), } if hasTime { b.times = make([]QuantizedTime, 0, size) @@ -544,6 +557,9 @@ func (b *Batch) Add(rec Row) error { case featurebase.FieldTypeBool: // If we want to support bools as string values, we would do // that here. + case featurebase.FieldTypeVarchar: + b.tupleValues[field.Name] = append(b.tupleValues[field.Name], val) + default: // nil-extend for len(b.rowIDs[i]) < curPos { @@ -647,6 +663,9 @@ func (b *Batch) Add(rec Row) error { boolNulls = append(boolNulls, uint64(curPos)) b.boolNulls[field.Name] = boolNulls + case featurebase.FieldTypeVarchar: + b.tupleValues[field.Name] = append(b.tupleValues[field.Name], nil) + default: // only append nil to rowIDs if this field already has // rowIDs. Otherwise, this could be a []string or @@ -780,15 +799,22 @@ func (b *Batch) Import() error { transTime := time.Now() b.log.Printf("translating batch of %d took: %v", size, transTime.Sub(transStart)) + var tuplefrags tuplefragments frags, clearFrags, err := b.makeFragments(b.frags, b.clearFrags) if err != nil { return errors.Wrap(err, "making fragments (flush)") } + if b.useShardTransactionalEndpoint { frags, clearFrags, err = b.makeSingleValFragments(frags, clearFrags) if err != nil { return errors.Wrap(err, "making single val fragments") } + + tuplefrags, err = b.makeTupleStoreFragments((b.tuplefrags)) + if err != nil { + return errors.Wrap(err, "making pvl fragments") + } } makeTime := time.Now() @@ -797,19 +823,24 @@ func (b *Batch) Import() error { if b.splitBatchMode { b.frags = frags b.clearFrags = clearFrags + b.tuplefrags = tuplefrags } else { b.frags = make(fragments) b.clearFrags = make(fragments) + b.tuplefrags = make(tuplefragments) // create bitmaps out of each field in b.rowIDs and import. Also // import int data. if !b.useShardTransactionalEndpoint { + if len(tuplefrags) > 0 { + return errors.Wrap(errors.Errorf("t-store values are not supported in non-transactional imports"), "doing import") + } err = b.doImport(frags, clearFrags) if err != nil { return errors.Wrap(err, "doing import") } b.log.Printf("importing fragments took %v", time.Since(makeTime)) } else { - err = b.doImportShardTransactional(frags, clearFrags) + err = b.doImportShardTransactional(frags, clearFrags, tuplefrags) if err != nil { return errors.Wrap(err, "doing shard transactional import") } @@ -1143,7 +1174,7 @@ func (b *Batch) createFieldKeys(field *featurebase.FieldInfo, keys ...string) (m return results, nil } -func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error { +func (b *Batch) doImportShardTransactional(frags, clearFrags fragments, tuplefrags tuplefragments) error { ctx := context.Background() start := time.Now() @@ -1200,6 +1231,68 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error { } } + // handle tuple frags + for fragKey, tupleFrag := range tuplefrags { + request := getOrCreate(requests, fragKey) + + // make a buffer for the tuple data + buf := new(bytes.Buffer) + + // get the bytes for the schema + b, err := wireprotocol.WriteSchema(tupleFrag.schema) + if err != nil { + return errors.Wrap(err, "serializing tuple schema") + } + _, err = buf.Write(b) + if err != nil { + return errors.Wrap(err, "serializing tuple schema") + } + + // now write each of the tuples + + // make a row, we're going to reuse this + rrow := make(types.Row, len(tupleFrag.schema)) + + // build a map of columnNames to column indexes in the schema + colMap := make(map[string]int) + for i, sc := range tupleFrag.schema { + colMap[sc.ColumnName] = i + } + + // iterate the tupleData - outside loop is rows + for id, trow := range tupleFrag.tupleData { + // write the _id column + rrow[0] = int64(id) + // now iterate the rest of the columns + for colName, tval := range trow { + j, ok := colMap[colName] + if !ok { + errors.Errorf("unexpected missing column '%s'", colName) + } + rrow[j] = tval + } + + // write the row to the buffer + rb, err := wireprotocol.WriteRow(rrow, tupleFrag.schema) + if err != nil { + return errors.Wrap(err, "serializing tuple row") + } + _, err = buf.Write(rb) + if err != nil { + return errors.Wrap(err, "serializing tuple row") + } + } + + // write done to the buffer + b = wireprotocol.WriteDone() + _, err = buf.Write(b) + if err != nil { + return errors.Wrap(err, "serializing tuples") + } + + request.Tuples = buf.Bytes() + } + featurebase.SummaryBatchShardImportBuildRequestsSeconds.Observe(time.Since(start).Seconds()) start = time.Now() eg := egpool.Group{PoolSize: 20} @@ -1695,6 +1788,69 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments, return frags, clearFrags, nil } +func (b *Batch) makeTupleStoreFragments(pvlfrags tuplefragments) (tuplefragments, error) { + shardWidth := b.shardWidth() + + // ------------------------- + // tuplestore fields + // ------------------------- + + // build a schema + tupleSchema := make(types.Schema, 0) + + var idType parser.ExprDataType = parser.NewDataTypeID() + if b.tbl.StringKeys() { + idType = parser.NewDataTypeString() + } + + tupleSchema = append(tupleSchema, &types.PlannerColumn{ + ColumnName: "_id", + RelationName: string(b.tbl.Name), + AliasName: "", + Type: idType, + }) + + for fieldname := range b.tupleValues { + field := b.headerMap[fieldname] + switch field.Options.Type { + case featurebase.FieldTypeVarchar: + tupleSchema = append(tupleSchema, &types.PlannerColumn{ + ColumnName: fieldname, + RelationName: string(b.tbl.Name), + AliasName: "", + Type: parser.NewDataTypeVarchar(field.Options.Length), + }) + default: + continue + } + } + + // for each of the varcharValues mapped, this is a column + for fieldname, varcharMap := range b.tupleValues { + field := b.headerMap[fieldname] + if field.Options.Type != featurebase.FieldTypeVarchar { + continue + } + + // for each of the row values for this column + for pos, varcharVal := range varcharMap { + recID := b.ids[pos] + + shard := recID / shardWidth + tf := pvlfrags.GetOrCreate(shard, tupleSchema) + + _, ok := tf.tupleData[recID] + if !ok { + tf.tupleData[recID] = make(map[string]interface{}) + } + tf.tupleData[recID][fieldname] = varcharVal + + } + } + + return pvlfrags, nil +} + type valsByIDsSortable struct { ids []uint64 vals []int64 @@ -1951,6 +2107,28 @@ func (b *Batch) reset() { } } +type tupleFragment struct { + // the schema of the fragment + schema types.Schema + // tupleData by id, by column name + tupleData map[uint64]map[string]interface{} +} + +// map[shard]tupleFragment +type tuplefragments map[uint64]*tupleFragment + +func (f tuplefragments) GetOrCreate(shard uint64, schema types.Schema) *tupleFragment { + tf, ok := f[shard] + if !ok { + tf = &tupleFragment{ + schema: schema, + tupleData: make(map[uint64]map[string]interface{}), + } + f[shard] = tf + } + return tf +} + // map[shard][field][view]fragmentData type fragments map[fragmentKey]map[string]*roaring.Bitmap diff --git a/bufferpool/bufferpool.go b/bufferpool/bufferpool.go index 5cd3f0f72..07c37e5de 100644 --- a/bufferpool/bufferpool.go +++ b/bufferpool/bufferpool.go @@ -1,6 +1,8 @@ package bufferpool import ( + "bytes" + "encoding/binary" "errors" "fmt" "sync" @@ -10,12 +12,30 @@ import ( type FrameID int // PageID is the type for page id -type PageID int +type PageID struct { + ObjectID int32 + Shard int32 + Page int64 +} + +func (p PageID) Bytes() []byte { + var valueBuf bytes.Buffer + v := make([]byte, 4) + binary.BigEndian.PutUint32(v, uint32(p.ObjectID)) + valueBuf.Write(v) + binary.BigEndian.PutUint32(v, uint32(p.Shard)) + valueBuf.Write(v) + vp := make([]byte, 8) + binary.BigEndian.PutUint64(vp, uint64(p.Page)) + valueBuf.Write(vp) + return valueBuf.Bytes() +} var pageSyncPool = sync.Pool{ New: func() any { pg := new(Page) - pg.id = PageID(INVALID_PAGE) + pg.latchState = None + pg.id = PageID{0, 0, INVALID_PAGE} pg.isDirty = false pg.pinCount = 0 return pg @@ -24,19 +44,26 @@ var pageSyncPool = sync.Pool{ // BufferPool represents a buffer pool of pages type BufferPool struct { + // the underlying storage diskManager DiskManager - // the actual pages in the buffer pool - pages []*Page - // the replacer that will elect replacements when buffer pool is full - replacer *ClockReplacer + + // the actual frames in the buffer pool + frames []*Page // the list of free frames freeList []FrameID + + framesMu sync.RWMutex + + // the replacer that will elect replacements when buffer pool is full + replacer *ClockReplacer + // the map of frames to page ids to frame ids - // frame ids are the offset into pages - // if you ask the pool for page 673, this will know at - // what offset in pages page 673 will exist - pageTable map[PageID]FrameID + // frame ids are the offset into frames (above) + // if you ask the pool for page 1:1:673, this will know at + // what offset in pages page 1:1:673 will exist (or not) + pageTable map[PageID]FrameID + pageTableMu sync.RWMutex } // TODO(pok) implement a lazy writer @@ -44,9 +71,6 @@ type BufferPool struct { // * increase size of cache if there is physical memory available // * write out old pages and boot them from the cache to increase free list -// TODO(pok) implement a checkpoint that scans the pool and writes out dirty pages every -// minute or so - // NewBufferPool returns a buffer pool func NewBufferPool(maxSize int, diskManager DiskManager) *BufferPool { freeList := make([]FrameID, 0) @@ -58,7 +82,7 @@ func NewBufferPool(maxSize int, diskManager DiskManager) *BufferPool { clockReplacer := NewClockReplacer(maxSize) return &BufferPool{ diskManager: diskManager, - pages: pages, + frames: pages, replacer: clockReplacer, freeList: freeList, pageTable: make(map[PageID]FrameID), @@ -70,7 +94,7 @@ func (b *BufferPool) Dump() { fmt.Println() fmt.Printf("------------------------------------------------------------------------------------------\n") fmt.Printf("BUFFER POOL\n") - for _, p := range b.pages { + for _, p := range b.frames { if p != nil { p.Dump("") } @@ -81,30 +105,40 @@ func (b *BufferPool) Dump() { // FetchPage fetches the requested page from the buffer pool. func (b *BufferPool) FetchPage(pageID PageID) (*Page, error) { + b.pageTableMu.Lock() + defer b.pageTableMu.Unlock() + + b.framesMu.RLock() // if it is in buffer pool already then just return it if frameID, ok := b.pageTable[pageID]; ok { - page := b.pages[frameID] + page := b.frames[frameID] page.pinCount++ b.replacer.Pin(frameID) + b.framesMu.RUnlock() return page, nil } + // we will need to write to frames, so unlock and take write lock + b.framesMu.RUnlock() + b.framesMu.Lock() + defer b.framesMu.Unlock() + // not in the buffer pool so try the free list or // the replacer will vote a page off the island frameID, isFromFreeList, err := b.getFrameID() if err != nil { + b.framesMu.RUnlock() return nil, err } if !isFromFreeList { // if it didn't come from the freelist then // remove page from current frame, writing it out if dirty - currentPage := b.pages[frameID] + currentPage := b.frames[frameID] if currentPage != nil { if currentPage.isDirty { b.diskManager.WritePage(currentPage) } - delete(b.pageTable, currentPage.id) } } @@ -116,8 +150,8 @@ func (b *BufferPool) FetchPage(pageID PageID) (*Page, error) { } page.pinCount = 1 b.pageTable[pageID] = frameID - pageSyncPool.Put(b.pages[frameID]) - b.pages[frameID] = page + pageSyncPool.Put(b.frames[frameID]) + b.frames[frameID] = page b.replacer.Pin(frameID) return page, nil @@ -125,8 +159,13 @@ func (b *BufferPool) FetchPage(pageID PageID) (*Page, error) { // UnpinPage unpins the target page from the buffer pool func (b *BufferPool) UnpinPage(pageID PageID) error { + b.framesMu.RLock() + b.pageTableMu.RLock() + defer b.framesMu.RUnlock() + defer b.pageTableMu.RUnlock() + if frameID, ok := b.pageTable[pageID]; ok { - page := b.pages[frameID] + page := b.frames[frameID] page.DecPinCount() if page.pinCount <= 0 { @@ -139,9 +178,15 @@ func (b *BufferPool) UnpinPage(pageID PageID) error { } // FlushPage Flushes the target page to disk +// This shold not be called during normal operation func (b *BufferPool) FlushPage(pageID PageID) bool { + b.framesMu.RLock() + b.pageTableMu.RLock() + defer b.framesMu.RUnlock() + defer b.pageTableMu.RUnlock() + if frameID, ok := b.pageTable[pageID]; ok { - page := b.pages[frameID] + page := b.frames[frameID] page.DecPinCount() b.diskManager.WritePage(page) @@ -153,7 +198,12 @@ func (b *BufferPool) FlushPage(pageID PageID) bool { } // NewPage allocates a new page in the buffer pool with the disk manager help -func (b *BufferPool) NewPage() (*Page, error) { +func (b *BufferPool) NewPage(fileID int32, shard int32) (*Page, error) { + b.framesMu.Lock() + b.pageTableMu.Lock() + defer b.framesMu.Unlock() + defer b.pageTableMu.Unlock() + // get a free frame frameID, isFromFreeList, err := b.getFrameID() if err != nil { @@ -162,7 +212,7 @@ func (b *BufferPool) NewPage() (*Page, error) { if !isFromFreeList { // remove page from current frame - currentPage := b.pages[frameID] + currentPage := b.frames[frameID] if currentPage != nil { if currentPage.isDirty { b.diskManager.WritePage(currentPage) @@ -173,20 +223,20 @@ func (b *BufferPool) NewPage() (*Page, error) { } // allocates new page - pageID, err := b.diskManager.AllocatePage() + pageID, err := b.diskManager.AllocatePage(fileID, shard) if err != nil { return nil, err } - page := &Page{pageID, 1, false, [PAGE_SIZE]byte{}} - page.WritePageNumber(int32(pageID)) + page := NewPage(pageID, 1) + page.WritePageNumber(pageID.Page) page.WriteFreeSpaceOffset(int16(PAGE_SIZE)) - page.WriteNextPointer(int32(INVALID_PAGE)) - page.WritePrevPointer(int32(INVALID_PAGE)) + page.WriteNextPointer(PageID{0, 0, INVALID_PAGE}) + page.WritePrevPointer(PageID{0, 0, INVALID_PAGE}) // update the frame table b.pageTable[pageID] = frameID - pageSyncPool.Put(b.pages[frameID]) - b.pages[frameID] = page + pageSyncPool.Put(b.frames[frameID]) + b.frames[frameID] = page return page, nil } @@ -197,27 +247,32 @@ func (b *BufferPool) NewPage() (*Page, error) { // and will copy the scratch page back over a real page later func (b *BufferPool) ScratchPage() *Page { page := &Page{ - id: PageID(INVALID_PAGE), + id: PageID{0, 0, INVALID_PAGE}, pinCount: 0, isDirty: false, data: [PAGE_SIZE]byte{}, } - page.WritePageNumber(int32(INVALID_PAGE)) + page.WritePageNumber(INVALID_PAGE) page.WriteFreeSpaceOffset(int16(PAGE_SIZE)) - page.WriteNextPointer(int32(INVALID_PAGE)) - page.WritePrevPointer(int32(INVALID_PAGE)) + page.WriteNextPointer(PageID{0, 0, INVALID_PAGE}) + page.WritePrevPointer(PageID{0, 0, INVALID_PAGE}) return page } // DeletePage deletes a page from the buffer pool func (b *BufferPool) DeletePage(pageID PageID) error { + b.framesMu.Lock() + b.pageTableMu.Lock() + defer b.framesMu.Unlock() + defer b.pageTableMu.Unlock() + var frameID FrameID var ok bool if frameID, ok = b.pageTable[pageID]; !ok { return nil } - page := b.pages[frameID] + page := b.frames[frameID] if page.pinCount > 0 { return errors.New("pin count greater than 0") @@ -231,14 +286,6 @@ func (b *BufferPool) DeletePage(pageID PageID) error { return nil } -// FlushAllpages flushes all the pages in the buffer pool to disk -// Yeah, never call this unless you know what you are doing -func (b *BufferPool) FlushAllpages() { - for pageID := range b.pageTable { - b.FlushPage(pageID) - } -} - func (b *BufferPool) getFrameID() (FrameID, bool, error) { if len(b.freeList) > 0 { frameID, newFreeList := b.freeList[0], b.freeList[1:] @@ -250,12 +297,6 @@ func (b *BufferPool) getFrameID() (FrameID, bool, error) { return victim, false, err } -// OnDiskSize exposes the on disk size of the backing store -// behind this buffer pool -func (b *BufferPool) OnDiskSize() int64 { - return b.diskManager.FileSize() -} - // Close closes the buffer pool func (b *BufferPool) Close() { b.diskManager.Close() diff --git a/bufferpool/diskmanager.go b/bufferpool/diskmanager.go index 6eebcdc10..27e7f9f17 100644 --- a/bufferpool/diskmanager.go +++ b/bufferpool/diskmanager.go @@ -4,17 +4,18 @@ package bufferpool type DiskManager interface { // reads a page from the disk ReadPage(PageID) (*Page, error) + // writes a page to the disk WritePage(*Page) error // allocates a page - AllocatePage() (PageID, error) + AllocatePage(objectID int32, shard int32) (PageID, error) // deallocates a page DeallocatePage(PageID) error // returns on disk file size - FileSize() int64 + FileSize(objectID int32, shard int32) int64 // closes and does any clean up Close() diff --git a/bufferpool/inmemdiskmanager.go b/bufferpool/inmemdiskmanager.go index 83b26d8a6..0b183e972 100644 --- a/bufferpool/inmemdiskmanager.go +++ b/bufferpool/inmemdiskmanager.go @@ -1,3 +1,4 @@ +// Copyright 2023 Molecula Corp. All rights reserved. package bufferpool import ( @@ -12,12 +13,12 @@ import ( // that can spill to disk when a threshold is reached type InMemDiskSpillingDiskManager struct { // tracks the number of pages - numPages int + numPages int64 - onDiskPages int + onDiskPages int64 // tracks the number of pages we can consume before spilling - thresholdPages int + thresholdPages int64 hasSpilled *struct{} fd *os.File @@ -26,7 +27,7 @@ type InMemDiskSpillingDiskManager struct { } // NewInMemDiskSpillingDiskManager returns a in-memory version of disk manager -func NewInMemDiskSpillingDiskManager(thresholdPages int) *InMemDiskSpillingDiskManager { +func NewInMemDiskSpillingDiskManager(thresholdPages int64) *InMemDiskSpillingDiskManager { dm := &InMemDiskSpillingDiskManager{ numPages: 0, thresholdPages: thresholdPages, @@ -38,11 +39,11 @@ func NewInMemDiskSpillingDiskManager(thresholdPages int) *InMemDiskSpillingDiskM // ReadPage reads a page from pages func (d *InMemDiskSpillingDiskManager) ReadPage(pageID PageID) (*Page, error) { // check we're not asking for page out of range - if pageID < 0 || int(pageID) >= d.numPages { + if pageID.Page < 0 || pageID.Page >= d.numPages { return nil, errors.New("page not found") } // check that the offset is within range - offset := int(pageID) * PAGE_SIZE + offset := pageID.Page * PAGE_SIZE var page = pageSyncPool.Get().(*Page) // we have to do this stupid check because if -cpuprofile is set for go test, this @@ -54,7 +55,7 @@ func (d *InMemDiskSpillingDiskManager) ReadPage(pageID PageID) (*Page, error) { // do the read if d.hasSpilled == nil { - if offset+PAGE_SIZE > len(d.data) { + if offset+PAGE_SIZE > int64(len(d.data)) { return nil, errors.New("offset out of range") } b := copy(page.data[:], d.data[offset:offset+PAGE_SIZE]) @@ -75,10 +76,10 @@ func (d *InMemDiskSpillingDiskManager) ReadPage(pageID PageID) (*Page, error) { // WritePage writes a page in memory to pages func (d *InMemDiskSpillingDiskManager) WritePage(page *Page) error { // make sure the offset is sensible - offset := int(page.ID()) * PAGE_SIZE + offset := page.ID().Page * PAGE_SIZE // do the write if d.hasSpilled == nil { - if offset+PAGE_SIZE > len(d.data) { + if offset+PAGE_SIZE > int64(len(d.data)) { return errors.New("offset out of range") } copy(d.data[offset:], page.data[:]) @@ -100,9 +101,9 @@ func (d *InMemDiskSpillingDiskManager) WritePage(page *Page) error { } // AllocatePage allocates a page and returns the page number -func (d *InMemDiskSpillingDiskManager) AllocatePage() (PageID, error) { +func (d *InMemDiskSpillingDiskManager) AllocatePage(objectID int32, shard int32) (PageID, error) { d.numPages = d.numPages + 1 - pageID := PageID(d.numPages - 1) + pageID := PageID{objectID, shard, int64(d.numPages - 1)} if d.hasSpilled == nil { // we have not spilled (yet), so make storage bigger @@ -113,16 +114,16 @@ func (d *InMemDiskSpillingDiskManager) AllocatePage() (PageID, error) { if d.numPages > d.thresholdPages { fileUUID, err := uuid.NewV4() if err != nil { - return PageID(INVALID_PAGE), err + return PageID{objectID, shard, INVALID_PAGE}, err } // TODO(pok) we should try to tell the OS not to cache this file d.fd, err = os.CreateTemp("", fmt.Sprintf("fb-ehash-%s", fileUUID.String())) if err != nil { - return PageID(INVALID_PAGE), err + return PageID{objectID, shard, INVALID_PAGE}, err } _, err = d.fd.WriteAt(d.data, 0) if err != nil { - return PageID(INVALID_PAGE), err + return PageID{objectID, shard, INVALID_PAGE}, err } d.data = []byte{} d.hasSpilled = &struct{}{} @@ -135,7 +136,7 @@ func (d *InMemDiskSpillingDiskManager) AllocatePage() (PageID, error) { size := int64(d.onDiskPages * PAGE_SIZE) _, err = d.fd.WriteAt([]byte{0}, size-1) if err != nil { - return PageID(INVALID_PAGE), err + return PageID{objectID, shard, INVALID_PAGE}, err } } } @@ -149,7 +150,7 @@ func (d *InMemDiskSpillingDiskManager) DeallocatePage(pageID PageID) error { return nil } -func (d *InMemDiskSpillingDiskManager) FileSize() int64 { +func (d *InMemDiskSpillingDiskManager) FileSize(fileID int32, shard int32) int64 { return int64(len(d.data)) } diff --git a/bufferpool/ondiskdiskmanager.go b/bufferpool/ondiskdiskmanager.go new file mode 100644 index 000000000..eb12e1dcd --- /dev/null +++ b/bufferpool/ondiskdiskmanager.go @@ -0,0 +1,267 @@ +// Copyright 2023 Molecula Corp. All rights reserved. +package bufferpool + +import ( + "fmt" + "os" + "sync" + + "github.com/pkg/errors" +) + +type FileShardID struct { + ObjectId int32 + Shard int32 +} + +// OnDiskDiskManager is a on disk implementation for a DiskManager interface +type OnDiskDiskManager struct { + mu sync.Mutex + + files map[FileShardID]*os.File +} + +// NewInMemDiskSpillingDiskManager returns a in-memory version of disk manager +func NewOnDiskDiskManager() *OnDiskDiskManager { + dm := &OnDiskDiskManager{ + files: make(map[FileShardID]*os.File), + } + return dm +} + +func (d *OnDiskDiskManager) CreateOrOpenShard(objectId int32, shard int32, dataFile string) error { + // serialize access here + d.mu.Lock() + defer d.mu.Unlock() + + fileID := FileShardID{objectId, shard} + + // do we have the file already open? + _, ok := d.files[fileID] + if ok { + return nil + } + + var fd *os.File + + // see if the file for this shard exists + _, err := os.Stat(dataFile) + if err != nil { + //create file + fd, err = os.OpenFile(dataFile, os.O_RDWR|os.O_CREATE, 0o600) + if err != nil { + return fmt.Errorf("open file: %w", err) + } + // write the root page + err = d.writeRootPage(fd, objectId, shard) + if err != nil { + return err + } + } else { + // open or create the file + fd, err = os.OpenFile(dataFile, os.O_RDWR, 0o600) + if err != nil { + return fmt.Errorf("open file: %w", err) + } + } + d.files[fileID] = fd + + return nil +} + +func (d *OnDiskDiskManager) writeRootPage(fd *os.File, objectId int32, shard int32) error { + headerPage := NewPage(PageID{objectId, shard, 0}, 0) + headerPage.WritePageNumber(0) + headerPage.WriteFreeSpaceOffset(int16(PAGE_SIZE)) + headerPage.WriteNextPointer(PageID{0, 0, INVALID_PAGE}) + headerPage.WritePrevPointer(PageID{0, 0, INVALID_PAGE}) + headerPage.WritePageType(PAGE_TYPE_BTREE_HEADER) + + // write a slot that points to the initial root page + rootPageID := PageID{objectId, shard, 1} + + // get the free space offset + freeSpaceOffset := headerPage.ReadFreeSpaceOffset() + + keyBytes := []byte{0, 0, 0, 0} + + // build a chunk + chunk := InternalPageChunk{ + KeyLength: int16(len(keyBytes)), + KeyBytes: keyBytes, + PtrValue: 1, + } + + // compute the new free space offset + freeSpaceOffset -= int16(chunk.Length()) + + headerPage.WriteInternalPageChunk(freeSpaceOffset, chunk) + + // update the free space offset + headerPage.WriteFreeSpaceOffset(int16(freeSpaceOffset)) + + // make a slot + slot := PageSlot{ + PayloadOffset: freeSpaceOffset, + } + // write the slot + headerPage.WritePageSlot(0, slot) + + // update the slot count + headerPage.WriteSlotCount(int16(1)) + + _, err := fd.WriteAt(headerPage.data[:], 0) + if err != nil { + return err + } + + rootPage := NewPage(rootPageID, 0) + rootPage.WritePageNumber(1) + rootPage.WriteFreeSpaceOffset(int16(PAGE_SIZE)) + rootPage.WriteNextPointer(PageID{0, 0, INVALID_PAGE}) + rootPage.WritePrevPointer(PageID{0, 0, INVALID_PAGE}) + rootPage.WritePageType(PAGE_TYPE_BTREE_LEAF) + + _, err = fd.WriteAt(rootPage.data[:], PAGE_SIZE) + if err != nil { + return err + } + + return nil +} + +// ReadPage reads a page from disk +func (d *OnDiskDiskManager) ReadPage(pageID PageID) (*Page, error) { + d.mu.Lock() + defer d.mu.Unlock() + + // do we have the file open + fileID := FileShardID{pageID.ObjectID, pageID.Shard} + + fd, ok := d.files[fileID] + if !ok { + return nil, errors.Errorf("file id '%d' not open", pageID.ObjectID) + } + + info, err := os.Stat(fd.Name()) + if err != nil { + return nil, err + } + + numPages := info.Size() / int64(PAGE_SIZE) + + // check we're not asking for page out of range + if pageID.Page < 0 || pageID.Page >= numPages { + return nil, errors.Errorf("requested page number '%d' out of range", pageID.Page) + } + offset := pageID.Page * int64(PAGE_SIZE) + + var page = pageSyncPool.Get().(*Page) + // we have to do this stupid check because if -cpuprofile is set for go test, this + // the previous line return a weird nil-ish thing... + if page == (*Page)(nil) { + page = pageSyncPool.New().(*Page) + } + page.id = pageID + + // do the read + _, err = fd.ReadAt(page.data[:], offset) + if err != nil { + return nil, err + } + return page, nil +} + +// WritePage writes a page in memory to pages +func (d *OnDiskDiskManager) WritePage(page *Page) error { + d.mu.Lock() + defer d.mu.Unlock() + + // do we have the file open + fileID := FileShardID{page.id.ObjectID, page.id.Shard} + + fd, ok := d.files[fileID] + if !ok { + return errors.Errorf("file id '%d' not open", page.id.ObjectID) + } + + info, err := os.Stat(fd.Name()) + if err != nil { + return err + } + + numPages := info.Size() / int64(PAGE_SIZE) + + // check we're not asking for page out of range + if page.id.Page < 0 || page.id.Page >= numPages { + return errors.Errorf("requested page number '%d' out of range", page.id.Page) + } + offset := page.id.Page * int64(PAGE_SIZE) + + // do the write + _, err = fd.WriteAt(page.data[:], offset) + if err != nil { + return err + } + // yeah, nah + // err = d.fd.Sync() + // if err != nil { + // return err + // } + return nil +} + +// AllocatePage allocates a page and returns the page number +func (d *OnDiskDiskManager) AllocatePage(objectId int32, shard int32) (PageID, error) { + d.mu.Lock() + defer d.mu.Unlock() + + // do we have the file open + fileID := FileShardID{objectId, shard} + + fd, ok := d.files[fileID] + if !ok { + return PageID{objectId, shard, INVALID_PAGE}, errors.Errorf("file id '%d' not open", objectId) + } + + info, err := os.Stat(fd.Name()) + if err != nil { + return PageID{objectId, shard, INVALID_PAGE}, err + } + + numPages := info.Size() / int64(PAGE_SIZE) + numPages = numPages + 1 + + pageID := PageID{objectId, shard, numPages - 1} + size := numPages * PAGE_SIZE + _, err = fd.WriteAt([]byte{0}, size-1) + if err != nil { + return PageID{objectId, shard, INVALID_PAGE}, err + } + return pageID, nil +} + +// DeallocatePage removes page from disk +func (d *OnDiskDiskManager) DeallocatePage(pageID PageID) error { + d.mu.Lock() + defer d.mu.Unlock() + + // nothing to do right now + return nil +} + +func (d *OnDiskDiskManager) FileSize(objectId int32, shard int32) int64 { + // TODO(pok) return correct file size + return 0 +} + +func (d *OnDiskDiskManager) Close() { + d.mu.Lock() + defer d.mu.Unlock() + + for _, fd := range d.files { + fd.Close() + } + // empty the efiles + d.files = make(map[FileShardID]*os.File, 0) +} diff --git a/bufferpool/page.go b/bufferpool/page.go index 4bd67b6c0..9535ebd78 100644 --- a/bufferpool/page.go +++ b/bufferpool/page.go @@ -4,17 +4,20 @@ import ( "encoding/binary" "errors" "fmt" + "sync" ) -const PAGE_SIZE int = 8192 +const PAGE_SIZE int64 = 8192 -const INVALID_PAGE int = -1 +const INVALID_PAGE int64 = -1 +const PAGE_TYPE_BTREE_OVERFLOW = 8 +const PAGE_TYPE_BTREE_HEADER = 9 const PAGE_TYPE_BTREE_INTERNAL = 10 const PAGE_TYPE_BTREE_LEAF = 11 const PAGE_TYPE_HASH_TABLE = 12 -// PAGE +// SLOTTED PAGE // page size 8192 bytes // byte aligned, big endian @@ -23,111 +26,146 @@ const PAGE_TYPE_HASH_TABLE = 12 // |----------------------------------------------------| // | header | // |====================================================| -// | 0 | 4 | pageNumber (int32) | -// | 4 | 2 | pageType (int16) | -// | 6 | 2 | slotCount (int16) | -// | 8 | 2 | localDepth (int16) | -// | 10 | 2 | freeSpaceOffset (int16) | -// | 12 | 4 | prevPointer (int32) | -// | 16 | 4 | nextPointer (int32) | +// | 0 | 8 | pageNumber (int64) | +// | 8 | 2 | pageType (int16) | +// | 10 | 2 | slotCount (int16) | +// | 12 | 2 | localDepth (int16) | // used only for hash tables +// | 14 | 2 | freeSpaceOffset (int16) | +// | 16 | 8 | prevPointer (int64) | +// | 24 | 8 | nextPointer (int64) | +// | 32 | 4 | checksum (int32) | +// | 36 | 16 | pageLSN (int64) | // |====================================================| // | | // |----------------------------------------------------| -// | 20 | slotcount | slot entry is 2 int16 | +// | 52 | slotcount | slot entry is int16 | // | | * slotwidth | values (payloadOffset, | // | | * #slots | payloadLength) | // |----------------------------------------------------| // | | // |----------------------------------------------------| // | | -// | payload entries are keylength (int16), key bytes, | -// | payload length (int32), payload bytes | +// | see below... | // |====================================================| -const PAGE_NUMBER_OFFSET = 0 // offset 0, length 4, end 4 -const PAGE_TYPE_OFFSET = 4 // offset 4, length 2, end 6 -const PAGE_SLOT_COUNT_OFFSET = 6 // offset 6, length 2, end 8 -const PAGE_LOCAL_DEPTH_OFFSET = 8 // offset 8, length 2, end 10 -const PAGE_FREE_SPACE_OFFSET = 10 // offset 10, length 2, end 12 -const PAGE_PREV_POINTER_OFFSET = 12 // offset 12, length 4, end 16 -const PAGE_NEXT_POINTER_OFFSET = 16 // offset 16, length 4, end 20 -const PAGE_SLOTS_START_OFFSET = 20 // offset 20 +// == internal payload chunk == +// keyLength (int16) +// keyBytes +// ptrValue (int64) (page number) -// PAGE_SLOT_LENGTH is the size of the page slot key/value. -// -// key offset int16 //offset 0, length 2, end 2 -// value offset int16 //offset 2, length 2, end 4 -const PAGE_SLOT_LENGTH = 4 +// == tuple payload chunk == +// keyLength (int16) +// keyBytes +// flags int8 +// overflowPtr (int64) [optional] +// rowPayloadTotalLen (int32) [optional] +// rowPayloadChunkLen (int16) +// rowPayloadChunkBytes -// Page represents a page on disk +// == row payload bytes == +// writeTID (int64) +// schemaVersion (int16) +// redoPtr (int64) +// fieldOffsets (one for each field, int16, FF is null) +// offsets point to: +// *fieldData (one for each field) +// valueLen (int32) only used for variable length types +// valueBytes + +const PAGE_NUMBER_OFFSET = 0 // offset 0, length 8, end 8 +const PAGE_TYPE_OFFSET = 8 // offset 8, length 2, end 10 +const PAGE_SLOT_COUNT_OFFSET = 10 // offset 10, length 2, end 12 +const PAGE_LOCAL_DEPTH_OFFSET = 12 // offset 12, length 2, end 14 +const PAGE_FREE_SPACE_OFFSET = 14 // offset 14, length 2, end 16 +const PAGE_PREV_POINTER_OFFSET = 16 // offset 16, length 8, end 24 +const PAGE_NEXT_POINTER_OFFSET = 24 // offset 24, length 8, end 32 +const PAGE_CHECKSUM = 32 // offset 32, length 4, end 36 +const PAGE_LSN = 36 // offset 36, length 16, end 52 +const PAGE_SLOTS_START_OFFSET = 52 // offset 52 + +// PAGE_SLOT_LENGTH is the size of the page slot offset value. +const PAGE_SLOT_LENGTH = 2 + +// 1k is the max key size for now +// we can make this bigger later with overflow +const MAX_KEY_ON_PAGE_SIZE = 1024 + +// 768 bytes is the max payload on a page before we overflow +// this gives us 1792 bytes for key and payload +// available space on a page after header is 8144 ish +// this gives us room for 4 key/value @ 2036 bytes per page, so +// there is a little bit of wiggle room +const MAX_PAYLOAD_CHUNK_SIZE_ON_PAGE = 768 + +type PageLatchState int + +const ( + None PageLatchState = iota + Read + Write +) + +// Page represents various types of data page on disk and in memory type Page struct { - id PageID - pinCount int - isDirty bool - data [PAGE_SIZE]byte + mu sync.RWMutex + latchState PageLatchState + id PageID + pinCount int + isDirty bool + data [PAGE_SIZE]byte } -type PageSlot struct { - KeyOffset int16 - ValueOffset int16 +func NewPage(pageID PageID, pinCount int) *Page { + return &Page{ + id: pageID, + latchState: None, + pinCount: pinCount, + isDirty: false, + data: [PAGE_SIZE]byte{}, + } } -func (s *PageSlot) KeyBytes(page *Page) []byte { - offset := s.KeyOffset - keyLen := int16(binary.BigEndian.Uint16(page.data[offset:])) - offset += 2 - result := make([]byte, keyLen) - copy(result, page.data[offset:offset+keyLen]) - return result +func (p *Page) TakeReadLatch() { + p.mu.RLock() + p.latchState = Read } -func (s *PageSlot) KeyAsInt(page *Page) int32 { - return int32(binary.BigEndian.Uint32(page.data[s.KeyOffset+2:])) +func (p *Page) ReleaseReadLatch() { + p.mu.RUnlock() + p.latchState = None } -func (s *PageSlot) ValueBytes(page *Page) []byte { - offset := s.ValueOffset - valueLen := int32(binary.BigEndian.Uint32(page.data[offset:])) - offset += 4 - result := make([]byte, valueLen) - copy(result, page.data[offset:int32(offset)+valueLen]) - return result +func (p *Page) TakeWriteLatch() { + p.mu.Lock() + p.latchState = Write } -func (s *PageSlot) ValueAsPagePointer(page *Page) int32 { - return int32(binary.BigEndian.Uint32(page.data[s.ValueOffset+4:])) +func (p *Page) ReleaseWriteLatch() { + p.mu.Unlock() + p.latchState = None } -type PageChunk struct { - KeyLength int16 - KeyBytes []byte - // TODO(pok) ValueBytes can be up to int32 long - // this requires an overflow page mechanism, that is not implemented - // yet, so be aware of this when storing stuff... - ValueLength int32 - ValueBytes []byte +func (p *Page) ReleaseAnyLatch() { + if p.latchState == Read { + p.ReleaseReadLatch() + } else if p.latchState == Write { + p.ReleaseWriteLatch() + } } -func (pc *PageChunk) Length() int { - return 2 + len(pc.KeyBytes) + 4 + len(pc.ValueBytes) +func (p *Page) LatchState() PageLatchState { + return p.latchState } -func (pc *PageChunk) ComputeKeyOffset(pageOffset int) int { - return pageOffset -} - -func (pc *PageChunk) ComputeValueOffset(pageOffset int) int { - return pageOffset + 2 + len(pc.KeyBytes) -} - -func (p *Page) WritePageNumber(pageNumber int32) { - p.id = PageID(pageNumber) - binary.BigEndian.PutUint32(p.data[PAGE_NUMBER_OFFSET:], uint32(pageNumber)) +// routines to read and write from page header information +func (p *Page) WritePageNumber(pageNumber int64) { + p.id.Page = pageNumber + binary.BigEndian.PutUint64(p.data[PAGE_NUMBER_OFFSET:], uint64(pageNumber)) p.isDirty = true } -func (p *Page) ReadPageNumber() int { - return int(binary.BigEndian.Uint32(p.data[PAGE_NUMBER_OFFSET:])) +func (p *Page) ReadPageNumber() int64 { + return int64(binary.BigEndian.Uint32(p.data[PAGE_NUMBER_OFFSET:])) } func (p *Page) WritePageType(pageType int16) { @@ -157,24 +195,6 @@ func (p *Page) ReadLocalDepth() int16 { return int16(binary.BigEndian.Uint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:])) } -func (p *Page) ReadSlot(slot int16) PageSlot { - offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot - keyOffset := int16(binary.BigEndian.Uint16(p.data[offset:])) - offset += 2 - valueOffset := int16(binary.BigEndian.Uint16(p.data[offset:])) - return PageSlot{ - KeyOffset: keyOffset, - ValueOffset: valueOffset, - } -} - -func (p *Page) WriteSlot(slot int16, value PageSlot) { - offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot - binary.BigEndian.PutUint16(p.data[offset:], uint16(value.KeyOffset)) - offset += 2 - binary.BigEndian.PutUint16(p.data[offset:], uint16(value.ValueOffset)) -} - func (p *Page) WriteFreeSpaceOffset(offset int16) { binary.BigEndian.PutUint16(p.data[PAGE_FREE_SPACE_OFFSET:], uint16(offset)) p.isDirty = true @@ -184,106 +204,152 @@ func (p *Page) ReadFreeSpaceOffset() int16 { return int16(binary.BigEndian.Uint16(p.data[PAGE_FREE_SPACE_OFFSET:])) } -func (p *Page) WritePrevPointer(prevPointer int32) { - binary.BigEndian.PutUint32(p.data[PAGE_PREV_POINTER_OFFSET:], uint32(prevPointer)) +func (p *Page) WritePrevPointer(prevPointer PageID) { + binary.BigEndian.PutUint64(p.data[PAGE_PREV_POINTER_OFFSET:], uint64(prevPointer.Page)) p.isDirty = true } -func (p *Page) ReadPrevPointer() int { - return int(binary.BigEndian.Uint32(p.data[PAGE_PREV_POINTER_OFFSET:])) +func (p *Page) ReadPrevPointer() PageID { + page := int64(binary.BigEndian.Uint64(p.data[PAGE_PREV_POINTER_OFFSET:])) + return PageID{ObjectID: p.id.ObjectID, Shard: p.id.Shard, Page: page} } -func (p *Page) WriteNextPointer(nextPointer int32) { - binary.BigEndian.PutUint32(p.data[PAGE_NEXT_POINTER_OFFSET:], uint32(nextPointer)) +func (p *Page) WriteNextPointer(nextPointer PageID) { + binary.BigEndian.PutUint64(p.data[PAGE_NEXT_POINTER_OFFSET:], uint64(nextPointer.Page)) p.isDirty = true } -func (p *Page) ReadNextPointer() int { - return int(binary.BigEndian.Uint32(p.data[PAGE_NEXT_POINTER_OFFSET:])) +func (p *Page) ReadNextPointer() PageID { + page := int64(binary.BigEndian.Uint64(p.data[PAGE_NEXT_POINTER_OFFSET:])) + return PageID{ObjectID: p.id.ObjectID, Shard: p.id.Shard, Page: page} } -func (p *Page) WriteChunk(offset int16, chunk PageChunk) { - binary.BigEndian.PutUint16(p.data[offset:], uint16(chunk.KeyLength)) +// read slots from pages +func (p *Page) ReadPageSlot(slot int16) PageSlot { + offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot + keyOffset := int16(binary.BigEndian.Uint16(p.data[offset:])) offset += 2 - copy(p.data[offset:], chunk.KeyBytes) - offset += int16(len(chunk.KeyBytes)) - binary.BigEndian.PutUint32(p.data[offset:], uint32(chunk.ValueLength)) - offset += 4 - copy(p.data[offset:], chunk.ValueBytes) - p.isDirty = true -} - -func (p *Page) ReadChunk(offset int16) PageChunk { - keyLen := int16(binary.BigEndian.Uint16(p.data[offset:])) - offset += 2 - keyBytes := make([]byte, keyLen) - copy(keyBytes, p.data[offset:offset+keyLen]) - offset += keyLen - valueLen := int32(binary.BigEndian.Uint32(p.data[offset:])) - offset += 4 - valueBytes := make([]byte, valueLen) - copy(valueBytes, p.data[offset:int32(offset)+valueLen]) - return PageChunk{ - KeyLength: keyLen, - KeyBytes: keyBytes, - ValueLength: valueLen, - ValueBytes: valueBytes, + return PageSlot{ + PayloadOffset: keyOffset, } } -func (p *Page) FreeSpace() int16 { - freeSpaceOffset := p.ReadFreeSpaceOffset() - freespace := freeSpaceOffset - (p.ReadSlotCount()*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET) - return freespace +func (p *Page) WritePageSlot(slot int16, value PageSlot) { + offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot + binary.BigEndian.PutUint16(p.data[offset:], uint16(value.PayloadOffset)) } -func (p *Page) WriteKeyValueInSlot(slotNumber int16, key []byte, value []byte) error { +// TODO(pok) deprecate this once we get b+tree working, this +// is just used in hash table right now +func (p *Page) PutKeyValueInPageSlot(slotNumber int16, keyBytes []byte, payloadBytes []byte) error { freeSpaceOffset := p.ReadFreeSpaceOffset() - // build a chunk - chunk := PageChunk{ - KeyLength: int16(len(key)), - KeyBytes: key, - ValueLength: int32(len(value)), - ValueBytes: value, + keyLength := len(keyBytes) + payloadChunkLength := len(payloadBytes) + + // check for overflow + if payloadChunkLength > MAX_PAYLOAD_CHUNK_SIZE_ON_PAGE { + return errors.New("overflow") } - // compute the new free space offset - freeSpaceOffset -= int16(chunk.Length()) + totalPayloadSize := p.ComputeLeafPayloadTotalLength(keyLength, payloadChunkLength) - // check we won't blow free space on page - slotCount := p.ReadSlotCount() - slotEndOffset := slotCount*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET - - // DEBUG!! - //fmt.Printf("freeSpaceOffset: %d, slotCount: %d, slotCount*4 + 4 + 20: %d, freeSpace: %d\n", freeSpaceOffset, slotCount, slotEndOffset, freeSpaceOffset-slotEndOffset) - - if freeSpaceOffset-slotEndOffset <= 0 { + // check to make sure we don't blow free space + if p.FreeSpaceOnPage() < int16(totalPayloadSize) { return errors.New("page is full") } - keyOffset := chunk.ComputeKeyOffset(int(freeSpaceOffset)) - valueOffset := chunk.ComputeValueOffset(int(freeSpaceOffset)) + // compute the new free space offset + freeSpaceOffset -= int16(totalPayloadSize) + offset := freeSpaceOffset - p.WriteChunk(freeSpaceOffset, chunk) + // write the header + offset = p.WriteLeafPagePayloadHeader(offset, int16(keyLength), keyBytes, 0, 0, int32(payloadChunkLength)) + + // write the payload + p.WriteLeafPagePayloadBytes(offset, int16(payloadChunkLength), payloadBytes) // update the free space offset p.WriteFreeSpaceOffset(int16(freeSpaceOffset)) // make a slot slot := PageSlot{ - KeyOffset: int16(keyOffset), - ValueOffset: int16(valueOffset), + PayloadOffset: freeSpaceOffset, } // write the slot - p.WriteSlot(slotNumber, slot) + p.WritePageSlot(slotNumber, slot) return nil } -func (p *Page) WritePage(page *Page) { +func (p *Page) ComputeInternalPayloadTotalLength(keyLength int) int32 { + l := /*keyLength*/ 2 + keyLength + /*ptrValue*/ 8 + return int32(l) +} + +func (p *Page) WriteInternalPagePayload(offset int16, keyLen int16, keyBytes []byte, ptrValue int64) { + binary.BigEndian.PutUint16(p.data[offset:], uint16(keyLen)) + offset += 2 + copy(p.data[offset:], keyBytes) + offset += keyLen + binary.BigEndian.PutUint64(p.data[offset:], uint64(ptrValue)) + p.isDirty = true +} + +func (p *Page) ComputeLeafPayloadTotalLength(keyLength int, payloadLength int) int32 { + l := /*keyLength*/ 2 + keyLength + /*flags*/ 1 + /*payLoadLength*/ 2 + payloadLength + if l > MAX_PAYLOAD_CHUNK_SIZE_ON_PAGE { + l += 8 + 4 // add overflow ptr and total payload size + } + return int32(l) +} + +func (p *Page) WriteLeafPagePayloadHeader(offset int16, keyLen int16, keyBytes []byte, flags int8, overflowPtr int64, payloadTotalLen int32) int16 { + binary.BigEndian.PutUint16(p.data[offset:], uint16(keyLen)) + offset += 2 + copy(p.data[offset:], keyBytes) + offset += keyLen + p.data[offset] = byte(flags) + offset += 1 + // if we are overflowing write the overflow ptr + if flags == 1 { + binary.BigEndian.PutUint64(p.data[offset:], uint64(overflowPtr)) + offset += 8 + // total length + binary.BigEndian.PutUint32(p.data[offset:], uint32(payloadTotalLen)) + offset += 4 + } + p.isDirty = true + return offset +} + +func (p *Page) WriteLeafPagePayloadBytes(offset int16, payloadChunkLength int16, payloadChunkBytes []byte) { + // chunk length + binary.BigEndian.PutUint16(p.data[offset:], uint16(payloadChunkLength)) + offset += 2 + // now copy the payload bytes + copy(p.data[offset:], payloadChunkBytes) + p.isDirty = true +} + +func (p *Page) WriteInternalPageChunk(offset int16, chunk InternalPageChunk) { + binary.BigEndian.PutUint16(p.data[offset:], uint16(chunk.KeyLength)) + offset += 2 + copy(p.data[offset:], chunk.KeyBytes) + offset += int16(len(chunk.KeyBytes)) + binary.BigEndian.PutUint64(p.data[offset:], uint64(chunk.PtrValue)) + p.isDirty = true +} + +func (p *Page) FreeSpaceOnPage() int16 { + freeSpaceOffset := p.ReadFreeSpaceOffset() + freespace := freeSpaceOffset - (p.ReadSlotCount()*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET) + return freespace +} + +func (p *Page) CopyPageTo(page *Page) { // copy everything but pageNumber & pageType - offset := PAGE_SLOT_COUNT_OFFSET + offset := int64(PAGE_SLOT_COUNT_OFFSET) copy(page.data[offset:], p.data[offset:offset+PAGE_SIZE-offset]) } @@ -301,6 +367,211 @@ func (p *Page) DecPinCount() { } } +func (pg *Page) Dump(label string) { + indent := 0 + if len(label) > 0 { + fmt.Printf("%s%s:\n", fmt.Sprintf("%*s", indent, ""), label) + indent += 4 + } + pageType := pg.ReadPageType() + fmt.Printf("%sPAGE(%d) pageType: %d slotCount: %d, prevPtr: %d, nextPtr: %d\n", fmt.Sprintf("%*s", indent, ""), pg.ID(), pageType, pg.ReadSlotCount(), pg.ReadPrevPointer(), pg.ReadNextPointer()) + fmt.Printf("%sKEYS: -->\n", fmt.Sprintf("%*s", indent, "")) + indent += 4 + + // get the keys off the page + keys := make([]int, 0) + pointers := make([]PageID, 0) + iter := NewPageSlotIterator(pg, 0) + for { + ps := iter.Next() + if ps == nil { + break + } + pl := ps.KeyPayload(pg) + keys = append(keys, int(pl.KeyAsInt(pg))) + if pageType == PAGE_TYPE_BTREE_INTERNAL { + ipl := ps.InternalPayload(pg) + pointers = append(pointers, ipl.ValueAsPagePointer(pg)) + } + } + + if pageType == PAGE_TYPE_BTREE_LEAF { + for _, key := range keys { + fmt.Printf("%s(%d)\n", fmt.Sprintf("%*s", indent, ""), key) + } + } else { + for idx, key := range keys { + ptr := pointers[idx] + fmt.Printf("%s(%d, %d)\n", fmt.Sprintf("%*s", indent, ""), key, ptr) + } + ptr := pg.ReadNextPointer() + fmt.Printf("%s(-->, %d)\n", fmt.Sprintf("%*s", indent, ""), ptr) + } + +} + +type KeyPayload struct { + BaseOffset int16 +} + +func (p *KeyPayload) KeyAsInt(page *Page) int32 { + return int32(binary.BigEndian.Uint32(page.data[p.BaseOffset+2:])) +} + +func (p *KeyPayload) KeyBytes(page *Page) []byte { + offset := p.BaseOffset + keyLen := int16(binary.BigEndian.Uint16(page.data[offset:])) + offset += 2 + result := make([]byte, keyLen) + copy(result, page.data[offset:offset+keyLen]) + return result +} + +type InternalPayload struct { + BaseOffset int16 +} + +func (p *InternalPayload) ptrOffset(page *Page) int16 { + offset := p.BaseOffset + keyLen := int16(binary.BigEndian.Uint16(page.data[offset:])) + offset += 2 + keyLen + return offset +} + +func (p *InternalPayload) ValueAsPagePointer(page *Page) PageID { + offset := p.ptrOffset(page) + pageid := int64(binary.BigEndian.Uint64(page.data[offset:])) + return PageID{ObjectID: page.id.ObjectID, Shard: page.id.Shard, Page: pageid} +} + +func (p *InternalPayload) PutPagePointer(page *Page, pagePtr PageID) error { + offset := p.ptrOffset(page) + binary.BigEndian.PutUint64(page.data[offset:], uint64(pagePtr.Page)) + page.isDirty = true + return nil +} + +func (l *InternalPayload) InternalPageChunk(page *Page) InternalPageChunk { + offset := l.BaseOffset + keyLen := int16(binary.BigEndian.Uint16(page.data[offset:])) + offset += 2 + keyBytes := make([]byte, keyLen) + copy(keyBytes, page.data[offset:offset+keyLen]) + offset += keyLen + ptrValue := int64(binary.BigEndian.Uint64(page.data[offset:])) + return InternalPageChunk{ + KeyLength: keyLen, + KeyBytes: keyBytes, + PtrValue: ptrValue, + } +} + +type LeafPayload struct { + BaseOffset int16 +} + +func (l *LeafPayload) valueOffset(page *Page) int16 { + offset := l.BaseOffset + keyLen := int16(binary.BigEndian.Uint16(page.data[offset:])) + offset += 2 + keyLen + return offset +} + +func (l *LeafPayload) ValueLength(page *Page) int32 { + offset := l.valueOffset(page) + valueLen := int32(binary.BigEndian.Uint32(page.data[offset:])) + return valueLen +} + +// this wil fail in overflow +func (l *LeafPayload) ValueAsBytes(page *Page) []byte { + offset := l.valueOffset(page) + valueLen := int32(binary.BigEndian.Uint32(page.data[offset:])) + offset += 4 + result := make([]byte, valueLen) + copy(result, page.data[offset:int32(offset)+valueLen]) + return result +} + +func (l *LeafPayload) GetPayloadReader(page *Page) LeafPagePayLoadReader { + offset := l.BaseOffset + keyLen := int16(binary.BigEndian.Uint16(page.data[offset:])) + offset += 2 + keyBytes := make([]byte, keyLen) + copy(keyBytes, page.data[offset:offset+keyLen]) + offset += keyLen + flags := int8(page.data[offset]) + offset += 1 + overflowPtr := int64(0) + payloadTotalLength := int32(0) + if flags == 1 { + overflowPtr = int64(binary.BigEndian.Uint64(page.data[offset:])) + offset += 8 + payloadTotalLength = int32(binary.BigEndian.Uint32(page.data[offset:])) + offset += 4 + } + valueLen := int16(binary.BigEndian.Uint16(page.data[offset:])) + offset += 2 + if payloadTotalLength == 0 { + payloadTotalLength = int32(valueLen) + } + valueBytes := make([]byte, valueLen) + copy(valueBytes, page.data[offset:int16(offset)+valueLen]) + return LeafPagePayLoadReader{ + KeyLength: keyLen, + KeyBytes: keyBytes, + Flags: flags, + OverflowPtr: overflowPtr, + PayloadTotalLength: payloadTotalLength, + PayloadChunkLength: valueLen, + PayloadChunkBytes: valueBytes, + } +} + +type PageSlot struct { + PayloadOffset int16 +} + +func (s *PageSlot) KeyPayload(page *Page) KeyPayload { + return KeyPayload{BaseOffset: s.PayloadOffset} +} + +func (s *PageSlot) InternalPayload(page *Page) InternalPayload { + return InternalPayload{BaseOffset: s.PayloadOffset} +} + +func (s *PageSlot) LeafPayload(page *Page) LeafPayload { + return LeafPayload{BaseOffset: s.PayloadOffset} +} + +type LeafPagePayLoadReader struct { + KeyLength int16 + KeyBytes []byte + Flags int8 + OverflowPtr int64 + PayloadTotalLength int32 + PayloadChunkLength int16 + PayloadChunkBytes []byte +} + +func (pc *LeafPagePayLoadReader) Length() int32 { + l := /*keyLength*/ 2 + pc.KeyLength + /*flags*/ 1 + /*payLoadLength*/ 2 + pc.PayloadChunkLength + if pc.Flags == 1 { + l += 8 + 4 // add overflow ptr and total payload size + } + return int32(l) +} + +type InternalPageChunk struct { + KeyLength int16 + KeyBytes []byte + PtrValue int64 +} + +func (pc *InternalPageChunk) Length() int { + return 2 + len(pc.KeyBytes) + 8 +} + type PageSlotIterator struct { page *Page slotCount int16 @@ -318,7 +589,7 @@ func NewPageSlotIterator(page *Page, fromSlot int16) *PageSlotIterator { func (i *PageSlotIterator) Next() *PageSlot { if i.cursor < i.slotCount { - s := i.page.ReadSlot(i.cursor) + s := i.page.ReadPageSlot(i.cursor) i.cursor++ return &s } @@ -328,44 +599,3 @@ func (i *PageSlotIterator) Next() *PageSlot { func (i *PageSlotIterator) Cursor() int16 { return i.cursor } - -func (pg *Page) Dump(label string) { - indent := 0 - if len(label) > 0 { - fmt.Printf("%s%s:\n", fmt.Sprintf("%*s", indent, ""), label) - indent += 4 - } - pageType := pg.ReadPageType() - fmt.Printf("%sPAGE(%d) pageType: %d slotCount: %d, prevPtr: %d, nextPtr: %d\n", fmt.Sprintf("%*s", indent, ""), pg.ID(), pageType, pg.ReadSlotCount(), pg.ReadPrevPointer(), pg.ReadNextPointer()) - fmt.Printf("%sKEYS: -->\n", fmt.Sprintf("%*s", indent, "")) - indent += 4 - - // get the keys off the page - keys := make([]int, 0) - pointers := make([]int, 0) - iter := NewPageSlotIterator(pg, 0) - for { - ps := iter.Next() - if ps == nil { - break - } - keys = append(keys, int(ps.KeyAsInt(pg))) - if pageType == /*nodeTypeInternal*/ 10 { - pointers = append(pointers, int(ps.ValueAsPagePointer(pg))) - } - } - - if pageType == /*nodeTypeLeaf*/ 11 { - for _, key := range keys { - fmt.Printf("%s(%d)\n", fmt.Sprintf("%*s", indent, ""), key) - } - } else { - for idx, key := range keys { - ptr := pointers[idx] - fmt.Printf("%s(%d, %d)\n", fmt.Sprintf("%*s", indent, ""), key, ptr) - } - ptr := pg.ReadNextPointer() - fmt.Printf("%s(-->, %d)\n", fmt.Sprintf("%*s", indent, ""), ptr) - } - -} diff --git a/cluster.go b/cluster.go index 2f915c010..a8b46778a 100644 --- a/cluster.go +++ b/cluster.go @@ -914,6 +914,7 @@ type CreateShardMessage struct { // CreateIndexMessage is an internal message indicating index creation. type CreateIndexMessage struct { + IndexID int32 Index string CreatedAt int64 Owner string diff --git a/dax/computer/logmessage.go b/dax/computer/logmessage.go index 3d7d9e5af..fee0dbf86 100644 --- a/dax/computer/logmessage.go +++ b/dax/computer/logmessage.go @@ -231,6 +231,7 @@ type ImportRoaringShardMessage struct { Partition int `json:"partition"` Shard uint64 `json:"shard"` Views []RoaringUpdate `json:"views"` + Tuples []byte `json:"tuples"` } // RoaringUpdate is identical to featurebase.RoaringUpdate, but we diff --git a/dax/table.go b/dax/table.go index 2f90cdaff..8cbd5c08f 100644 --- a/dax/table.go +++ b/dax/table.go @@ -83,6 +83,7 @@ const ( BaseTypeStringSet = "stringset" // keyed set BaseTypeStringSetQ = "stringsetq" // keyed set timequantum BaseTypeTimestamp = "timestamp" // + BaseTypeVarchar = "varchar" // DefaultPartitionN = 256 @@ -684,7 +685,8 @@ func BaseTypeFromString(s string) (BaseType, error) { BaseTypeString, BaseTypeStringSet, BaseTypeStringSetQ, - BaseTypeTimestamp: + BaseTypeTimestamp, + BaseTypeVarchar: return BaseType(lowered), nil default: return "", errors.Errorf("invalid field type: %s", s) @@ -824,4 +826,5 @@ type FieldOptions struct { TTL time.Duration `json:"ttl,omitempty"` ForeignIndex string `json:"foreign-index,omitempty"` TrackExistence bool `json:"track-existence"` + Length int64 `json:"length,omitempty"` } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index b1fce6e41..7ab5742d9 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -589,6 +589,7 @@ func (s Serializer) encodeIndexInfos(idxs []*pilosa.IndexInfo) []*pb.Index { func (s Serializer) encodeIndexInfo(idx *pilosa.IndexInfo) *pb.Index { return &pb.Index{ + IndexID: idx.ID, Name: idx.Name, CreatedAt: idx.CreatedAt, Options: s.encodeIndexMeta(&idx.Options), @@ -638,6 +639,7 @@ func (s Serializer) encodeFieldOptions(o *pilosa.FieldOptions) *pb.FieldOptions ForeignIndex: o.ForeignIndex, NoStandardView: o.NoStandardView, TrackExistence: o.TrackExistence, + Length: o.Length, } } @@ -688,6 +690,7 @@ func (s Serializer) encodeCreateShardMessage(m *pilosa.CreateShardMessage) *pb.C func (s Serializer) encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *pb.CreateIndexMessage { return &pb.CreateIndexMessage{ + IndexID: m.IndexID, Index: m.Index, CreatedAt: m.CreatedAt, Owner: m.Owner, @@ -929,6 +932,7 @@ func (s Serializer) decodeIndexes(idxs []*pb.Index, m []*pilosa.IndexInfo) { } func (s Serializer) decodeIndex(idx *pb.Index, m *pilosa.IndexInfo) { + m.ID = idx.IndexID m.Name = idx.Name m.CreatedAt = idx.CreatedAt m.Options = pilosa.IndexOptions{} @@ -963,6 +967,7 @@ func (s Serializer) decodeFieldOptions(options *pb.FieldOptions, m *pilosa.Field s.decodeDecimal(options.Max, &m.Max) m.Base = options.Base m.Scale = options.Scale + m.Length = options.Length m.BitDepth = uint64(options.BitDepth) m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum) ttlValue, err := time.ParseDuration(options.TTL) @@ -1032,6 +1037,7 @@ func (s Serializer) decodeCreateShardMessage(pb *pb.CreateShardMessage, m *pilos } func (s Serializer) decodeCreateIndexMessage(pb *pb.CreateIndexMessage, m *pilosa.CreateIndexMessage) { + m.IndexID = pb.IndexID m.Index = pb.Index m.CreatedAt = pb.CreatedAt m.Owner = pb.Owner diff --git a/extendiblehash/extendiblehash.go b/extendiblehash/extendiblehash.go index ff18c97b1..8b354c782 100644 --- a/extendiblehash/extendiblehash.go +++ b/extendiblehash/extendiblehash.go @@ -19,10 +19,10 @@ type ExtendibleHashTable struct { // NewExtendibleHashTable creates a new ExtendibleHashTable func NewExtendibleHashTable(keyLength int, valueLength int, bufferPool *bufferpool.BufferPool) (*ExtendibleHashTable, error) { bytesPerKV := keyLength + valueLength + bufferpool.PAGE_SLOT_LENGTH - keysPerPage := (bufferpool.PAGE_SIZE - bufferpool.PAGE_SLOTS_START_OFFSET) / bytesPerKV + keysPerPage := int(bufferpool.PAGE_SIZE-bufferpool.PAGE_SLOTS_START_OFFSET) / bytesPerKV //create the root page - page, err := bufferPool.NewPage() + page, err := bufferPool.NewPage(0, 0) if err != nil { return nil, err } @@ -53,8 +53,9 @@ func (e *ExtendibleHashTable) Get(key []byte) ([]byte, bool, error) { index, found := e.findKey(page, key) if found { - slot := page.ReadSlot(int16(index)) - return slot.ValueBytes(page), true, nil + slot := page.ReadPageSlot(int16(index)) + lpl := slot.LeafPayload(page) + return lpl.ValueAsBytes(page), true, nil } return []byte{}, false, nil } @@ -99,7 +100,7 @@ func (e *ExtendibleHashTable) hashFunction(k Hashable) int { func (e *ExtendibleHashTable) getPageID(key []byte) (bufferpool.PageID, error) { hash := e.hashFunction(Key(key)) if hash > len(e.directory)-1 { - return 0, fmt.Errorf("hash (%d) out of the directory array bounds (%d)", hash, len(e.directory)) + return bufferpool.PageID{ObjectID: 0, Shard: 0, Page: bufferpool.INVALID_PAGE}, fmt.Errorf("hash (%d) out of the directory array bounds (%d)", hash, len(e.directory)) } id := e.directory[hash] return bufferpool.PageID(id), nil @@ -111,8 +112,9 @@ func (e *ExtendibleHashTable) findKey(page *bufferpool.Page, key []byte) (int, b for onePastMaxIndex != minIndex { index := (minIndex + onePastMaxIndex) / 2 - s := page.ReadSlot(int16(index)) - keyAtIndex := s.KeyBytes(page) + s := page.ReadPageSlot(int16(index)) + pl := s.KeyPayload(page) + keyAtIndex := pl.KeyBytes(page) if bytes.Equal(keyAtIndex, key) { return index, true @@ -135,11 +137,11 @@ func (e *ExtendibleHashTable) splitOnKey(page *bufferpool.Page, key []byte) erro // scratch page for left p0 := e.bufferPool.ScratchPage() - p0.WritePageNumber(int32(page.ID())) + p0.WritePageNumber(page.ID().Page) p0.WritePageType(bufferpool.PAGE_TYPE_HASH_TABLE) // allocate new page for split - p1, err := e.bufferPool.NewPage() + p1, err := e.bufferPool.NewPage(0, 0) if err != nil { return err } @@ -160,19 +162,21 @@ func (e *ExtendibleHashTable) splitOnKey(page *bufferpool.Page, key []byte) erro if slot == nil { break } - keyBytes := slot.KeyBytes(page) + pl := slot.KeyPayload(page) + lpl := slot.LeafPayload(page) + keyBytes := pl.KeyBytes(page) k := string(keyBytes) h := Key(k).Hash() if h&hiBit > 0 { sc := p1.ReadSlotCount() - p1.WriteKeyValueInSlot(sc, keyBytes, slot.ValueBytes(page)) + p1.PutKeyValueInPageSlot(sc, keyBytes, lpl.ValueAsBytes(page)) // update the slot count p1.WriteSlotCount(int16(sc + 1)) } else { sc := p0.ReadSlotCount() - p0.WriteKeyValueInSlot(sc, keyBytes, slot.ValueBytes(page)) + p0.PutKeyValueInPageSlot(sc, keyBytes, lpl.ValueAsBytes(page)) // update the slot count p0.WriteSlotCount(int16(sc + 1)) } @@ -186,7 +190,7 @@ func (e *ExtendibleHashTable) splitOnKey(page *bufferpool.Page, key []byte) erro } // copy p1 back into page - p0.WritePage(page) + p0.CopyPageTo(page) return nil } @@ -194,7 +198,7 @@ func (e *ExtendibleHashTable) splitOnKey(page *bufferpool.Page, key []byte) erro func (e *ExtendibleHashTable) cleanPage(page *bufferpool.Page) error { scratch := e.bufferPool.ScratchPage() // copy page number - scratch.WritePageNumber(int32(page.ID())) + scratch.WritePageNumber(page.ID().Page) // set the page type scratch.WritePageType(bufferpool.PAGE_TYPE_HASH_TABLE) // copy local depth @@ -207,14 +211,16 @@ func (e *ExtendibleHashTable) cleanPage(page *bufferpool.Page) error { if slot == nil { break } - scratch.WriteKeyValueInSlot(si.Cursor(), slot.KeyBytes(page), slot.ValueBytes(page)) + pl := slot.KeyPayload(page) + lpl := slot.LeafPayload(page) + scratch.PutKeyValueInPageSlot(si.Cursor(), pl.KeyBytes(page), lpl.ValueAsBytes(page)) } // update the slot count scratch.WriteSlotCount(page.ReadSlotCount()) // write scratch back to page - scratch.WritePage(page) + scratch.CopyPageTo(page) return nil } @@ -222,7 +228,7 @@ func (e *ExtendibleHashTable) keyValueWillFit(page *bufferpool.Page, key, value // will this k/v fit on the page? slotLen := 4 // we need 2 len words for the slot chunkLen := 6 + len(key) + len(value) // int16 len + int32 len + len of respective []byte - fs := page.FreeSpace() + fs := page.FreeSpaceOnPage() return fs > (int16(slotLen) + int16(chunkLen)) } @@ -242,7 +248,7 @@ func (e *ExtendibleHashTable) putKeyValue(page *bufferpool.Page, key, value []by slotCount := int(page.ReadSlotCount()) if found { // we found the key, so we will update the value - err := page.WriteKeyValueInSlot(int16(newIndex), []byte(key), []byte(value)) + err := page.PutKeyValueInPageSlot(int16(newIndex), []byte(key), []byte(value)) if err != nil { return err } @@ -252,10 +258,10 @@ func (e *ExtendibleHashTable) putKeyValue(page *bufferpool.Page, key, value []by // TODO(pok) we should move all the slots in one fell swoop, because,... performance // move all the slots after where we are going to insert for j := slotCount; j > newIndex; j-- { - sl := page.ReadSlot(int16(j - 1)) - page.WriteSlot(int16(j), sl) + sl := page.ReadPageSlot(int16(j - 1)) + page.WritePageSlot(int16(j), sl) } - err := page.WriteKeyValueInSlot(int16(newIndex), []byte(key), []byte(value)) + err := page.PutKeyValueInPageSlot(int16(newIndex), []byte(key), []byte(value)) if err != nil { return err } diff --git a/extendiblehash/extendiblehash_test.go b/extendiblehash/extendiblehash_test.go index 98d412e41..d1535a4a4 100644 --- a/extendiblehash/extendiblehash_test.go +++ b/extendiblehash/extendiblehash_test.go @@ -8,18 +8,19 @@ import ( "github.com/stretchr/testify/assert" ) -func makeDirectory() (*ExtendibleHashTable, error) { +func makeDirectory() (*ExtendibleHashTable, bufferpool.DiskManager, error) { diskManager := bufferpool.NewInMemDiskSpillingDiskManager(128) bufferPool := bufferpool.NewBufferPool(128, diskManager) keySize := 12 valueSize := 20 - return NewExtendibleHashTable(keySize, valueSize, bufferPool) + eht, err := NewExtendibleHashTable(keySize, valueSize, bufferPool) + return eht, diskManager, err } func TestHashTable_ExtendibleHash(t *testing.T) { - d, err := makeDirectory() + d, _, err := makeDirectory() if err != nil { t.Fatal(err) } @@ -36,7 +37,7 @@ func TestHashTable_ExtendibleHash(t *testing.T) { } func TestHashTable_GetPage(t *testing.T) { - d, err := makeDirectory() + d, _, err := makeDirectory() if err != nil { t.Fatal(err) } @@ -44,18 +45,18 @@ func TestHashTable_GetPage(t *testing.T) { d.directory = make([]bufferpool.PageID, 16) key := "478" - d.directory[14] = 2 + d.directory[14] = bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 2} pageID, err := d.getPageID([]byte(key)) if err != nil { t.Fatal(err) } - assert.Equal(t, 2, int(pageID)) + assert.Equal(t, 2, int(pageID.Page)) } func TestHashTable_GetPage_ShouldReturnError_WhenOffsetIsNotLimitedToDataSize(t *testing.T) { - d, err := makeDirectory() + d, _, err := makeDirectory() if err != nil { t.Fatal(err) } @@ -67,7 +68,7 @@ func TestHashTable_GetPage_ShouldReturnError_WhenOffsetIsNotLimitedToDataSize(t } func TestHashTable_GetPage_ShouldReturnError_WhenPageIDIsOutOfTheTable(t *testing.T) { - d, err := makeDirectory() + d, _, err := makeDirectory() if err != nil { t.Fatal(err) } @@ -79,24 +80,24 @@ func TestHashTable_GetPage_ShouldReturnError_WhenPageIDIsOutOfTheTable(t *testin } func TestHashTable_Get(t *testing.T) { - d, err := makeDirectory() + d, _, err := makeDirectory() if err != nil { t.Fatal(err) } d.globalDepth = 4 d.directory = make([]bufferpool.PageID, 16) - d.directory[14] = 2 + d.directory[14] = bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 2} // force there to be two pages - page, err := d.bufferPool.NewPage() //1 + page, err := d.bufferPool.NewPage(0, 0) //1 if err != nil { t.Fatal(err) } page.WritePageType(bufferpool.PAGE_TYPE_HASH_TABLE) d.bufferPool.FlushPage(page.ID()) - page, err = d.bufferPool.NewPage() //2 + page, err = d.bufferPool.NewPage(0, 0) //2 if err != nil { t.Fatal(err) } @@ -104,7 +105,7 @@ func TestHashTable_Get(t *testing.T) { d.bufferPool.FlushPage(page.ID()) // now do the test - page, err = d.bufferPool.FetchPage(2) + page, err = d.bufferPool.FetchPage(bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 2}) if err != nil { t.Fatal(err) } @@ -113,7 +114,7 @@ func TestHashTable_Get(t *testing.T) { key := "478" value := "Hi" - page.WriteKeyValueInSlot(0, []byte(key), []byte(value)) + page.PutKeyValueInPageSlot(0, []byte(key), []byte(value)) page.WriteSlotCount(int16(1)) result, _, err := d.Get([]byte(key)) @@ -125,7 +126,7 @@ func TestHashTable_Get(t *testing.T) { } func TestHashTable_Get_ShouldHandleError(t *testing.T) { - d, err := makeDirectory() + d, _, err := makeDirectory() if err != nil { t.Fatal(err) } @@ -139,12 +140,12 @@ func TestHashTable_Get_ShouldHandleError(t *testing.T) { } func TestHashTable_Put(t *testing.T) { - d, err := makeDirectory() + d, _, err := makeDirectory() if err != nil { t.Fatal(err) } - page, err := d.bufferPool.FetchPage(0) + page, err := d.bufferPool.FetchPage(bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 0}) if err != nil { t.Fatal(err) } @@ -165,12 +166,12 @@ func TestHashTable_Put(t *testing.T) { } func TestHashTable_Put_ShouldIncreaseSize_WhenTableIsFull(t *testing.T) { - d, err := makeDirectory() + d, _, err := makeDirectory() if err != nil { t.Fatal(err) } - page, err := d.bufferPool.FetchPage(0) + page, err := d.bufferPool.FetchPage(bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 0}) if err != nil { t.Fatal(err) } @@ -192,12 +193,12 @@ func TestHashTable_Put_ShouldIncreaseSize_WhenTableIsFull(t *testing.T) { } func TestHashTable_PutShouldIncrementLD_WhenPageIsFull(t *testing.T) { - d, err := makeDirectory() + d, ds, err := makeDirectory() if err != nil { t.Fatal(err) } - page, err := d.bufferPool.FetchPage(0) + page, err := d.bufferPool.FetchPage(bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 0}) if err != nil { t.Fatal(err) } @@ -209,17 +210,17 @@ func TestHashTable_PutShouldIncrementLD_WhenPageIsFull(t *testing.T) { d.Put([]byte("12345678"), []byte("Yolo !")) - assert.Equal(t, int64(8192*2), d.bufferPool.OnDiskSize()) + assert.Equal(t, int64(8192*2), ds.FileSize(0, 0)) assert.Equal(t, 1, int(d.globalDepth)) - p0, err := d.bufferPool.FetchPage(0) + p0, err := d.bufferPool.FetchPage(bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 0}) if err != nil { t.Fatal(err) } defer d.bufferPool.UnpinPage(p0.ID()) assert.Equal(t, int16(1), p0.ReadLocalDepth()) - p1, err := d.bufferPool.FetchPage(1) + p1, err := d.bufferPool.FetchPage(bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 1}) if err != nil { t.Fatal(err) } @@ -228,7 +229,7 @@ func TestHashTable_PutShouldIncrementLD_WhenPageIsFull(t *testing.T) { } func TestHashTable_Put_INT(t *testing.T) { - d, err := makeDirectory() + d, _, err := makeDirectory() if err != nil { t.Fatal(err) } @@ -240,11 +241,44 @@ func TestHashTable_Put_INT(t *testing.T) { } } - assert.Equal(t, []bufferpool.PageID{0, 1, 2, 3, 4, 7, 6, 5, 13, 14, 12, 9, 8, 15, 10, 11, 28, 24, 21, 18, 4, 19, 29, 20, 27, 22, 25, 23, 17, 15, 16, 26}, d.directory) + assert.Equal(t, []bufferpool.PageID{ + {ObjectID: 0, Shard: 0, Page: 0}, + {ObjectID: 0, Shard: 0, Page: 1}, + {ObjectID: 0, Shard: 0, Page: 2}, + {ObjectID: 0, Shard: 0, Page: 3}, + {ObjectID: 0, Shard: 0, Page: 4}, + {ObjectID: 0, Shard: 0, Page: 7}, + {ObjectID: 0, Shard: 0, Page: 6}, + {ObjectID: 0, Shard: 0, Page: 5}, + {ObjectID: 0, Shard: 0, Page: 13}, + {ObjectID: 0, Shard: 0, Page: 15}, + {ObjectID: 0, Shard: 0, Page: 11}, + {ObjectID: 0, Shard: 0, Page: 9}, + {ObjectID: 0, Shard: 0, Page: 8}, + {ObjectID: 0, Shard: 0, Page: 14}, + {ObjectID: 0, Shard: 0, Page: 10}, + {ObjectID: 0, Shard: 0, Page: 12}, + {ObjectID: 0, Shard: 0, Page: 28}, + {ObjectID: 0, Shard: 0, Page: 25}, + {ObjectID: 0, Shard: 0, Page: 21}, + {ObjectID: 0, Shard: 0, Page: 18}, + {ObjectID: 0, Shard: 0, Page: 4}, + {ObjectID: 0, Shard: 0, Page: 20}, + {ObjectID: 0, Shard: 0, Page: 29}, + {ObjectID: 0, Shard: 0, Page: 19}, + {ObjectID: 0, Shard: 0, Page: 27}, + {ObjectID: 0, Shard: 0, Page: 24}, + {ObjectID: 0, Shard: 0, Page: 23}, + {ObjectID: 0, Shard: 0, Page: 22}, + {ObjectID: 0, Shard: 0, Page: 17}, + {ObjectID: 0, Shard: 0, Page: 14}, + {ObjectID: 0, Shard: 0, Page: 16}, + {ObjectID: 0, Shard: 0, Page: 26}, + }, d.directory) } func TestHashTable_Put_SameKey_ALotOfTime(t *testing.T) { - d, err := makeDirectory() + d, ds, err := makeDirectory() if err != nil { t.Fatal(err) } @@ -260,11 +294,11 @@ func TestHashTable_Put_SameKey_ALotOfTime(t *testing.T) { assert.Equal(t, "Yolo ! 9999", string(value)) assert.Equal(t, 1, len(d.directory)) - assert.Equal(t, int64(8192), d.bufferPool.OnDiskSize()) + assert.Equal(t, int64(8192), ds.FileSize(0, 0)) } func TestHashTable_Put_Many_Keys(t *testing.T) { - d, err := makeDirectory() + d, _, err := makeDirectory() if err != nil { t.Fatal(err) } @@ -289,7 +323,7 @@ func TestHashTable_Put_Many_Keys(t *testing.T) { func BenchmarkHashTable_Put_Many_Keys(b *testing.B) { for i := 0; i < b.N; i++ { - d, err := makeDirectory() + d, _, err := makeDirectory() if err != nil { b.Fatal(err) } @@ -317,7 +351,7 @@ func addToPage(page *bufferpool.Page, numberOfRecords int) error { for i := 0; i < numberOfRecords; i++ { //fmt.Printf("writing record %d\n", i+1) itoa := strconv.Itoa(i) - err := page.WriteKeyValueInSlot(int16(i), []byte("key"+itoa), []byte("value foo bar")) + err := page.PutKeyValueInPageSlot(int16(i), []byte("key"+itoa), []byte("value foo bar")) if err != nil { return err } diff --git a/field.go b/field.go index 34763ec79..2a3222ce1 100644 --- a/field.go +++ b/field.go @@ -33,6 +33,8 @@ const ( // Default ranked field cache DefaultCacheSize = 50000 + DefaultVarcharLength = 50 + bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64 maxInt = 1<<(bitsPerWord-1) - 1 // either 1<<31 - 1 or 1<<63 - 1 @@ -47,6 +49,7 @@ const ( FieldTypeBool = "bool" FieldTypeDecimal = "decimal" FieldTypeTimestamp = "timestamp" + FieldTypeVarchar = "varchar" ) type protected struct { @@ -141,6 +144,19 @@ func OptFieldForeignIndex(index string) FieldOption { } } +// OptFieldTypeNonRBFDefault is a functional option on FieldOptions +// used to set the field type and cache setting to the default values. +func OptFieldTypeNonRBFDefault() FieldOption { + return func(fo *FieldOptions) error { + if fo.Type != "" { + return errors.Errorf("field type is already set to: %s", fo.Type) + } + fo.Type = FieldTypeVarchar + fo.Length = DefaultVarcharLength + return nil + } +} + // OptFieldTypeDefault is a functional option on FieldOptions // used to set the field type and cache setting to the default values. func OptFieldTypeDefault() FieldOption { @@ -389,6 +405,20 @@ func OptFieldTrackExistence() FieldOption { } } +// OptFieldTypeVarchar is a functional option on FieldOptions +// used to specify the field as being type `varchar` and to +// provide any respective configuration values. +func OptFieldTypeVarchar(length int64) FieldOption { + return func(fo *FieldOptions) error { + if fo.Type != "" { + return errors.Errorf("field type is already set to: %s", fo.Type) + } + fo.Type = FieldTypeVarchar + fo.Length = length + return nil + } +} + // newField returns a new instance of field (without name validation). func newField(holder *Holder, path, index, name string, opts ...FieldOption) (*Field, error) { // Apply functional option. @@ -582,39 +612,50 @@ func (f *Field) Open() error { return errors.Wrap(err, "creating field dir") } - f.holder.Logger.Debugf("load available shards for index/field: %s/%s", f.index, f.name) + if strings.EqualFold(f.options.Type, FieldTypeVarchar) { + f.holder.Logger.Debugf("opening b-tree for index/field: %s/%s", f.index, f.name) - if err := f.loadAvailableShards(); err != nil { - return errors.Wrap(err, "loading available shards") - } - - // Apply the field options loaded from etcd (or set via setOptions()). - f.holder.Logger.Debugf("apply options for index/field: %s/%s", f.index, f.name) - if err := f.applyOptions(f.options); err != nil { - return errors.Wrap(err, "applying options") - } - - f.holder.Logger.Debugf("open views for index/field: %s/%s", f.index, f.name) - if err := f.openViews(); err != nil { - return errors.Wrap(err, "opening views") - } - - // Apply the field-specific translateStore. - if err := f.applyTranslateStore(); err != nil { - return errors.Wrap(err, "applying translate store") - } - - // If the field has a foreign index, make sure the index - // exists. - if f.options.ForeignIndex != "" { - if err := f.holder.checkForeignIndex(f); err != nil { - return errors.Wrap(err, "checking foreign index") + // Apply the field options loaded from etcd (or set via setOptions()). + f.holder.Logger.Debugf("apply options for index/field: %s/%s", f.index, f.name) + if err := f.applyOptions(f.options); err != nil { + return errors.Wrap(err, "applying options") } - } - f.availableShardChan = make(chan struct{}, 1) - f.wg.Add(1) - go f.writeAvailableShards() + } else { + f.holder.Logger.Debugf("load available shards for index/field: %s/%s", f.index, f.name) + + if err := f.loadAvailableShards(); err != nil { + return errors.Wrap(err, "loading available shards") + } + + // Apply the field options loaded from etcd (or set via setOptions()). + f.holder.Logger.Debugf("apply options for index/field: %s/%s", f.index, f.name) + if err := f.applyOptions(f.options); err != nil { + return errors.Wrap(err, "applying options") + } + + f.holder.Logger.Debugf("open views for index/field: %s/%s", f.index, f.name) + if err := f.openViews(); err != nil { + return errors.Wrap(err, "opening views") + } + + // Apply the field-specific translateStore. + if err := f.applyTranslateStore(); err != nil { + return errors.Wrap(err, "applying translate store") + } + + // If the field has a foreign index, make sure the index + // exists. + if f.options.ForeignIndex != "" { + if err := f.holder.checkForeignIndex(f); err != nil { + return errors.Wrap(err, "checking foreign index") + } + } + + f.availableShardChan = make(chan struct{}, 1) + f.wg.Add(1) + go f.writeAvailableShards() + } return nil }(); err != nil { f.unprotectedClose() @@ -883,6 +924,9 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.TTL = 0 f.options.Keys = false f.options.ForeignIndex = "" + case FieldTypeVarchar: + f.options.Type = FieldTypeVarchar + f.options.Length = opt.Length default: return errors.New("invalid field type") } @@ -2195,6 +2239,7 @@ type FieldOptions struct { TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` ForeignIndex string `json:"foreignIndex"` TTL time.Duration `json:"ttl,omitempty"` + Length int64 `json:"length,omitempty"` } // newFieldOptions returns a new instance of FieldOptions @@ -2360,6 +2405,14 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { }{ o.Type, }) + case FieldTypeVarchar: + return json.Marshal(struct { + Type string `json:"type"` + Length int64 `json:"length"` + }{ + o.Type, + o.Length, + }) } return nil, errors.Errorf("invalid field type: '%s'", o.Type) } diff --git a/handler.go b/handler.go index cbad04d34..99fec2f70 100644 --- a/handler.go +++ b/handler.go @@ -464,6 +464,8 @@ type ImportRoaringShardRequest struct { Remote bool Views []RoaringUpdate + Tuples []byte + // SuppressLog requests we not write to the write log. Typically // that would be because this request is being replayed from a // write log. diff --git a/holder.go b/holder.go index 3d8459583..856b1259c 100644 --- a/holder.go +++ b/holder.go @@ -10,9 +10,11 @@ import ( "path/filepath" "runtime" "sort" + "strings" "sync" "time" + "github.com/featurebasedb/featurebase/v3/bufferpool" "github.com/featurebasedb/featurebase/v3/dax" "github.com/featurebasedb/featurebase/v3/disco" "github.com/featurebasedb/featurebase/v3/logger" @@ -43,6 +45,9 @@ const ( // DataframesDir is the directory where we store the dataframe files (currently Apache Arrow) DataframesDir = "dataframes" + + // TStoreDir is the directory where we store the t-store files + TStoreDir = "tstore" ) func init() { @@ -143,6 +148,10 @@ type Holder struct { // snapshotter/writelogger; then the Controller should only start directing // queries to that computer once it has completed applying the snapshot. directiveApplied bool + + // t-store + tstorepool *bufferpool.BufferPool + tstoredisk *bufferpool.OnDiskDiskManager } // HolderOpts holds information about the holder which other things might want @@ -254,6 +263,8 @@ type HolderConfig struct { Sharder disco.Sharder CacheFlushInterval time.Duration Logger logger.Logger + TStoreBufferPool *bufferpool.BufferPool + TStoreDiskManager *bufferpool.OnDiskDiskManager StorageConfig *storage.Config RBFConfig *rbfcfg.Config @@ -266,6 +277,7 @@ type HolderConfig struct { // need to override these; that's usually handled by server options // such as OptServerOpenTranslateStore. func DefaultHolderConfig() *HolderConfig { + dm := bufferpool.NewOnDiskDiskManager() return &HolderConfig{ PartitionN: disco.DefaultPartitionN, OpenTranslateStore: OpenInMemTranslateStore, @@ -280,6 +292,8 @@ func DefaultHolderConfig() *HolderConfig { Logger: logger.NopLogger, StorageConfig: storage.NewDefaultConfig(), RBFConfig: rbfcfg.NewDefaultConfig(), + TStoreDiskManager: dm, + TStoreBufferPool: bufferpool.NewBufferPool(1024, dm), } } @@ -328,6 +342,8 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { sharder: cfg.Sharder, Schemator: cfg.Schemator, Logger: cfg.Logger, + tstorepool: cfg.TStoreBufferPool, + tstoredisk: cfg.TStoreDiskManager, Opts: HolderOpts{StorageBackend: cfg.StorageConfig.Backend}, Auditor: NewAuditor(), @@ -494,6 +510,7 @@ func (h *Holder) Open() error { // from the API, which already comes from etcd. In that case, this logic // could be removed, and the createdAt on the index struct could be // removed. + index.ID = cim.IndexID index.createdAt = cim.CreatedAt index.owner = cim.Owner index.description = cim.Meta.Description @@ -724,6 +741,7 @@ func (h *Holder) schema(ctx context.Context, includeViews bool) ([]*IndexInfo, e } di := &IndexInfo{ + ID: cim.IndexID, Name: cim.Index, CreatedAt: cim.CreatedAt, Owner: cim.Owner, @@ -992,6 +1010,7 @@ func (h *Holder) createIndex(cim *CreateIndexMessage, broadcast bool) (*Index, e index.keys = cim.Meta.Keys index.trackExistence = cim.Meta.TrackExistence + index.ID = cim.IndexID index.createdAt = cim.CreatedAt index.owner = cim.Owner index.description = cim.Meta.Description @@ -1035,6 +1054,7 @@ func (h *Holder) createIndexWithPartitions(cim *CreateIndexMessage, translatePar index.keys = cim.Meta.Keys index.trackExistence = cim.Meta.TrackExistence + index.ID = cim.IndexID index.createdAt = cim.CreatedAt index.translatePartitions = translatePartitions @@ -1475,7 +1495,9 @@ func (s *holderSyncer) setTranslateReadOnlyFlags(snap *disco.ClusterSnapshot) { } for _, field := range index.Fields() { - field.TranslateStore().SetReadOnly(!isPrimaryFieldTranslator) + if !strings.EqualFold(field.options.Type, FieldTypeVarchar) { + field.TranslateStore().SetReadOnly(!isPrimaryFieldTranslator) + } } } s.Cluster.mu.RUnlock() diff --git a/http_handler.go b/http_handler.go index 5d7eb79c2..9f89beb82 100644 --- a/http_handler.go +++ b/http_handler.go @@ -2072,6 +2072,8 @@ func fieldOptionsToFunctionalOpts(opt fieldOptions) []FieldOption { fos = append(fos, OptFieldTypeMutex(*opt.CacheType, *opt.CacheSize)) case FieldTypeBool: fos = append(fos, OptFieldTypeBool()) + case FieldTypeVarchar: + fos = append(fos, OptFieldTypeVarchar(*opt.Length)) } if opt.Keys != nil { if *opt.Keys { @@ -2193,6 +2195,7 @@ type fieldOptions struct { ForeignIndex *string `json:"foreignIndex,omitempty"` TTL *string `json:"ttl,omitempty"` Base *int64 `json:"base,omitempty"` + Length *int64 `json:"length,omitempty"` } func (o *fieldOptions) validate() error { @@ -2309,6 +2312,10 @@ func (o *fieldOptions) validate() error { } else if o.ForeignIndex != nil { return NewBadRequestError(errors.New("bool field cannot be a foreign key")) } + case FieldTypeVarchar: + if o.Length == nil { + return NewBadRequestError(errors.New("varchar field requires a length argument")) + } default: return errors.Errorf("invalid field type: %s", o.Type) } diff --git a/index.go b/index.go index 5887d2a41..308278f4c 100644 --- a/index.go +++ b/index.go @@ -10,6 +10,7 @@ import ( "path/filepath" "sort" "strconv" + "strings" "sync" "time" @@ -26,6 +27,7 @@ import ( // Index represents a container for fields. type Index struct { mu sync.RWMutex + ID int32 createdAt int64 owner string description string @@ -112,6 +114,11 @@ func (i *Index) DataframesPath() string { return filepath.Join(i.path, DataframesDir) } +// TStorePath returns the path of the t-store files specific to an index +func (i *Index) TStorePath() string { + return filepath.Join(i.path, TStoreDir) +} + // Name returns name of the index. func (i *Index) Name() string { return i.name } @@ -180,6 +187,7 @@ func (i *Index) OpenWithSchema(idx *disco.Index) error { return errors.Wrap(err, "decoding create index message") } i.createdAt = cim.CreatedAt + i.ID = cim.IndexID i.trackExistence = cim.Meta.TrackExistence i.keys = cim.Meta.Keys @@ -206,6 +214,12 @@ func (i *Index) open(idx *disco.Index) (err error) { return errors.Wrap(err, "creating dataframes directory") } + // Ensure the t-store path exists + i.holder.Logger.Debugf("ensure t-store path exists: %s", i.TStorePath()) + if err := os.MkdirAll(i.TStorePath(), 0o750); err != nil { + return errors.Wrap(err, "creating tstore directory") + } + i.closing = make(chan struct{}) // fmt.Printf("new channel %p for index %p\n", i.closing, i) @@ -240,6 +254,12 @@ func (i *Index) open(idx *disco.Index) (err error) { return errors.Wrap(err, "setting field bitDepths") } + // if the table has store fields then open the b-tree files + if i.hasTStoreFields() { + // TODO (pok) kick off recovery here + i.holder.Logger.Debugf("open t-store index: %s", i.name) + } + if i.trackExistence { if err := i.openExistenceField(); err != nil { return errors.Wrap(err, "opening existence field") @@ -353,13 +373,18 @@ fileLoop: // openField opens the field directory, initializes the field, and adds it to // the in-memory map of fields maintained by Index. func (i *Index) openField(mu *sync.Mutex, cfm *CreateFieldMessage, file string) (*Field, error) { + var err error + var fld *Field mu.Lock() - fld, err := i.newField(i.fieldPath(filepath.Base(file)), filepath.Base(file)) + if strings.EqualFold(cfm.Meta.Type, FieldTypeVarchar) { + fld, err = i.newNonRBFField(i.fieldPath(filepath.Base(file)), filepath.Base(file)) + } else { + fld, err = i.newField(i.fieldPath(filepath.Base(file)), filepath.Base(file)) + } mu.Unlock() if err != nil { return nil, errors.Wrapf(ErrName, "'%s'", file) } - // Pass holder through to the field for use in looking // up a foreign index. fld.holder = i.holder @@ -432,6 +457,16 @@ func (i *Index) setFieldBitDepths() error { return nil } +func (i *Index) hasTStoreFields() bool { + for _, f := range i.fields { + switch f.Type() { + case FieldTypeVarchar: + return true + } + } + return false +} + // Close closes the index and its fields. func (i *Index) Close() error { i.mu.Lock() @@ -933,11 +968,22 @@ func (i *Index) createField(cfm *CreateFieldMessage) (*Field, error) { return nil, ErrInvalidCacheType } - // Initialize field. - f, err := i.newField(i.fieldPath(cfm.Field), cfm.Field) - if err != nil { - return nil, errors.Wrap(err, "initializing") + var err error + var f *Field + // initialize non-rbf field + if strings.EqualFold(cfm.Meta.Type, FieldTypeVarchar) { + f, err = i.newNonRBFField(i.fieldPath(cfm.Field), cfm.Field) + if err != nil { + return nil, errors.Wrap(err, "initializing") + } + } else { + // initialize rbf field + f, err = i.newField(i.fieldPath(cfm.Field), cfm.Field) + if err != nil { + return nil, errors.Wrap(err, "initializing") + } } + f.createdAt = cfm.CreatedAt f.owner = cfm.Owner @@ -978,6 +1024,17 @@ func (i *Index) newField(path, name string) (*Field, error) { return f, nil } +func (i *Index) newNonRBFField(path, name string) (*Field, error) { + f, err := newField(i.holder, path, i.name, name, OptFieldTypeNonRBFDefault()) + if err != nil { + return nil, err + } + f.idx = i + f.broadcaster = i.broadcaster + f.serializer = i.serializer + return f, nil +} + // DeleteField removes a field from the index. func (i *Index) DeleteField(name string) error { i.mu.Lock() @@ -1047,6 +1104,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { + ID int32 `json:"id"` Name string `json:"name"` CreatedAt int64 `json:"createdAt,omitempty"` UpdatedAt int64 `json:"updatedAt"` diff --git a/pb/private.pb.go b/pb/private.pb.go index 6ea3b50de..56849eacf 100644 --- a/pb/private.pb.go +++ b/pb/private.pb.go @@ -103,6 +103,7 @@ type FieldOptions struct { TimeUnit string `protobuf:"bytes,19,opt,name=TimeUnit,proto3" json:"TimeUnit,omitempty"` TTL string `protobuf:"bytes,20,opt,name=TTL,proto3" json:"TTL,omitempty"` TrackExistence bool `protobuf:"varint,21,opt,name=TrackExistence,proto3" json:"TrackExistence,omitempty"` + Length int64 `protobuf:"varint,22,opt,name=Length,proto3" json:"Length,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -260,6 +261,13 @@ func (m *FieldOptions) GetTrackExistence() bool { return false } +func (m *FieldOptions) GetLength() int64 { + if m != nil { + return m.Length + } + return 0 +} + type ImportResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -650,6 +658,7 @@ type CreateIndexMessage struct { Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` Owner string `protobuf:"bytes,5,opt,name=Owner,proto3" json:"Owner,omitempty"` + IndexID int32 `protobuf:"varint,6,opt,name=IndexID,proto3" json:"IndexID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -716,6 +725,13 @@ func (m *CreateIndexMessage) GetOwner() string { return "" } +func (m *CreateIndexMessage) GetIndexID() int32 { + if m != nil { + return m.IndexID + } + return 0 +} + type CreateFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -1146,6 +1162,7 @@ type Index struct { CreatedAt int64 `protobuf:"varint,2,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` Options *IndexMeta `protobuf:"bytes,5,opt,name=Options,proto3" json:"Options,omitempty"` Fields []*Field `protobuf:"bytes,4,rep,name=Fields,proto3" json:"Fields,omitempty"` + IndexID int32 `protobuf:"varint,6,opt,name=IndexID,proto3" json:"IndexID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1212,6 +1229,13 @@ func (m *Index) GetFields() []*Field { return nil } +func (m *Index) GetIndexID() int32 { + if m != nil { + return m.IndexID + } + return 0 +} + type URI struct { Scheme string `protobuf:"bytes,1,opt,name=Scheme,proto3" json:"Scheme,omitempty"` Host string `protobuf:"bytes,2,opt,name=Host,proto3" json:"Host,omitempty"` @@ -2959,117 +2983,119 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1750 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcd, 0x6e, 0x24, 0x49, - 0x11, 0xa6, 0x7e, 0xdc, 0x3f, 0xd1, 0x6e, 0x4f, 0x3b, 0xd7, 0x6b, 0x6a, 0xbc, 0x83, 0xd5, 0x93, - 0xa0, 0x99, 0x66, 0x24, 0x8c, 0xf0, 0x1e, 0x16, 0xb1, 0x97, 0x1d, 0x77, 0x7b, 0x96, 0x66, 0x77, - 0x7e, 0x36, 0xed, 0x99, 0x23, 0x28, 0x5d, 0x9d, 0xd8, 0xa5, 0xa9, 0xae, 0x6a, 0xaa, 0xaa, 0x3d, - 0xdd, 0x7b, 0x40, 0x02, 0x09, 0xc1, 0x85, 0x3b, 0xe2, 0xc0, 0x5b, 0xf0, 0x0e, 0x5c, 0x90, 0x78, - 0x04, 0x34, 0xdc, 0x78, 0x0a, 0x94, 0x91, 0x99, 0x55, 0xd9, 0xed, 0xb2, 0xbd, 0x58, 0x7b, 0xab, - 0xf8, 0x22, 0x2b, 0xf2, 0x8b, 0x9f, 0x8c, 0x8c, 0x2a, 0xe8, 0xce, 0xb2, 0xe8, 0x92, 0x17, 0xe2, - 0x60, 0x96, 0xa5, 0x45, 0x4a, 0xdc, 0xd9, 0xd9, 0xde, 0xe6, 0x6c, 0x7e, 0x16, 0x47, 0xa1, 0x42, - 0x68, 0x04, 0xed, 0x71, 0x32, 0x11, 0x8b, 0xe7, 0xa2, 0xe0, 0x84, 0x80, 0xff, 0x85, 0x58, 0xe6, - 0x81, 0xd7, 0x77, 0x06, 0x2d, 0x86, 0xcf, 0xe4, 0x11, 0x6c, 0x9d, 0x66, 0x3c, 0x7c, 0x7b, 0xbc, - 0x88, 0xf2, 0x42, 0x24, 0xa1, 0x08, 0x7c, 0xd4, 0xae, 0xa1, 0xa4, 0x0f, 0x9d, 0x91, 0xc8, 0xc3, - 0x2c, 0x9a, 0x15, 0x51, 0x9a, 0x04, 0x1b, 0x7d, 0x67, 0xd0, 0x66, 0x36, 0x44, 0xff, 0xeb, 0xc1, - 0xe6, 0xb3, 0x48, 0xc4, 0x93, 0x97, 0x28, 0xe7, 0x72, 0xbb, 0xd3, 0xe5, 0x4c, 0x04, 0x2d, 0x5c, - 0x8b, 0xcf, 0xe4, 0x01, 0xb4, 0x87, 0x3c, 0xbc, 0x10, 0xa8, 0xf0, 0x50, 0x51, 0x01, 0xa5, 0xf6, - 0x24, 0xfa, 0x5a, 0xf1, 0xe8, 0xb2, 0x0a, 0x90, 0x14, 0x4e, 0xa3, 0xa9, 0xf8, 0x6a, 0xce, 0x93, - 0x62, 0x3e, 0x35, 0x14, 0x2c, 0x88, 0xec, 0x42, 0xe3, 0x65, 0x3c, 0x79, 0x1e, 0x25, 0x41, 0xbb, - 0xef, 0x0c, 0x3c, 0xa6, 0x25, 0x83, 0xf3, 0x45, 0x00, 0x15, 0xce, 0x17, 0x65, 0x40, 0x3a, 0xab, - 0x01, 0x79, 0x91, 0x9e, 0x14, 0x3c, 0x99, 0xf0, 0x6c, 0xf2, 0x26, 0x12, 0xef, 0x82, 0x4d, 0x15, - 0x90, 0x55, 0x54, 0xbe, 0x7b, 0xc4, 0x73, 0x11, 0x74, 0xd1, 0x22, 0x3e, 0x93, 0x3d, 0x68, 0x1d, - 0x45, 0xc5, 0x48, 0xcc, 0x8a, 0x8b, 0x60, 0xab, 0xef, 0x0c, 0x7c, 0x56, 0xca, 0x64, 0x07, 0x36, - 0x4e, 0x42, 0x1e, 0x8b, 0xe0, 0x1e, 0xbe, 0xa0, 0x04, 0x42, 0x61, 0xf3, 0x59, 0x9a, 0x89, 0xe8, - 0x3c, 0xc1, 0x34, 0x05, 0x3d, 0x74, 0x6a, 0x05, 0x23, 0xdf, 0x03, 0x4f, 0xba, 0xb4, 0xdd, 0x77, - 0x06, 0x9d, 0xc3, 0xce, 0xc1, 0xec, 0xec, 0x60, 0x24, 0xc2, 0x68, 0xca, 0x63, 0x26, 0x71, 0x54, - 0xf3, 0x45, 0x40, 0xea, 0xd4, 0x7c, 0x21, 0x39, 0xc9, 0x10, 0xbd, 0x4e, 0xa2, 0x22, 0xf8, 0x00, - 0xad, 0x97, 0x32, 0xe9, 0x81, 0x77, 0x7a, 0xfa, 0x65, 0xb0, 0x83, 0xb0, 0x7c, 0xac, 0x29, 0x87, - 0x0f, 0xeb, 0xca, 0x81, 0x52, 0xd8, 0x1a, 0x4f, 0x67, 0x69, 0x56, 0x30, 0x91, 0xcf, 0xd2, 0x24, - 0x17, 0xd2, 0xd6, 0x71, 0x96, 0x05, 0x8e, 0xb2, 0x75, 0x9c, 0x65, 0xf4, 0xb7, 0xd0, 0x3b, 0x8a, - 0xd3, 0xf0, 0xed, 0x88, 0x17, 0x9c, 0x89, 0xdf, 0xcc, 0x45, 0x5e, 0xc8, 0x28, 0x28, 0x47, 0xd5, - 0x3a, 0x25, 0x48, 0x14, 0x2b, 0x27, 0x70, 0x15, 0x8a, 0x82, 0x8c, 0x30, 0xc6, 0x5f, 0x25, 0x1a, - 0x9f, 0x31, 0x8a, 0x17, 0x3c, 0x9b, 0x60, 0x75, 0xf8, 0x4c, 0x09, 0x12, 0xc5, 0x9d, 0xb0, 0xa2, - 0x7c, 0xa6, 0x04, 0x3a, 0x86, 0x6d, 0x6b, 0x7f, 0x4d, 0x73, 0x17, 0x1a, 0x2c, 0x7d, 0x37, 0x1e, - 0xe5, 0x81, 0xd3, 0xf7, 0x06, 0x3e, 0xd3, 0x12, 0x96, 0x5e, 0x1a, 0xcf, 0xa7, 0x89, 0x54, 0xb9, - 0xa8, 0xaa, 0x00, 0x7a, 0x1f, 0x36, 0xb0, 0x0e, 0xa5, 0x97, 0xd5, 0xbb, 0xf2, 0x91, 0xfe, 0xce, - 0x81, 0xf6, 0x73, 0xbe, 0x40, 0x22, 0x39, 0xf9, 0x04, 0x5a, 0xa6, 0x4a, 0x70, 0x51, 0xe7, 0xf0, - 0x23, 0x99, 0x91, 0x72, 0xc1, 0x81, 0xd1, 0x1e, 0x27, 0x45, 0xb6, 0x64, 0xe5, 0xe2, 0xbd, 0x4f, - 0xa1, 0xbb, 0xa2, 0x92, 0x3b, 0xbd, 0x15, 0x4b, 0x13, 0xcf, 0xb7, 0x62, 0x29, 0xbd, 0xbc, 0xe4, - 0xf1, 0x5c, 0x60, 0x94, 0x7c, 0xa6, 0x84, 0x9f, 0xb9, 0x3f, 0x75, 0xe8, 0x1b, 0x20, 0xc3, 0x4c, - 0xf0, 0x42, 0xe0, 0x26, 0xcf, 0x45, 0x9e, 0xf3, 0x73, 0x71, 0x5b, 0xac, 0x3d, 0x3b, 0xd6, 0x65, - 0x5c, 0x5d, 0x2b, 0xae, 0xf4, 0x09, 0x90, 0x91, 0x88, 0x45, 0x21, 0x74, 0x0f, 0xb9, 0xc1, 0xae, - 0x8c, 0x83, 0x26, 0x71, 0xfb, 0x62, 0xf2, 0x10, 0x7c, 0xd9, 0x91, 0x70, 0xb7, 0xce, 0x61, 0x57, - 0x86, 0xa8, 0x6c, 0x53, 0x0c, 0x55, 0x98, 0x10, 0x34, 0x37, 0x79, 0x5a, 0x20, 0x57, 0x8f, 0x55, - 0x80, 0x34, 0xfb, 0xf2, 0x5d, 0x22, 0x32, 0x5d, 0x1c, 0x4a, 0xa0, 0x7f, 0x2d, 0x39, 0xa0, 0x57, - 0xdf, 0x30, 0x10, 0x2b, 0x45, 0xf7, 0x03, 0xcd, 0xcc, 0x43, 0x66, 0x3d, 0xc9, 0xcc, 0x6e, 0x6a, - 0x75, 0xe4, 0xfc, 0x6f, 0x46, 0xee, 0x0f, 0x0e, 0x90, 0xd7, 0xb3, 0xc9, 0x3a, 0xb9, 0x67, 0x75, - 0x94, 0x91, 0x69, 0xe7, 0x70, 0x57, 0x6e, 0x7f, 0x55, 0xcb, 0xea, 0x9c, 0x7c, 0x0c, 0x0d, 0x65, - 0x5d, 0x07, 0xf5, 0x5e, 0x49, 0x5d, 0xc1, 0x4c, 0xab, 0xe9, 0xa7, 0xd0, 0xb1, 0x60, 0xec, 0x8d, - 0xaa, 0xa7, 0xab, 0xe8, 0x68, 0x49, 0x3a, 0xf1, 0xa6, 0xac, 0xb6, 0x36, 0x53, 0x02, 0xfd, 0xcc, - 0x54, 0xc4, 0x5d, 0x03, 0x4c, 0x43, 0xf8, 0x48, 0x59, 0x78, 0x7a, 0xc9, 0xa3, 0x98, 0x9f, 0xc5, - 0xff, 0x57, 0xd1, 0xae, 0xe4, 0x2a, 0x80, 0x26, 0xbe, 0x3b, 0x1e, 0xe9, 0x83, 0x6f, 0x44, 0x3a, - 0x87, 0xaa, 0x87, 0xbc, 0xe0, 0x53, 0xa1, 0xad, 0xe1, 0x73, 0x99, 0x62, 0xf7, 0xc6, 0x14, 0x4b, - 0xff, 0x23, 0xf1, 0x4e, 0xde, 0x96, 0x1e, 0xfa, 0x2f, 0x85, 0x9b, 0x13, 0x4f, 0x7f, 0x04, 0x8d, - 0x93, 0xf0, 0x42, 0x4c, 0x39, 0xf9, 0x3e, 0x34, 0x91, 0xb9, 0xc8, 0x75, 0x1b, 0x68, 0x97, 0x35, - 0xce, 0x8c, 0x46, 0x56, 0x84, 0xf6, 0xaf, 0x8e, 0xe6, 0xca, 0x56, 0xee, 0x7a, 0x8d, 0x3d, 0x86, - 0xa6, 0xe6, 0x8b, 0x55, 0x76, 0xe5, 0x10, 0x19, 0x2d, 0x79, 0x08, 0x0d, 0xf4, 0x2e, 0x0f, 0xfc, - 0x8a, 0x08, 0x22, 0x4c, 0x2b, 0xe8, 0x31, 0x78, 0xaf, 0xd9, 0x58, 0x56, 0x02, 0xb2, 0x37, 0x34, - 0xb4, 0x24, 0xc9, 0xfd, 0x3c, 0xcd, 0x0b, 0x1d, 0x7b, 0x7c, 0x96, 0xd8, 0xab, 0x34, 0x53, 0x07, - 0xb3, 0xcb, 0xf0, 0x99, 0xfe, 0xc9, 0x01, 0xff, 0x45, 0x3a, 0x11, 0x64, 0x0b, 0xdc, 0xf1, 0x48, - 0x1b, 0x71, 0xc7, 0x23, 0x72, 0x1f, 0xed, 0xeb, 0x78, 0x37, 0xe5, 0xfe, 0xaf, 0xd9, 0x98, 0xe1, - 0x9e, 0x0f, 0xa0, 0x3d, 0xce, 0x5f, 0x65, 0xd1, 0x94, 0x67, 0x4b, 0x3d, 0x97, 0x54, 0x00, 0x76, - 0xa5, 0x42, 0x96, 0xb4, 0xaf, 0xd2, 0x8e, 0x02, 0x79, 0x08, 0xcd, 0xcf, 0xd9, 0xab, 0xa1, 0x34, - 0xb9, 0xb1, 0x6a, 0xd2, 0xe0, 0xf4, 0x33, 0xe8, 0x49, 0x26, 0xb8, 0xde, 0x54, 0xd6, 0x2e, 0x34, - 0x24, 0x56, 0x32, 0xd3, 0x52, 0xb5, 0x89, 0x6b, 0x6d, 0x42, 0x9f, 0x29, 0x0b, 0xc7, 0x97, 0x22, - 0x29, 0xac, 0xda, 0x44, 0x19, 0x0d, 0x74, 0x99, 0x12, 0xc8, 0x03, 0xe5, 0xb5, 0x76, 0xaf, 0x25, - 0xb9, 0x48, 0x99, 0x21, 0x4a, 0x97, 0x00, 0x86, 0xc9, 0x3c, 0x2f, 0xd7, 0x3a, 0x75, 0x6b, 0x09, - 0x35, 0xe5, 0xa3, 0xbb, 0x0f, 0x48, 0xbd, 0x42, 0x98, 0x29, 0xac, 0x1f, 0x56, 0x85, 0xa5, 0xf2, - 0x79, 0xaf, 0xcc, 0xbb, 0xda, 0xa3, 0x2a, 0xaf, 0x0b, 0xe8, 0x58, 0x78, 0x6d, 0x8d, 0x3d, 0x2e, - 0x8b, 0xc3, 0xad, 0x8c, 0x21, 0xa2, 0x8d, 0x69, 0xf5, 0xcd, 0xdd, 0x98, 0x46, 0xba, 0xa5, 0xdc, - 0xb0, 0xd3, 0x00, 0xee, 0xad, 0x1e, 0x78, 0x73, 0xcb, 0xae, 0xc3, 0xb7, 0x6c, 0xf5, 0x47, 0x07, - 0xba, 0xc3, 0x78, 0x9e, 0x17, 0x22, 0x2b, 0x63, 0xda, 0xd6, 0x40, 0x99, 0xda, 0x0a, 0xa8, 0xcf, - 0x2e, 0xd9, 0x87, 0x0d, 0x19, 0x71, 0x75, 0xb8, 0xed, 0x44, 0x28, 0xd8, 0xca, 0x84, 0x7f, 0x5d, - 0x26, 0xe8, 0x1b, 0x68, 0x1d, 0x9d, 0x8c, 0x3f, 0xcf, 0xd2, 0xf9, 0xac, 0xd6, 0x63, 0x33, 0xfe, - 0xba, 0xd6, 0xf8, 0xdb, 0x53, 0xa3, 0x9c, 0xf2, 0x0a, 0xa7, 0xb7, 0x9e, 0x9a, 0xde, 0x7c, 0x8d, - 0xf0, 0x05, 0x3d, 0x81, 0x6d, 0xe5, 0xae, 0xec, 0x38, 0x77, 0x69, 0x8b, 0x66, 0x6e, 0xf2, 0xaa, - 0xb9, 0x49, 0x1a, 0x55, 0x5d, 0xf7, 0xdb, 0x34, 0xfa, 0x4f, 0x17, 0xb6, 0x99, 0xc8, 0xa3, 0xaf, - 0xc5, 0x38, 0xc9, 0x8b, 0x6c, 0x1e, 0x9a, 0x8b, 0xe3, 0x17, 0xe9, 0x99, 0xce, 0x85, 0xc7, 0x94, - 0x70, 0xf3, 0x29, 0x21, 0x14, 0x9a, 0x76, 0x13, 0xb0, 0x17, 0x18, 0x05, 0x79, 0x02, 0xcd, 0x93, - 0x74, 0x9e, 0x85, 0x65, 0xe5, 0x63, 0xe7, 0x56, 0xfb, 0x2b, 0x05, 0x33, 0x0b, 0xc8, 0x17, 0x40, - 0x4e, 0x33, 0x9e, 0xe4, 0x31, 0x97, 0x94, 0xcc, 0x6b, 0xad, 0x6a, 0x20, 0xb3, 0xb4, 0x2b, 0x16, - 0x6a, 0x5e, 0x23, 0x07, 0xf6, 0x11, 0x0e, 0x9a, 0xc8, 0x6f, 0xcb, 0xf0, 0xd3, 0xe7, 0xc4, 0x3e, - 0xe4, 0x9f, 0xac, 0x55, 0x68, 0xd0, 0xc0, 0x57, 0xb6, 0xf1, 0x32, 0xb7, 0x15, 0x6c, 0x75, 0x1d, - 0xfd, 0xbd, 0x03, 0x9b, 0x36, 0x9b, 0x5b, 0xda, 0x45, 0x99, 0x3e, 0xf7, 0xf6, 0xf9, 0xce, 0xa4, - 0xcf, 0xaf, 0x9b, 0xa5, 0x37, 0xec, 0x99, 0x2f, 0x85, 0xef, 0x5e, 0x13, 0x9c, 0x3b, 0xd1, 0xe9, - 0x43, 0xe7, 0x15, 0xcf, 0x8a, 0x48, 0x1a, 0xd3, 0xf7, 0xf4, 0x06, 0xb3, 0x21, 0x2a, 0xe0, 0xfe, - 0x95, 0x22, 0x1a, 0xa6, 0xd3, 0x99, 0xac, 0xd6, 0x3b, 0x15, 0x93, 0x6c, 0xd3, 0x59, 0x96, 0x66, - 0x26, 0x02, 0x28, 0xd0, 0x23, 0x68, 0x9d, 0xa6, 0xb3, 0x34, 0x4e, 0xcf, 0x97, 0xb7, 0xb4, 0x8c, - 0x00, 0x9a, 0xea, 0x6a, 0x50, 0x2d, 0xaa, 0xcd, 0x8c, 0x48, 0x3f, 0x90, 0xf5, 0x1e, 0xf2, 0x38, - 0x9c, 0xc7, 0xbc, 0x10, 0xf8, 0x45, 0x80, 0xe0, 0x97, 0x29, 0x9f, 0xa8, 0xae, 0xa0, 0x8f, 0x16, - 0xfd, 0x95, 0x2e, 0x40, 0x8e, 0xee, 0x58, 0x57, 0xd0, 0xd3, 0xd0, 0x9e, 0xb5, 0x94, 0x44, 0x7e, - 0x02, 0x1d, 0x6b, 0xb5, 0x3d, 0xc0, 0x59, 0x30, 0xb3, 0xd7, 0xd0, 0xbf, 0x3b, 0x2b, 0xef, 0x5c, - 0xb9, 0x73, 0xf5, 0x56, 0x97, 0x2a, 0x48, 0x2d, 0xa6, 0x25, 0xe9, 0xfa, 0xf1, 0x22, 0x8c, 0xe7, - 0xb9, 0x54, 0xe9, 0x0b, 0xb7, 0x04, 0xa4, 0xeb, 0xf2, 0xe3, 0x30, 0x9d, 0x9b, 0xe1, 0xc6, 0x88, - 0xf2, 0x33, 0x72, 0x24, 0xf8, 0x24, 0x8e, 0x12, 0x81, 0xf5, 0xe2, 0xb1, 0x52, 0x26, 0x4f, 0x54, - 0x8f, 0x35, 0x85, 0xbe, 0xb3, 0x46, 0x1c, 0x75, 0xaa, 0xf3, 0xe6, 0x94, 0x40, 0x6f, 0x5d, 0x45, - 0x77, 0x80, 0xa8, 0x0a, 0x78, 0x7a, 0x96, 0x66, 0xe6, 0xb6, 0xa5, 0x43, 0xd3, 0x5c, 0x64, 0xf4, - 0x6f, 0xbb, 0xc4, 0xab, 0xc8, 0xba, 0x76, 0x64, 0xe9, 0x2f, 0x61, 0x4b, 0xcf, 0x76, 0x22, 0xc3, - 0x82, 0x96, 0x01, 0x60, 0x22, 0x4c, 0xe5, 0x98, 0x68, 0xbe, 0xe3, 0x2a, 0x40, 0xda, 0xc1, 0x41, - 0xd7, 0xdc, 0x4e, 0x5a, 0xc2, 0xd9, 0x28, 0x3a, 0x4f, 0xc4, 0x04, 0x6f, 0x0c, 0x8f, 0x69, 0x89, - 0xfe, 0xd9, 0x85, 0x1d, 0x35, 0x74, 0x26, 0xe7, 0x22, 0x2f, 0xaa, 0x6d, 0x70, 0xac, 0xc6, 0xfe, - 0x5f, 0x8e, 0xd5, 0x78, 0x03, 0x3c, 0x82, 0xad, 0x61, 0x2c, 0x78, 0x56, 0x71, 0x50, 0x1b, 0xad, - 0xa1, 0xf2, 0xdc, 0x20, 0xa2, 0xaf, 0x67, 0x35, 0x84, 0xda, 0x10, 0x39, 0x82, 0x96, 0x76, 0xcd, - 0x34, 0xc4, 0x47, 0x78, 0x4b, 0xd5, 0xb0, 0x31, 0xf3, 0x6d, 0xae, 0xbf, 0x3a, 0x8d, 0xb8, 0xf7, - 0x12, 0xba, 0x2b, 0xaa, 0x9a, 0xaf, 0xce, 0x81, 0xfd, 0xd5, 0xd9, 0x39, 0x24, 0xd6, 0xb8, 0xac, - 0xad, 0xdb, 0x5f, 0xa2, 0x43, 0xf8, 0xb0, 0x8e, 0x40, 0x4e, 0x9e, 0x80, 0x27, 0x89, 0xaa, 0x61, - 0x38, 0xb8, 0x8e, 0x28, 0x93, 0x8b, 0xe8, 0xdf, 0x1c, 0x1d, 0x54, 0xa1, 0xf5, 0xe6, 0xef, 0xc1, - 0xc7, 0xb6, 0x91, 0x87, 0xa5, 0x91, 0xb5, 0x65, 0x07, 0xa5, 0xa3, 0x72, 0xf5, 0xde, 0x57, 0xd0, - 0xaa, 0x73, 0xcf, 0x57, 0xee, 0xfd, 0x78, 0xd5, 0xbd, 0xfb, 0xd7, 0x31, 0xcb, 0x6d, 0x2f, 0x0f, - 0x60, 0x57, 0xdd, 0xa6, 0x23, 0x5e, 0xf0, 0x5f, 0x67, 0x7c, 0x2a, 0x6e, 0xbc, 0x52, 0x8f, 0x7a, - 0xff, 0x78, 0xbf, 0xef, 0xfc, 0xeb, 0xfd, 0xbe, 0xf3, 0xef, 0xf7, 0xfb, 0xce, 0x5f, 0xfe, 0xb3, - 0xff, 0x9d, 0xb3, 0x06, 0xfe, 0x9e, 0xfb, 0xf8, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x0d, - 0x5d, 0xdb, 0xc1, 0x13, 0x00, 0x00, + // 1781 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdd, 0x6e, 0x23, 0x49, + 0x15, 0xa6, 0x7f, 0xe2, 0x9f, 0xe3, 0x38, 0xe3, 0xd4, 0x66, 0x43, 0x4f, 0x76, 0x88, 0x3c, 0x05, + 0x9a, 0x31, 0x23, 0x11, 0x44, 0xf6, 0x62, 0x11, 0x7b, 0xb3, 0x13, 0x3b, 0xb3, 0x98, 0x9d, 0xbf, + 0xad, 0x64, 0xe6, 0x12, 0x54, 0x69, 0x17, 0x49, 0x6b, 0xda, 0xdd, 0xa6, 0xbb, 0x9d, 0xb1, 0xf7, + 0x02, 0x89, 0x95, 0x10, 0xdc, 0x70, 0x8f, 0x40, 0x82, 0xa7, 0xe0, 0x1d, 0xb8, 0x41, 0xe2, 0x11, + 0xd0, 0xf0, 0x22, 0xa8, 0x4e, 0x55, 0x75, 0x97, 0x3d, 0x3d, 0xce, 0x32, 0xda, 0xbb, 0x3e, 0xdf, + 0x29, 0x9f, 0xfa, 0xce, 0xa9, 0xaf, 0x4f, 0x9d, 0x36, 0x74, 0x67, 0x59, 0x74, 0xcd, 0x0b, 0x71, + 0x34, 0xcb, 0xd2, 0x22, 0x25, 0xee, 0xec, 0xe2, 0x60, 0x7b, 0x36, 0xbf, 0x88, 0xa3, 0x50, 0x21, + 0x34, 0x82, 0xf6, 0x38, 0x99, 0x88, 0xc5, 0x13, 0x51, 0x70, 0x42, 0xc0, 0xff, 0x42, 0x2c, 0xf3, + 0xc0, 0xeb, 0x3b, 0x83, 0x16, 0xc3, 0x67, 0x72, 0x0f, 0x76, 0xce, 0x33, 0x1e, 0xbe, 0x3a, 0x5d, + 0x44, 0x79, 0x21, 0x92, 0x50, 0x04, 0x3e, 0x7a, 0xd7, 0x50, 0xd2, 0x87, 0xce, 0x48, 0xe4, 0x61, + 0x16, 0xcd, 0x8a, 0x28, 0x4d, 0x82, 0xad, 0xbe, 0x33, 0x68, 0x33, 0x1b, 0xa2, 0x5f, 0xfb, 0xb0, + 0xfd, 0x28, 0x12, 0xf1, 0xe4, 0x19, 0xda, 0xb9, 0xdc, 0xee, 0x7c, 0x39, 0x13, 0x41, 0x0b, 0xd7, + 0xe2, 0x33, 0xb9, 0x03, 0xed, 0x21, 0x0f, 0xaf, 0x04, 0x3a, 0x3c, 0x74, 0x54, 0x40, 0xe9, 0x3d, + 0x8b, 0xbe, 0x52, 0x3c, 0xba, 0xac, 0x02, 0x24, 0x85, 0xf3, 0x68, 0x2a, 0xbe, 0x9c, 0xf3, 0xa4, + 0x98, 0x4f, 0x0d, 0x05, 0x0b, 0x22, 0xfb, 0xd0, 0x78, 0x16, 0x4f, 0x9e, 0x44, 0x49, 0xd0, 0xee, + 0x3b, 0x03, 0x8f, 0x69, 0xcb, 0xe0, 0x7c, 0x11, 0x40, 0x85, 0xf3, 0x45, 0x59, 0x90, 0xce, 0x6a, + 0x41, 0x9e, 0xa6, 0x67, 0x05, 0x4f, 0x26, 0x3c, 0x9b, 0xbc, 0x8c, 0xc4, 0xeb, 0x60, 0x5b, 0x15, + 0x64, 0x15, 0x95, 0xbf, 0x3d, 0xe1, 0xb9, 0x08, 0xba, 0x18, 0x11, 0x9f, 0xc9, 0x01, 0xb4, 0x4e, + 0xa2, 0x62, 0x24, 0x66, 0xc5, 0x55, 0xb0, 0xd3, 0x77, 0x06, 0x3e, 0x2b, 0x6d, 0xb2, 0x07, 0x5b, + 0x67, 0x21, 0x8f, 0x45, 0x70, 0x0b, 0x7f, 0xa0, 0x0c, 0x42, 0x61, 0xfb, 0x51, 0x9a, 0x89, 0xe8, + 0x32, 0xc1, 0x63, 0x0a, 0x7a, 0x98, 0xd4, 0x0a, 0x46, 0xbe, 0x07, 0x9e, 0x4c, 0x69, 0xb7, 0xef, + 0x0c, 0x3a, 0xc7, 0x9d, 0xa3, 0xd9, 0xc5, 0xd1, 0x48, 0x84, 0xd1, 0x94, 0xc7, 0x4c, 0xe2, 0xe8, + 0xe6, 0x8b, 0x80, 0xd4, 0xb9, 0xf9, 0x42, 0x72, 0x92, 0x25, 0x7a, 0x91, 0x44, 0x45, 0xf0, 0x01, + 0x46, 0x2f, 0x6d, 0xd2, 0x03, 0xef, 0xfc, 0xfc, 0x71, 0xb0, 0x87, 0xb0, 0x7c, 0xac, 0x91, 0xc3, + 0x87, 0xb5, 0x72, 0xd8, 0x87, 0xc6, 0x63, 0x91, 0x5c, 0x16, 0x57, 0xc1, 0xbe, 0xaa, 0xa8, 0xb2, + 0x28, 0x85, 0x9d, 0xf1, 0x74, 0x96, 0x66, 0x05, 0x13, 0xf9, 0x2c, 0x4d, 0x72, 0x21, 0xf7, 0x38, + 0xcd, 0xb2, 0xc0, 0x51, 0x7b, 0x9c, 0x66, 0x19, 0xfd, 0x2d, 0xf4, 0x4e, 0xe2, 0x34, 0x7c, 0x35, + 0xe2, 0x05, 0x67, 0xe2, 0x37, 0x73, 0x91, 0x17, 0xb2, 0x3a, 0xaa, 0x00, 0x6a, 0x9d, 0x32, 0x24, + 0x8a, 0x8a, 0x0a, 0x5c, 0x85, 0xa2, 0x21, 0x2b, 0x8f, 0xe7, 0xa2, 0x04, 0x80, 0xcf, 0x58, 0xdd, + 0x2b, 0x9e, 0x4d, 0x50, 0x35, 0x3e, 0x53, 0x86, 0x44, 0x71, 0x27, 0x54, 0x9a, 0xcf, 0x94, 0x41, + 0xc7, 0xb0, 0x6b, 0xed, 0xaf, 0x69, 0xee, 0x43, 0x83, 0xa5, 0xaf, 0xc7, 0xa3, 0x3c, 0x70, 0xfa, + 0xde, 0xc0, 0x67, 0xda, 0x42, 0x49, 0xa6, 0xf1, 0x7c, 0x9a, 0x48, 0x97, 0x8b, 0xae, 0x0a, 0xa0, + 0xb7, 0x61, 0x0b, 0xf5, 0x29, 0xb3, 0xac, 0x7e, 0x2b, 0x1f, 0xe9, 0xef, 0x1c, 0x68, 0x3f, 0xe1, + 0x0b, 0x24, 0x92, 0x93, 0x4f, 0xa0, 0x65, 0xd4, 0x83, 0x8b, 0x3a, 0xc7, 0x1f, 0xc9, 0x93, 0x2a, + 0x17, 0x1c, 0x19, 0xef, 0x69, 0x52, 0x64, 0x4b, 0x56, 0x2e, 0x3e, 0xf8, 0x14, 0xba, 0x2b, 0x2e, + 0xb9, 0xd3, 0x2b, 0xb1, 0x34, 0xf5, 0x7c, 0x25, 0x96, 0x32, 0xcb, 0x6b, 0x1e, 0xcf, 0x05, 0x56, + 0xc9, 0x67, 0xca, 0xf8, 0x99, 0xfb, 0x53, 0x87, 0xbe, 0x04, 0x32, 0xcc, 0x04, 0x2f, 0x04, 0x6e, + 0xf2, 0x44, 0xe4, 0x39, 0xbf, 0x14, 0x37, 0xd5, 0xda, 0xb3, 0x6b, 0x5d, 0xd6, 0xd5, 0xb5, 0xea, + 0x4a, 0x1f, 0x00, 0x19, 0x89, 0x58, 0x14, 0x42, 0xf7, 0x96, 0x0d, 0x71, 0xe9, 0x5f, 0x1d, 0x43, + 0xe2, 0xe6, 0xc5, 0xe4, 0x2e, 0xf8, 0xb2, 0x53, 0xe1, 0x6e, 0x9d, 0xe3, 0xae, 0x2c, 0x51, 0xd9, + 0xbe, 0x18, 0xba, 0xf0, 0x40, 0x30, 0xdc, 0xe4, 0x61, 0x81, 0x5c, 0x3d, 0x56, 0x01, 0x32, 0xec, + 0xb3, 0xd7, 0x89, 0xc8, 0xb4, 0x38, 0x94, 0x41, 0x02, 0x68, 0x62, 0x98, 0xf1, 0x28, 0x68, 0xf4, + 0x9d, 0xc1, 0x16, 0x33, 0x26, 0xfd, 0x4b, 0xc9, 0x0e, 0xf3, 0xfd, 0x86, 0x25, 0x5a, 0x91, 0xe3, + 0x0f, 0x34, 0x67, 0x0f, 0x39, 0xf7, 0x24, 0x67, 0xbb, 0x0d, 0xd6, 0xd1, 0xf6, 0xbf, 0x11, 0x6d, + 0xfa, 0x7b, 0x07, 0xc8, 0x8b, 0xd9, 0x64, 0x9d, 0xdc, 0xa3, 0x3a, 0xca, 0xc8, 0xb4, 0x73, 0xbc, + 0x2f, 0xb7, 0x7f, 0xdb, 0xcb, 0xea, 0x92, 0xbc, 0x0f, 0x0d, 0x15, 0x5d, 0x97, 0xfb, 0x56, 0x49, + 0x5d, 0xc1, 0x4c, 0xbb, 0xe9, 0xa7, 0xd0, 0xb1, 0x60, 0xec, 0xa6, 0xea, 0x16, 0x50, 0xd5, 0xd1, + 0x96, 0x4c, 0xe2, 0x65, 0xa9, 0xc3, 0x36, 0x53, 0x06, 0xfd, 0xcc, 0x68, 0xe5, 0x7d, 0x0b, 0x4c, + 0x43, 0xf8, 0x48, 0x45, 0x78, 0x78, 0xcd, 0xa3, 0x98, 0x5f, 0xc4, 0xff, 0x97, 0x9c, 0x57, 0xce, + 0x2a, 0x80, 0x26, 0xfe, 0x76, 0x3c, 0xd2, 0x2d, 0xc1, 0x98, 0x74, 0x0e, 0x55, 0x77, 0x79, 0xca, + 0xa7, 0x42, 0x47, 0xc3, 0xe7, 0xf2, 0x88, 0xdd, 0x8d, 0x47, 0x2c, 0xf3, 0x8f, 0xc4, 0x6b, 0x79, + 0xbf, 0x7a, 0x98, 0xbf, 0x34, 0x36, 0x1f, 0x3c, 0xfd, 0x11, 0x34, 0xce, 0xc2, 0x2b, 0x31, 0xe5, + 0xe4, 0xfb, 0x5a, 0xa3, 0x22, 0xd7, 0x0d, 0xa2, 0x5d, 0xaa, 0x9f, 0x19, 0x0f, 0xfd, 0xbb, 0xa3, + 0x93, 0xad, 0xa5, 0xb9, 0xb2, 0x95, 0xbb, 0xae, 0xb1, 0xfb, 0xd0, 0xd4, 0x7c, 0x51, 0x65, 0x6f, + 0xbd, 0x5e, 0xc6, 0x4b, 0xee, 0x42, 0x03, 0xb3, 0xcb, 0x03, 0xbf, 0x22, 0x82, 0x08, 0xd3, 0x8e, + 0x0d, 0x2f, 0xd4, 0x29, 0x78, 0x2f, 0xd8, 0x58, 0x6a, 0x04, 0xf3, 0x32, 0x04, 0xb5, 0x25, 0x69, + 0xff, 0x3c, 0xcd, 0x0b, 0x7d, 0x2a, 0xf8, 0x2c, 0xb1, 0xe7, 0x69, 0xa6, 0x5e, 0xe6, 0x2e, 0xc3, + 0x67, 0xfa, 0x47, 0x07, 0xfc, 0xa7, 0xe9, 0x44, 0x90, 0x1d, 0x70, 0xc7, 0x23, 0x1d, 0xc4, 0x1d, + 0x8f, 0xc8, 0x6d, 0x8c, 0xaf, 0x4f, 0xa2, 0x29, 0x99, 0xbd, 0x60, 0x63, 0x86, 0x7b, 0xde, 0x81, + 0xf6, 0x38, 0x7f, 0x9e, 0x45, 0x53, 0x9e, 0x2d, 0xf5, 0x8c, 0x53, 0x01, 0xd8, 0xc9, 0x0a, 0x29, + 0x76, 0x5f, 0x09, 0x02, 0x0d, 0x72, 0x17, 0x9a, 0x9f, 0xb3, 0xe7, 0x43, 0x19, 0x72, 0x6b, 0x35, + 0xa4, 0xc1, 0xe9, 0x67, 0xd0, 0x93, 0x4c, 0x70, 0xbd, 0xd1, 0xdc, 0x3e, 0x34, 0x24, 0x56, 0x32, + 0xd3, 0x56, 0xb5, 0x89, 0x6b, 0x6d, 0x42, 0x1f, 0xa9, 0x08, 0xa7, 0xd7, 0x22, 0x29, 0x2c, 0xd5, + 0xa2, 0x8d, 0x01, 0xba, 0x4c, 0x19, 0xe4, 0x8e, 0xca, 0x5a, 0xa7, 0xd7, 0x92, 0x5c, 0xa4, 0xcd, + 0x10, 0xa5, 0x4b, 0x00, 0xc3, 0x64, 0x9e, 0x97, 0x6b, 0x9d, 0xba, 0xb5, 0x84, 0x1a, 0x61, 0xe9, + 0xbe, 0x04, 0xd2, 0xaf, 0x10, 0x66, 0x24, 0xf7, 0xc3, 0x4a, 0x72, 0xea, 0xa4, 0x6f, 0x95, 0x8a, + 0x50, 0x7b, 0x54, 0xc2, 0xbb, 0x82, 0x8e, 0x85, 0xd7, 0xaa, 0xef, 0x7e, 0x29, 0x1b, 0xb7, 0x0a, + 0x86, 0x88, 0x0e, 0x66, 0xc4, 0xb3, 0xb1, 0x83, 0xd3, 0x48, 0x37, 0x9b, 0x0d, 0x3b, 0x0d, 0xe0, + 0xd6, 0x6a, 0x2b, 0x30, 0x37, 0xf3, 0x3a, 0x7c, 0xc3, 0x56, 0x7f, 0x70, 0xa0, 0x3b, 0x8c, 0xe7, + 0x79, 0x21, 0xb2, 0xb2, 0xa6, 0x6d, 0x0d, 0x94, 0x47, 0x5b, 0x01, 0xf5, 0xa7, 0x4b, 0x0e, 0x61, + 0x4b, 0x56, 0x5c, 0xbd, 0xf6, 0xf6, 0x41, 0x28, 0xd8, 0x3a, 0x09, 0xff, 0x5d, 0x27, 0x41, 0x5f, + 0x42, 0xeb, 0xe4, 0x6c, 0xfc, 0x79, 0x96, 0xce, 0x67, 0xb5, 0x19, 0x9b, 0x51, 0xda, 0xb5, 0x46, + 0xe9, 0x9e, 0x1a, 0x0b, 0x55, 0x56, 0x38, 0x09, 0xf6, 0xd4, 0x24, 0xe8, 0x6b, 0x84, 0x2f, 0xe8, + 0x19, 0xec, 0xaa, 0x74, 0x65, 0x2f, 0x7a, 0x9f, 0x86, 0x69, 0x66, 0x2d, 0xaf, 0x9a, 0xb5, 0x64, + 0x50, 0xd5, 0x8f, 0xbf, 0xcd, 0xa0, 0xff, 0x72, 0x61, 0x97, 0x89, 0x3c, 0xfa, 0x4a, 0x8c, 0x93, + 0xbc, 0xc8, 0xe6, 0xa1, 0xb9, 0x52, 0x7e, 0x91, 0x5e, 0xe8, 0xb3, 0xf0, 0x98, 0x32, 0x36, 0xbf, + 0x25, 0x84, 0x42, 0xd3, 0x6e, 0x02, 0xf6, 0x02, 0xe3, 0x20, 0x0f, 0xa0, 0x79, 0x96, 0xce, 0xb3, + 0xb0, 0x54, 0x3e, 0xf6, 0x74, 0xb5, 0xbf, 0x72, 0x30, 0xb3, 0x80, 0x7c, 0x01, 0xe4, 0x3c, 0xe3, + 0x49, 0x1e, 0x73, 0x49, 0xc9, 0xfc, 0xac, 0x55, 0x0d, 0x71, 0x96, 0x77, 0x25, 0x42, 0xcd, 0xcf, + 0xc8, 0x91, 0xfd, 0x0a, 0x07, 0x4d, 0xe4, 0xb7, 0x63, 0xf8, 0xe9, 0xf7, 0xc4, 0x7e, 0xc9, 0x3f, + 0x59, 0x53, 0x28, 0xb6, 0xdb, 0xce, 0xf1, 0x2e, 0x5e, 0xf3, 0xb6, 0x83, 0xad, 0xae, 0xa3, 0x5f, + 0x3b, 0xb0, 0x6d, 0xb3, 0xb9, 0xa1, 0x5d, 0x94, 0xc7, 0xe7, 0xde, 0x3c, 0x13, 0x9a, 0xe3, 0xf3, + 0xeb, 0xe6, 0xef, 0x2d, 0x7b, 0x4e, 0x4c, 0xe1, 0xbb, 0xef, 0x28, 0xce, 0x7b, 0xd1, 0xe9, 0x43, + 0xe7, 0x39, 0xcf, 0x8a, 0x48, 0x06, 0xd3, 0x37, 0xf8, 0x16, 0xb3, 0x21, 0x2a, 0xe0, 0xf6, 0x5b, + 0x22, 0x1a, 0xa6, 0xd3, 0x99, 0x54, 0xeb, 0x7b, 0x89, 0x49, 0xb6, 0xe9, 0x2c, 0x4b, 0x33, 0x53, + 0x01, 0x34, 0xe8, 0x09, 0xb4, 0xce, 0xd3, 0x59, 0x1a, 0xa7, 0x97, 0xcb, 0x1b, 0x5a, 0x46, 0x00, + 0x4d, 0x75, 0x35, 0xa8, 0x16, 0xd5, 0x66, 0xc6, 0xa4, 0x1f, 0x48, 0xbd, 0x87, 0x3c, 0x0e, 0xe7, + 0x31, 0x2f, 0x04, 0x7e, 0x45, 0x20, 0xf8, 0x38, 0xe5, 0x13, 0xd5, 0x15, 0xf4, 0xab, 0x45, 0x7f, + 0xa5, 0x05, 0xc8, 0x31, 0x1d, 0xeb, 0x0a, 0x7a, 0x18, 0xda, 0x53, 0x98, 0xb2, 0xc8, 0x4f, 0xa0, + 0x63, 0xad, 0xb6, 0x47, 0x3b, 0x0b, 0x66, 0xf6, 0x1a, 0xfa, 0x0f, 0x67, 0xe5, 0x37, 0x6f, 0xdd, + 0xb9, 0x7a, 0xab, 0x6b, 0x55, 0xa4, 0x16, 0xd3, 0x96, 0x4c, 0xfd, 0x74, 0x11, 0xc6, 0xf3, 0x5c, + 0xba, 0xf4, 0x85, 0x5b, 0x02, 0x32, 0x75, 0xf9, 0xa1, 0x99, 0xce, 0xcd, 0xd8, 0x63, 0x4c, 0xf9, + 0x49, 0x3a, 0x12, 0x7c, 0x12, 0x47, 0x89, 0x40, 0xbd, 0x78, 0xac, 0xb4, 0xc9, 0x03, 0xd5, 0x63, + 0x8d, 0xd0, 0xf7, 0xd6, 0x88, 0xa3, 0x4f, 0x75, 0xde, 0x9c, 0x12, 0xe8, 0xad, 0xbb, 0xe8, 0x1e, + 0x10, 0xa5, 0x80, 0x87, 0x17, 0x69, 0x66, 0x6e, 0x5b, 0x3a, 0x34, 0xcd, 0x45, 0x56, 0xff, 0xa6, + 0x4b, 0xbc, 0xaa, 0xac, 0x6b, 0x57, 0x96, 0xfe, 0x12, 0x76, 0xf4, 0xd4, 0x27, 0x32, 0x14, 0xb4, + 0x2c, 0x00, 0x13, 0x61, 0x2a, 0x07, 0x48, 0xf3, 0xed, 0x57, 0x01, 0x32, 0x0e, 0x8e, 0xc0, 0xe6, + 0x76, 0xd2, 0x16, 0xce, 0x46, 0xd1, 0x65, 0x22, 0x26, 0x78, 0x63, 0x78, 0x4c, 0x5b, 0xf4, 0x4f, + 0x2e, 0xec, 0xa9, 0x71, 0x34, 0xb9, 0x14, 0x79, 0x51, 0x6d, 0x83, 0x03, 0x37, 0xf6, 0xff, 0x72, + 0xe0, 0xc6, 0x1b, 0xe0, 0x1e, 0xec, 0x0c, 0x63, 0xc1, 0xb3, 0x8a, 0x83, 0xda, 0x68, 0x0d, 0x95, + 0xef, 0x0d, 0x22, 0xfa, 0x7a, 0x56, 0xe3, 0xa9, 0x0d, 0x91, 0x13, 0x68, 0xe9, 0xd4, 0x4c, 0x43, + 0xbc, 0x87, 0xb7, 0x54, 0x0d, 0x1b, 0x33, 0xf9, 0xe6, 0xfa, 0x4b, 0xd5, 0x98, 0x07, 0xcf, 0xa0, + 0xbb, 0xe2, 0xaa, 0xf9, 0x52, 0x1d, 0xd8, 0x5f, 0xaa, 0x9d, 0x63, 0x62, 0x0d, 0xd2, 0x3a, 0xba, + 0xfd, 0xf5, 0x3a, 0x84, 0x0f, 0xeb, 0x08, 0xe4, 0xe4, 0x01, 0x78, 0x92, 0xa8, 0x1a, 0x93, 0x83, + 0x77, 0x11, 0x65, 0x72, 0x11, 0xfd, 0x9b, 0xa3, 0x8b, 0x2a, 0xb4, 0xdf, 0xfc, 0xe3, 0xf0, 0xb1, + 0x1d, 0xe4, 0x6e, 0x19, 0x64, 0x6d, 0xd9, 0x51, 0x99, 0xa8, 0x5c, 0x7d, 0xf0, 0x25, 0xb4, 0xea, + 0xd2, 0xf3, 0x55, 0x7a, 0x3f, 0x5e, 0x4d, 0xef, 0xf6, 0xbb, 0x98, 0xe5, 0x76, 0x96, 0x47, 0xb0, + 0xaf, 0x6e, 0xd3, 0x11, 0x2f, 0xf8, 0xaf, 0x33, 0x3e, 0x15, 0x1b, 0xaf, 0xd4, 0x93, 0xde, 0x3f, + 0xdf, 0x1c, 0x3a, 0xff, 0x7e, 0x73, 0xe8, 0xfc, 0xe7, 0xcd, 0xa1, 0xf3, 0xe7, 0xff, 0x1e, 0x7e, + 0xe7, 0xa2, 0x81, 0x7f, 0xf5, 0x7d, 0xfc, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xe2, 0xbe, 0x5d, + 0xe0, 0x0d, 0x14, 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -3150,6 +3176,13 @@ func (m *FieldOptions) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.Length != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.Length)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xb0 + } if m.TrackExistence { i-- if m.TrackExistence { @@ -3639,6 +3672,11 @@ func (m *CreateIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.IndexID != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.IndexID)) + i-- + dAtA[i] = 0x30 + } if len(m.Owner) > 0 { i -= len(m.Owner) copy(dAtA[i:], m.Owner) @@ -4042,6 +4080,11 @@ func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.IndexID != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.IndexID)) + i-- + dAtA[i] = 0x30 + } if m.Options != nil { { size, err := m.Options.MarshalToSizedBuffer(dAtA[:i]) @@ -5660,6 +5703,9 @@ func (m *FieldOptions) Size() (n int) { if m.TrackExistence { n += 3 } + if m.Length != 0 { + n += 2 + sovPrivate(uint64(m.Length)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5837,6 +5883,9 @@ func (m *CreateIndexMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.IndexID != 0 { + n += 1 + sovPrivate(uint64(m.IndexID)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -6027,6 +6076,9 @@ func (m *Index) Size() (n int) { l = m.Options.Size() n += 1 + l + sovPrivate(uint64(l)) } + if m.IndexID != 0 { + n += 1 + sovPrivate(uint64(m.IndexID)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7299,6 +7351,25 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } } m.TrackExistence = bool(v != 0) + case 22: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Length", wireType) + } + m.Length = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Length |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8448,6 +8519,25 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { } m.Owner = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexID", wireType) + } + m.IndexID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexID |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -9564,6 +9654,25 @@ func (m *Index) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexID", wireType) + } + m.IndexID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexID |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) diff --git a/pb/private.proto b/pb/private.proto index 6822a872d..117cc4362 100644 --- a/pb/private.proto +++ b/pb/private.proto @@ -28,6 +28,7 @@ message FieldOptions { string TimeUnit = 19; string TTL = 20; bool TrackExistence = 21; + int64 Length = 22; } message ImportResponse { @@ -70,6 +71,7 @@ message CreateIndexMessage { IndexMeta Meta = 2; int64 CreatedAt = 3; string Owner = 5; + int32 IndexID = 6; } message CreateFieldMessage { @@ -117,6 +119,7 @@ message Index { int64 CreatedAt = 2; IndexMeta Options = 5; repeated Field Fields = 4; + int32 IndexID = 6; } message URI { diff --git a/pb/public.pb.go b/pb/public.pb.go index b01493b11..bf6af62af 100644 --- a/pb/public.pb.go +++ b/pb/public.pb.go @@ -575,6 +575,7 @@ func (m *KeyList) GetKeys() []string { type ExtractedTableValue struct { // Types that are valid to be assigned to Value: + // // *ExtractedTableValue_IDs // *ExtractedTableValue_Keys // *ExtractedTableValue_BSIValue @@ -715,6 +716,7 @@ func (*ExtractedTableValue) XXX_OneofWrappers() []interface{} { type ExtractedTableColumn struct { // Types that are valid to be assigned to KeyOrID: + // // *ExtractedTableColumn_Key // *ExtractedTableColumn_ID KeyOrID isExtractedTableColumn_KeyOrID `protobuf_oneof:"KeyOrID"` diff --git a/schema.go b/schema.go index a8944dd62..52db8475c 100644 --- a/schema.go +++ b/schema.go @@ -202,6 +202,7 @@ func FieldInfoToField(fi *FieldInfo) *dax.Field { var epoch time.Time var foreignIndex string var timeQuantum dax.TimeQuantum + var length int64 fo := &fi.Options @@ -245,6 +246,9 @@ func FieldInfoToField(fi *FieldInfo) *dax.Field { fieldType = dax.BaseTypeIDSetQ } timeQuantum = dax.TimeQuantum(fo.TimeQuantum) + case FieldTypeVarchar: + fieldType = dax.BaseTypeVarchar + length = fo.Length default: panic(fmt.Sprintf("unhandled featurebase field type: %s", fo.Type)) } @@ -265,6 +269,7 @@ func FieldInfoToField(fi *FieldInfo) *dax.Field { TTL: fo.TTL, ForeignIndex: foreignIndex, TrackExistence: fo.TrackExistence, + Length: length, }, } } @@ -360,6 +365,7 @@ func FieldToFieldInfo(fld *dax.Field) *FieldInfo { Min: min, Max: max, Scale: fld.Options.Scale, + Length: fld.Options.Length, Keys: fld.StringKeys(), NoStandardView: fld.Options.NoStandardView, CacheType: fld.Options.CacheType, @@ -491,6 +497,11 @@ func FieldOptionsFromField(fld *dax.Field) ([]FieldOption, error) { opts = append(opts, OptFieldTypeTimestamp(fld.Options.Epoch, fld.Options.TimeUnit), ) + case dax.BaseTypeVarchar: + opts = append(opts, + OptFieldTypeVarchar(fld.Options.Length), + ) + default: return nil, errors.Errorf("unsupport field type: %s", fld.Type) } diff --git a/sql3/errors.go b/sql3/errors.go index 676534fcb..56495ecc6 100644 --- a/sql3/errors.go +++ b/sql3/errors.go @@ -46,6 +46,9 @@ const ( // decimal ErrDecimalScaleExpected errors.Code = "ErrDecimalScaleExpected" + // varchar + ErrVarcharLengthExpected errors.Code = "ErrVarcharLengthExpected" + ErrInvalidCast errors.Code = "ErrInvalidCast" ErrInvalidTypeCoercion errors.Code = "ErrInvalidTypeCoercion" @@ -507,6 +510,13 @@ func NewErrDecimalScaleExpected(line, col int) error { ) } +func NewErrVarcharLengthExpected(line, col int) error { + return errors.New( + ErrVarcharLengthExpected, + fmt.Sprintf("[%d:%d] varchar length expected", line, col), + ) +} + func NewErrInvalidTimeUnit(line, col int, unit string) error { return errors.New( ErrInvalidTimeUnit, diff --git a/sql3/parser/ast.go b/sql3/parser/ast.go index 17f8738b2..47e7f98b0 100644 --- a/sql3/parser/ast.go +++ b/sql3/parser/ast.go @@ -1787,11 +1787,10 @@ func (i *Variable) VarName() string { } type Type struct { - Name *Ident // type name - Lparen Pos // position of left paren (optional) - Precision *IntegerLit // precision (optional) - Scale *IntegerLit // scale (optional) - Rparen Pos // position of right paren (optional) + Name *Ident // type name + Lparen Pos // position of left paren (optional) + Modifier *IntegerLit // scale, length etc. (optional) + Rparen Pos // position of right paren (optional) } // Clone returns a deep copy of t. @@ -1801,20 +1800,14 @@ func (t *Type) Clone() *Type { } other := *t other.Name = t.Name.Clone() - other.Precision = t.Precision.Clone() - other.Scale = t.Scale.Clone() + other.Modifier = t.Modifier.Clone() return &other } // String returns the string representation of the type. func (t *Type) String() string { - if t.Precision != nil && t.Scale != nil { - return fmt.Sprintf("%s(%s,%s)", t.Name.Name, t.Precision.String(), t.Scale.String()) - } else if t.Precision != nil { - return fmt.Sprintf("%s(%s)", t.Name.Name, t.Precision.String()) - } else if t.Scale != nil { - // I'm not sure how you're supposed to tell this from the t.Precision case. - return fmt.Sprintf("%s(%s)", t.Name.Name, t.Scale.String()) + if t.Modifier != nil { + return fmt.Sprintf("%s(%s)", t.Name.Name, t.Modifier.String()) } return t.Name.Name } diff --git a/sql3/parser/ast_test.go b/sql3/parser/ast_test.go index 44ce35e6b..206d58dfc 100644 --- a/sql3/parser/ast_test.go +++ b/sql3/parser/ast_test.go @@ -347,7 +347,7 @@ func OLDTestCreateTableStatement_String(t *testing.T) { Name: &parser.Ident{Name: "foo"}, Columns: []*parser.ColumnDefinition{{ Name: &parser.Ident{Name: "bar"}, - Type: &parser.Type{Name: &parser.Ident{Name: "DECIMAL"}, Precision: &parser.IntegerLit{Value: "100"}}, + Type: &parser.Type{Name: &parser.Ident{Name: "DECIMAL"}, Modifier: &parser.IntegerLit{Value: "100"}}, }}, Constraints: []parser.Constraint{ &parser.PrimaryKeyConstraint{ @@ -375,7 +375,7 @@ func OLDTestCreateTableStatement_String(t *testing.T) { Name: &parser.Ident{Name: "foo"}, Columns: []*parser.ColumnDefinition{{ Name: &parser.Ident{Name: "bar"}, - Type: &parser.Type{Name: &parser.Ident{Name: "DECIMAL"}, Precision: &parser.IntegerLit{Value: "100"}, Scale: &parser.IntegerLit{Value: "200"}}, + Type: &parser.Type{Name: &parser.Ident{Name: "DECIMAL"}, Modifier: &parser.IntegerLit{Value: "200"}}, }}, Constraints: []parser.Constraint{ &parser.ForeignKeyConstraint{ @@ -385,7 +385,7 @@ func OLDTestCreateTableStatement_String(t *testing.T) { ForeignColumns: []*parser.Ident{{Name: "y"}, {Name: "z"}}, }, }, - }, `CREATE TABLE "foo" ("bar" DECIMAL(100,200), CONSTRAINT "fk" FOREIGN KEY ("a", "b") REFERENCES "x" ("y", "z"))`) + }, `CREATE TABLE "foo" ("bar" DECIMAL(200), CONSTRAINT "fk" FOREIGN KEY ("a", "b") REFERENCES "x" ("y", "z"))`) AssertStatementStringer(t, &parser.CreateTableStatement{ Name: &parser.Ident{Name: "foo"}, diff --git a/sql3/parser/astdatatype.go b/sql3/parser/astdatatype.go index 2b9114624..87ee634cf 100644 --- a/sql3/parser/astdatatype.go +++ b/sql3/parser/astdatatype.go @@ -18,7 +18,8 @@ func IsValidTypeName(typeName string) bool { dax.BaseTypeString, dax.BaseTypeStringSet, dax.BaseTypeStringSetQ, - dax.BaseTypeTimestamp: + dax.BaseTypeTimestamp, + dax.BaseTypeVarchar: return true default: return false @@ -54,6 +55,7 @@ func (*DataTypeString) exprDataType() {} func (*DataTypeStringSet) exprDataType() {} func (*DataTypeStringSetQuantum) exprDataType() {} func (*DataTypeTimestamp) exprDataType() {} +func (*DataTypeVarchar) exprDataType() {} type DataTypeVoid struct { } @@ -202,6 +204,30 @@ func (d *DataTypeDecimal) TypeInfo() map[string]interface{} { } } +type DataTypeVarchar struct { + Length int64 +} + +func NewDataTypeVarchar(length int64) *DataTypeVarchar { + return &DataTypeVarchar{ + Length: length, + } +} + +func (d *DataTypeVarchar) BaseTypeName() string { + return dax.BaseTypeVarchar +} + +func (d *DataTypeVarchar) TypeDescription() string { + return fmt.Sprintf("%s(%d)", dax.BaseTypeVarchar, d.Length) +} + +func (d *DataTypeVarchar) TypeInfo() map[string]interface{} { + return map[string]interface{}{ + "length": d.Length, + } +} + type DataTypeID struct { } diff --git a/sql3/parser/parser.go b/sql3/parser/parser.go index 50533472f..6f3990cfd 100644 --- a/sql3/parser/parser.go +++ b/sql3/parser/parser.go @@ -1769,10 +1769,10 @@ func (p *Parser) parseType() (_ *Type, err error) { return &typ, err } - // Optionally parse scale. + // Optionally parse modifier. if p.peek() == LP { typ.Lparen, _, _ = p.scan() - if typ.Scale, err = p.parseIntegerLiteral("scale"); err != nil { + if typ.Modifier, err = p.parseIntegerLiteral("scale"); err != nil { return &typ, err } diff --git a/sql3/parser/parser_test.go b/sql3/parser/parser_test.go index b818a1956..31d11adc8 100644 --- a/sql3/parser/parser_test.go +++ b/sql3/parser/parser_test.go @@ -997,10 +997,10 @@ func TestParser_ParseStatement(t *testing.T) { { Name: &parser.Ident{NamePos: pos(29), Name: "col2"}, Type: &parser.Type{ - Name: &parser.Ident{NamePos: pos(34), Name: "DECIMAL"}, - Lparen: pos(41), - Scale: &parser.IntegerLit{ValuePos: pos(42), Value: "2"}, - Rparen: pos(43), + Name: &parser.Ident{NamePos: pos(34), Name: "DECIMAL"}, + Lparen: pos(41), + Modifier: &parser.IntegerLit{ValuePos: pos(42), Value: "2"}, + Rparen: pos(43), }, }, }, diff --git a/sql3/parser/walk.go b/sql3/parser/walk.go index 10701c8e2..8dfc04b8c 100644 --- a/sql3/parser/walk.go +++ b/sql3/parser/walk.go @@ -716,22 +716,13 @@ func walk(v Visitor, node Node) (_ Node, err error) { if err := walkIdent(v, &n.Name); err != nil { return node, err } - if n.Precision != nil { - if p, err := walk(v, n.Precision); err != nil { - return node, err - } else if p != nil { - n.Precision = p.(*IntegerLit) - } else { - n.Precision = nil - } - } - if n.Scale != nil { - if scale, err := walk(v, n.Scale); err != nil { + if n.Modifier != nil { + if scale, err := walk(v, n.Modifier); err != nil { return node, err } else if scale != nil { - n.Scale = scale.(*IntegerLit) + n.Modifier = scale.(*IntegerLit) } else { - n.Scale = nil + n.Modifier = nil } } } diff --git a/sql3/planner/compilebulkinsert.go b/sql3/planner/compilebulkinsert.go index 3f42fb4f8..9675a1057 100644 --- a/sql3/planner/compilebulkinsert.go +++ b/sql3/planner/compilebulkinsert.go @@ -114,7 +114,7 @@ func (p *ExecutionPlanner) compileBulkInsertStatement(ctx context.Context, stmt for _, m := range stmt.Columns { for idx, fld := range tbl.Fields { if strings.EqualFold(string(fld.Name), m.Name) { - options.targetColumns = append(options.targetColumns, newQualifiedRefPlanExpression(tableName, strings.ToLower(m.Name), idx, fieldSQLDataType(pilosa.FieldToFieldInfo(fld)))) + options.targetColumns = append(options.targetColumns, newQualifiedRefPlanExpression(tableName, strings.ToLower(m.Name), idx, FieldSQLDataType(pilosa.FieldToFieldInfo(fld)))) break } } @@ -348,7 +348,7 @@ func (p *ExecutionPlanner) analyzeBulkInsertStatement(ctx context.Context, stmt for _, fld := range tbl.Fields { if strings.EqualFold(cm.Name, string(fld.Name)) { found = true - colDataType := fieldSQLDataType(pilosa.FieldToFieldInfo(fld)) + colDataType := FieldSQLDataType(pilosa.FieldToFieldInfo(fld)) // if we have transforms check that type and target colum ref are assignment compatible // else check that the map expressions type and target column ref are assignment compatible diff --git a/sql3/planner/compilecreatetable.go b/sql3/planner/compilecreatetable.go index c079a54f3..4d2ef91e4 100644 --- a/sql3/planner/compilecreatetable.go +++ b/sql3/planner/compilecreatetable.go @@ -93,6 +93,7 @@ func (p *ExecutionPlanner) compileColumn(ctx context.Context, col *parser.Column var cacheType string = pilosa.DefaultCacheType var cacheSize uint32 = pilosa.DefaultCacheSize var scale int64 + var length int64 min, max := pql.MinMax(0) var epoch = pilosa.DefaultEpoch var timeUnit string = pilosa.TimeUnitSeconds @@ -190,11 +191,11 @@ func (p *ExecutionPlanner) compileColumn(ctx context.Context, col *parser.Column case dax.BaseTypeDecimal: // if we don't have a scale, it's an error - if col.Type.Scale == nil { + if col.Type.Modifier == nil { return nil, sql3.NewErrDecimalScaleExpected(col.Type.Name.NamePos.Line, col.Type.Name.NamePos.Column) } // get the scale value - scale, err = strconv.ParseInt(col.Type.Scale.Value, 10, 64) + scale, err = strconv.ParseInt(col.Type.Modifier.Value, 10, 64) if err != nil { return nil, err } @@ -235,6 +236,19 @@ func (p *ExecutionPlanner) compileColumn(ctx context.Context, col *parser.Column case dax.BaseTypeTimestamp: column.fos = append(column.fos, pilosa.OptFieldTypeTimestamp(epoch, timeUnit)) + case dax.BaseTypeVarchar: + // if we don't have a length, it's an error + if col.Type.Modifier == nil { + return nil, sql3.NewErrVarcharLengthExpected(col.Type.Name.NamePos.Line, col.Type.Name.NamePos.Column) + } + + // get the modifier value + length, err = strconv.ParseInt(col.Type.Modifier.Value, 10, 64) + if err != nil { + return nil, err + } + + column.fos = append(column.fos, pilosa.OptFieldTypeVarchar(length)) } return column, nil } diff --git a/sql3/planner/compileinsert.go b/sql3/planner/compileinsert.go index 1a0a9e51f..7c77a9dcf 100644 --- a/sql3/planner/compileinsert.go +++ b/sql3/planner/compileinsert.go @@ -40,7 +40,7 @@ func (p *ExecutionPlanner) compileInsertStatement(ctx context.Context, stmt *par for idx, field := range tbl.Fields { if strings.EqualFold(colName, string(field.Name)) { - targetColumns = append(targetColumns, newQualifiedRefPlanExpression(tableName, colName, idx, fieldSQLDataType(pilosa.FieldToFieldInfo(field)))) + targetColumns = append(targetColumns, newQualifiedRefPlanExpression(tableName, colName, idx, FieldSQLDataType(pilosa.FieldToFieldInfo(field)))) break } } @@ -50,7 +50,7 @@ func (p *ExecutionPlanner) compileInsertStatement(ctx context.Context, stmt *par if strings.EqualFold("_exists", string(field.Name)) { continue } - targetColumns = append(targetColumns, newQualifiedRefPlanExpression(tableName, string(field.Name), idx, fieldSQLDataType(pilosa.FieldToFieldInfo(field)))) + targetColumns = append(targetColumns, newQualifiedRefPlanExpression(tableName, string(field.Name), idx, FieldSQLDataType(pilosa.FieldToFieldInfo(field)))) } } @@ -94,7 +94,7 @@ func (p *ExecutionPlanner) analyzeInsertStatement(ctx context.Context, stmt *par if strings.EqualFold("_exists", string(field.Name)) { continue } - typeNames = append(typeNames, fieldSQLDataType(pilosa.FieldToFieldInfo(field))) + typeNames = append(typeNames, FieldSQLDataType(pilosa.FieldToFieldInfo(field))) } // Make sure (implicit) insert list and expression list have the same // number of items. @@ -131,7 +131,7 @@ func (p *ExecutionPlanner) analyzeInsertStatement(ctx context.Context, stmt *par columnFound := false for _, field := range tbl.Fields { if strings.EqualFold(colName, string(field.Name)) { - typeName = fieldSQLDataType(pilosa.FieldToFieldInfo(field)) + typeName = FieldSQLDataType(pilosa.FieldToFieldInfo(field)) columnFound = true break } diff --git a/sql3/planner/compileselect.go b/sql3/planner/compileselect.go index 544efa2d8..d35c8e995 100644 --- a/sql3/planner/compileselect.go +++ b/sql3/planner/compileselect.go @@ -588,7 +588,7 @@ func (p *ExecutionPlanner) analyzeSource(ctx context.Context, source parser.Sour TableName: objectName, ColumnName: string(fld.Name), ColumnIndex: i, - Datatype: fieldSQLDataType(pilosa.FieldToFieldInfo(fld)), + Datatype: FieldSQLDataType(pilosa.FieldToFieldInfo(fld)), } source.OutputColumns = append(source.OutputColumns, soc) } diff --git a/sql3/planner/compileshow.go b/sql3/planner/compileshow.go index fb3748727..129f00cc0 100644 --- a/sql3/planner/compileshow.go +++ b/sql3/planner/compileshow.go @@ -94,9 +94,9 @@ func (p *ExecutionPlanner) compileShowTablesStatement(ctx context.Context, stmt columns := []types.PlanExpression{ &qualifiedRefPlanExpression{ tableName: "fb_tables", - columnName: string(dax.PrimaryKeyFieldName), + columnName: "onject_id", columnIndex: 0, - dataType: parser.NewDataTypeString(), + dataType: parser.NewDataTypeInt(), }, &qualifiedRefPlanExpression{ tableName: "fb_tables", @@ -203,33 +203,38 @@ func (p *ExecutionPlanner) compileShowColumnsStatement(ctx context.Context, stmt dataType: parser.NewDataTypeInt(), }, &qualifiedRefPlanExpression{ tableName: "fb_table_columns", - columnName: "min", + columnName: "length", columnIndex: 8, dataType: parser.NewDataTypeInt(), }, &qualifiedRefPlanExpression{ tableName: "fb_table_columns", - columnName: "max", + columnName: "min", columnIndex: 9, dataType: parser.NewDataTypeInt(), }, &qualifiedRefPlanExpression{ tableName: "fb_table_columns", - columnName: "timeunit", + columnName: "max", columnIndex: 10, + dataType: parser.NewDataTypeInt(), + }, &qualifiedRefPlanExpression{ + tableName: "fb_table_columns", + columnName: "timeunit", + columnIndex: 11, dataType: parser.NewDataTypeString(), }, &qualifiedRefPlanExpression{ tableName: "fb_table_columns", columnName: "epoch", - columnIndex: 11, + columnIndex: 12, dataType: parser.NewDataTypeInt(), }, &qualifiedRefPlanExpression{ tableName: "fb_table_columns", columnName: "timequantum", - columnIndex: 12, + columnIndex: 13, dataType: parser.NewDataTypeString(), }, &qualifiedRefPlanExpression{ tableName: "fb_table_columns", columnName: "ttl", - columnIndex: 13, + columnIndex: 14, dataType: parser.NewDataTypeString(), }} diff --git a/sql3/planner/expressiontypes.go b/sql3/planner/expressiontypes.go index b7ee9b51e..3d2df6b18 100644 --- a/sql3/planner/expressiontypes.go +++ b/sql3/planner/expressiontypes.go @@ -13,7 +13,7 @@ import ( ) // takes a *pilosa.FieldInfo and returns a sql data type -func fieldSQLDataType(f *pilosa.FieldInfo) parser.ExprDataType { +func FieldSQLDataType(f *pilosa.FieldInfo) parser.ExprDataType { // This is special handling for the primary key (_id) field. The normal // handling below was not well suited to this field because there isn't a // `pilosa.FieldTypeID` to compare against. One option would have been to @@ -68,6 +68,9 @@ func fieldSQLDataType(f *pilosa.FieldInfo) parser.ExprDataType { case pilosa.FieldTypeTimestamp: return parser.NewDataTypeTimestamp() + case pilosa.FieldTypeVarchar: + return parser.NewDataTypeVarchar(f.Options.Length) + default: return parser.NewDataTypeVoid() } @@ -81,10 +84,10 @@ func dataTypeFromParserType(typ *parser.Type) (parser.ExprDataType, error) { return parser.NewDataTypeBool(), nil case dax.BaseTypeDecimal: - if typ.Scale == nil { + if typ.Modifier == nil { return nil, sql3.NewErrDecimalScaleExpected(typ.Name.NamePos.Line, typ.Name.NamePos.Column) } - scale, err := strconv.Atoi(typ.Scale.Value) + scale, err := strconv.Atoi(typ.Modifier.Value) if err != nil { return nil, err } @@ -358,6 +361,19 @@ func typesAreAssignmentCompatible(targetType parser.ExprDataType, sourceType par case *parser.DataTypeString: switch sourceType.(type) { + case *parser.DataTypeString: + return true + case *parser.DataTypeVarchar: + return true + default: + return false + } + + case *parser.DataTypeVarchar: + switch rhs := sourceType.(type) { + case *parser.DataTypeVarchar: + //if lhs length is >= rhs length, we're good + return lhs.Length >= rhs.Length case *parser.DataTypeString: return true default: diff --git a/sql3/planner/opfeaturebasecolumns.go b/sql3/planner/opfeaturebasecolumns.go index ec9b4e412..760d879b6 100644 --- a/sql3/planner/opfeaturebasecolumns.go +++ b/sql3/planner/opfeaturebasecolumns.go @@ -87,6 +87,11 @@ func (p *PlanOpFeatureBaseColumns) Schema() types.Schema { ColumnName: "scale", Type: parser.NewDataTypeInt(), }, + &types.PlannerColumn{ + RelationName: "fb_table_columns", + ColumnName: "length", + Type: parser.NewDataTypeInt(), + }, &types.PlannerColumn{ RelationName: "fb_table_columns", ColumnName: "min", @@ -156,6 +161,7 @@ func (i *showColumnsRowIter) Next(ctx context.Context) (types.Row, error) { fields[i.rowIndex].Options.CacheType, fields[i.rowIndex].Options.CacheSize, fields[i.rowIndex].Options.Scale, + fields[i.rowIndex].Options.Length, fields[i.rowIndex].Options.Min.ToInt64(0), fields[i.rowIndex].Options.Max.ToInt64(0), fields[i.rowIndex].Options.TimeUnit, diff --git a/sql3/planner/opfeaturebasetables.go b/sql3/planner/opfeaturebasetables.go index 99d50c973..59b3d27d6 100644 --- a/sql3/planner/opfeaturebasetables.go +++ b/sql3/planner/opfeaturebasetables.go @@ -9,7 +9,6 @@ import ( "time" pilosa "github.com/featurebasedb/featurebase/v3" - "github.com/featurebasedb/featurebase/v3/dax" "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" ) @@ -55,8 +54,8 @@ func (p *PlanOpFeatureBaseTables) Schema() types.Schema { return types.Schema{ &types.PlannerColumn{ RelationName: "fb_tables", - ColumnName: string(dax.PrimaryKeyFieldName), - Type: parser.NewDataTypeString(), + ColumnName: "object_id", + Type: parser.NewDataTypeInt(), }, &types.PlannerColumn{ RelationName: "fb_tables", diff --git a/sql3/planner/opinsert.go b/sql3/planner/opinsert.go index f7512456f..b7e86b346 100644 --- a/sql3/planner/opinsert.go +++ b/sql3/planner/opinsert.go @@ -421,6 +421,14 @@ func (i *insertRowIter) Next(ctx context.Context) (types.Row, error) { return nil, sql3.NewErrInternalf("unsupported timestamp type: %T", eval) } + case pilosa.FieldTypeVarchar: + switch v := eval.(type) { + case string: + row.Values[posVals[idx]] = v + default: + return nil, sql3.NewErrInternalf("unexpected varchar type '%T'", v) + } + default: row.Values[posVals[idx]] = eval } diff --git a/sql3/planner/oppqldistinctscan.go b/sql3/planner/oppqldistinctscan.go index 47a2a6a4f..db3d795c2 100644 --- a/sql3/planner/oppqldistinctscan.go +++ b/sql3/planner/oppqldistinctscan.go @@ -98,7 +98,7 @@ func (p *PlanOpPQLDistinctScan) Schema() types.Schema { result = append(result, &types.PlannerColumn{ ColumnName: string(fld.Name), RelationName: p.tableName, - Type: fieldSQLDataType(pilosa.FieldToFieldInfo(fld)), + Type: FieldSQLDataType(pilosa.FieldToFieldInfo(fld)), }) break } @@ -158,7 +158,7 @@ func (i *distinctScanRowIter) Next(ctx context.Context) (types.Row, error) { for _, fld := range table.Fields { if strings.EqualFold(i.column, string(fld.Name)) { - i.columnDataType = fieldSQLDataType(pilosa.FieldToFieldInfo(fld)) + i.columnDataType = FieldSQLDataType(pilosa.FieldToFieldInfo(fld)) break } } diff --git a/sql3/planner/oppqltablescan.go b/sql3/planner/oppqltablescan.go index 739524d18..79323c491 100644 --- a/sql3/planner/oppqltablescan.go +++ b/sql3/planner/oppqltablescan.go @@ -109,7 +109,7 @@ func (p *PlanOpPQLTableScan) Schema() types.Schema { result = append(result, &types.PlannerColumn{ ColumnName: string(fld.Name), RelationName: p.tableName, - Type: fieldSQLDataType(pilosa.FieldToFieldInfo(fld)), + Type: FieldSQLDataType(pilosa.FieldToFieldInfo(fld)), }) break } @@ -198,7 +198,7 @@ func (i *tableScanRowIter) Next(ctx context.Context) (types.Row, error) { columnIdx: idx, srcColumnIdx: -1, columnName: string(fld.Name), - dataType: fieldSQLDataType(pilosa.FieldToFieldInfo(fld)), + dataType: FieldSQLDataType(pilosa.FieldToFieldInfo(fld)), } break } diff --git a/sql3/planner/planoptimizer.go b/sql3/planner/planoptimizer.go index fe73539eb..59f18d930 100644 --- a/sql3/planner/planoptimizer.go +++ b/sql3/planner/planoptimizer.go @@ -28,6 +28,7 @@ type OptimizerFunc func(context.Context, *ExecutionPlanner, types.PlanOperator, // a list of optimzer rules; order can be important important var optimizerFunctions = []OptimizerFunc{ + // fix expression references for having removeUnusedExtractColumnReferences, @@ -1391,7 +1392,7 @@ func fixFieldRefIndexesForHaving(ctx context.Context, scope *OptimizerScope, a * *percentilePlanExpression: for i, col := range schema { if strings.EqualFold(typedExpr.String(), col.ColumnName) { - e := newQualifiedRefPlanExpression("", "", i, typedExpr.Type()) + e := newQualifiedRefPlanExpression("", col.ColumnName, i, typedExpr.Type()) return e, false, nil } } diff --git a/tstore/btree.go b/tstore/btree.go new file mode 100644 index 000000000..6dbf93f24 --- /dev/null +++ b/tstore/btree.go @@ -0,0 +1,1172 @@ +// Copyright 2023 Molecula Corp. All rights reserved. + +package tstore + +import ( + "bytes" + "fmt" + "sync" + + "github.com/featurebasedb/featurebase/v3/bufferpool" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" + "github.com/featurebasedb/featurebase/v3/wireprotocol" + "github.com/pkg/errors" +) + +// TODO(pok) +// ▶ on create/alter table, update the schema version +// ▶ on open index run the aries recovery + +// ▶ move latchState into Page ✅ +// ▶ latch buffer pool ✅ +// ▶ schema versioning on the page 🔧 +// ▶ write initial schema version on the root page ✅ +// ▶ write new schema versions on the root page +// ▶ handle schema version overflow + +// ▶ put the insert into the split routine, so we don't have the do the insert after the fact + +// - [x] Tuple format +// - [ ] Tid (int64) - transaction id of last write +// - [ ] Redo page ptr - pointer to last version of row +// - [ ] schema version (int16) +// - [ ] count of fields (int16) +// - [ ] array of field offsets +// - [ ] offset for each field (-1 for null) +// - [ ] data payload +// - [ ] for each column (len, only for variable columns) data + +// - math that that works out max payload length for a tuple +// - tries to key fanout min >= 4 + +// ▶ WAL +// ▶ implement log buffers for wal files +// ▶ implement write log event for PageID, event id etc. + +// ▶ lazy writer on the buffer pool +// ▶ implement a checkpoint that scans the pool and writes out dirty pages every minute or so + +// ▶ MVCC versioning +// ▶ 3000+ columns (thanks Q2) + +// ▶ handle nulls on insert +// ▶ backup/restore + +// later + +// ▶ do a test on concurrent inserts +// ▶ latch buffer I/Os + +// BTree represents a b+tree structure used for storing and +// retrieving tuple data for a given shard in a table +type BTree struct { + mu sync.RWMutex + schema types.Schema + schemaVersion int + + keysPerLeafPage int64 + keysPerInternalPage int64 + + objectID int32 + shard int32 + rootNode bufferpool.PageID + bufferpool *bufferpool.BufferPool +} + +func NewBTree(maxKeySize int, objectID int32, shard int32, schema types.Schema, bpool *bufferpool.BufferPool) (*BTree, error) { + // use key size to calculate keys per leaf and internal page + if maxKeySize > bufferpool.MAX_KEY_ON_PAGE_SIZE { + return nil, errors.Errorf("max key size exceeded") + } + + sizeWithoutHeader := bufferpool.PAGE_SIZE - bufferpool.PAGE_SLOTS_START_OFFSET + + // for internal pages chunk size is: + // keyLength 2 + // keyBytes maxKeySize + // ptrValue 8 + chunkSize := int64(2 + maxKeySize + 8 + bufferpool.PAGE_SLOT_LENGTH) + keysPerInternalPage := sizeWithoutHeader / (bufferpool.PAGE_SLOT_LENGTH + chunkSize) + + // for leaf pages chunk size is: + // keyLength 2 + // keyBytes maxKeySize + // rowPayloadLen 4 + // rowPayloadBytes [payload length] + + // get the payload length from the schema + payLoadLength := 0 + for _, s := range schema { + switch ty := s.Type.(type) { + case *parser.DataTypeVarchar: + payLoadLength += 4 // offset or null + payLoadLength += 4 + int(ty.Length) // actual data + default: + return nil, errors.Errorf("unsupported t-store data type '%T'", ty) + } + } + + chunkSize = int64(2 + maxKeySize + 1 + 2 + payLoadLength) + keysPerLeafPage := int64(4) + if payLoadLength <= bufferpool.MAX_PAYLOAD_CHUNK_SIZE_ON_PAGE { + keysPerLeafPage = sizeWithoutHeader / (bufferpool.PAGE_SLOT_LENGTH + chunkSize) + } + + tree := &BTree{ + objectID: objectID, + shard: shard, + bufferpool: bpool, + keysPerLeafPage: keysPerLeafPage, + keysPerInternalPage: keysPerInternalPage, + } + + headerNode, err := tree.fetchNode(bufferpool.PageID{ObjectID: objectID, Shard: shard, Page: 0}) + if err != nil { + return nil, err + } + headerNode.takeWriteLatch() + defer headerNode.releaseWriteLatch() + defer tree.unpin(headerNode) + + slot := headerNode.page.ReadPageSlot(0) + // no need to protect updating this with a RWMutex yet + ipl := slot.InternalPayload(headerNode.page) + tree.rootNode = ipl.ValueAsPagePointer(headerNode.page) + + // see if the headerPage is in overflow + nextHeader := headerNode.page.ReadNextPointer() + if nextHeader.Page != bufferpool.INVALID_PAGE { + // right now overflow is an error + return nil, errors.Errorf("headerPage overflow") + } + // get the slot count from the headerPage + slotCount := headerNode.page.ReadSlotCount() + if slotCount > 1 { + // we have schema versions + slot = headerNode.page.ReadPageSlot(slotCount - 1) + + pl := slot.KeyPayload(headerNode.page) + lpl := slot.LeafPayload(headerNode.page) + + latestVersion := int(pl.KeyAsInt(headerNode.page)) + + b := lpl.ValueAsBytes(headerNode.page) + + rdr := bytes.NewReader(b) + _, err = wireprotocol.ExpectToken(rdr, wireprotocol.TOKEN_SCHEMA_INFO) + if err != nil { + return nil, err + } + + s, err := wireprotocol.ReadSchema(rdr) + if err != nil { + return nil, err + } + + newSchema := false + if len(s) != len(schema) { + newSchema = true + } else { + for i, c := range s { + if schema[i].ColumnName != c.ColumnName || schema[i].Type.BaseTypeName() != c.Type.BaseTypeName() { + newSchema = true + break + } + } + } + if newSchema { + return nil, errors.Errorf("schema mismatch") + } + tree.schemaVersion = latestVersion + tree.schema = schema + } else { + // no schema versions so write first one + b, err := wireprotocol.WriteSchema(schema) + if err != nil { + return nil, err + } + + err = tree.writeLeafEntryInSlot(headerNode, 1, Int(1).Bytes(), b) + if err != nil { + return nil, err + } + headerNode.page.WriteSlotCount(2) + + tree.schemaVersion = 1 + tree.schema = schema + tree.bufferpool.FlushPage(headerNode.page.ID()) + } + return tree, nil +} + +// protocol for latching +// ▶ latch parent node +// ▶ get latch for childNode +// ▶ Release latch for parent if “safe”. +// • A safe node is one that will not split or merge when updated. +// ▶ Not full (on insertion) +// ▶ More than half-full (on deletion) +// +// + +//Latching for Search --> start at root with a read latch and go down; repeatedly, +// ▶ Acquire read latch on child +// ▶ Then unlatch parent + +func (b *BTree) Search(currentNode *BTreeNode, k Sortable) (Sortable, *BTreeTuple) { + if currentNode != nil { + if currentNode.isLeaf() { + defer currentNode.releaseReadLatch() + defer b.unpin(currentNode) + // search for the key + i, found := currentNode.findKey(k) + if found { + slot := currentNode.page.ReadPageSlot(int16(i)) + lpl := slot.LeafPayload(currentNode.page) + return k, NewBTreeTupleFromBytes(lpl.ValueAsBytes(currentNode.page), b.schema) + } + return nil, nil + } else { + nodePtr := b.findNextPointer(currentNode, k) + node, _ := b.fetchNode(nodePtr) + node.takeReadLatch() + currentNode.releaseReadLatch() + b.unpin(currentNode) + return b.Search(node, k) + } + } else { + n, _ := b.fetchNode(b.rootNode) + n.takeReadLatch() + return b.Search(n, k) + } +} + +//Latching for Insert --> start at root and go down, start at root with a read latch and go down; repeatedly, +// ▶ latch parent node +// ▶ get latch for childNode +// ▶ if childNode is a leaf and will split and we only have a read latch, bail and start from the top in exclusive mode +// ▶ release latch for parent if “safe”. +// • A safe node is one that will not split or merge when updated. +// ▶ Not full (on insertion) + +func (b *BTree) Insert(tup *BTreeTuple) error { + // go get the root page from the buffer pool + node, err := b.fetchNode(b.rootNode) + if err != nil { + return err + } + + key := tup.keyValue() + + // special handling for the case where root is a leaf + // if it is a leaf, we are always going to be writing + // here so take a write latch + if node.isLeaf() { + node.takeWriteLatch() + } else { + node.takeReadLatch() + } + + forceExclusive := false + for { + // does the root need to split? + if b.isNodeFull(node) { + + // to do a split, we need a write latch so check to see if we have one + // if not, we need to retry in exclusive mode + if node.latchState() != bufferpool.Write { + // release the read latch & take a write latch + node.releaseAnyLatch() + node.takeWriteLatch() + // retry in exclusive mode + forceExclusive = true + continue + } + + //split the node + lhs, pivot, rhs, err := b.splitNode(node) + if err != nil { + return err + } + lhsPtr := lhs.page.ID() + rhsPtr := rhs.page.ID() + + // decide which of the node to do the pending insert into + // and release write latch an unpin on the other node + var n *BTreeNode + if key.Less(pivot) { + n = lhs + rhs.releaseWriteLatch() + b.unpin(rhs) + } else { + n = rhs + lhs.releaseWriteLatch() + b.unpin(lhs) + } + + // do the insert into the node + err = b.insertNonFull(n, key, tup, forceExclusive) + if err != nil { + return err + } + + // this is the root node splitting so handle that... + err = b.handleRootNodeSplit(pivot, lhsPtr, rhsPtr) + if err != nil { + return err + } + + return nil + } else { + err := b.insertNonFull(node, key, tup, forceExclusive) + if err != nil { + if err == ErrNeedsExclusive { + // release the read latch & take a write latch + node.releaseAnyLatch() + node.takeWriteLatch() + // retry in exclusive mode + forceExclusive = true + continue + } + return err + } + return nil + } + } +} + +// private methods + +func (b *BTree) fetchNode(pageID bufferpool.PageID) (*BTreeNode, error) { + page, err := b.bufferpool.FetchPage(pageID) + if err != nil { + return nil, err + } + node := &BTreeNode{ + page: page, + } + return node, nil +} + +func (b *BTree) unpin(node *BTreeNode) error { + return b.bufferpool.UnpinPage(node.page.ID()) +} + +func (b *BTree) newOverflow() (*BTreeNode, error) { + page, err := b.bufferpool.NewPage(b.objectID, b.shard) + if err != nil { + return nil, err + } + page.WritePageType(int16(bufferpool.PAGE_TYPE_BTREE_OVERFLOW)) + node := &BTreeNode{ + page: page, + } + return node, nil +} + +func (b *BTree) newLeaf() (*BTreeNode, error) { + page, err := b.bufferpool.NewPage(b.objectID, b.shard) + if err != nil { + return nil, err + } + page.WritePageType(int16(bufferpool.PAGE_TYPE_BTREE_LEAF)) + node := &BTreeNode{ + page: page, + } + return node, nil +} + +func (b *BTree) newInternal() (*BTreeNode, error) { + page, err := b.bufferpool.NewPage(b.objectID, b.shard) + if err != nil { + return nil, err + } + page.WritePageType(int16(bufferpool.PAGE_TYPE_BTREE_INTERNAL)) + node := &BTreeNode{ + page: page, + } + return node, nil +} + +func (b *BTree) isNodeFull(n *BTreeNode) bool { + if n.latchState() == bufferpool.None { + panic("unexpected latch state") + } + + if n.isLeaf() { + sc := n.slotCount() + if sc >= int(b.keysPerLeafPage) { + return true + } + } else { + sc := n.slotCount() + if sc >= int(b.keysPerInternalPage) { + return true + } + } + return false +} + +func (b *BTree) findNextPointer(node *BTreeNode, key Sortable) bufferpool.PageID { + slotCount := int(node.page.ReadSlotCount()) + + minIndex := 0 + onePastMaxIndex := slotCount + for onePastMaxIndex != minIndex { + index := (minIndex + onePastMaxIndex) / 2 + slot := node.page.ReadPageSlot(int16(index)) + pl := slot.KeyPayload(node.page) + keyAtIndex := Int(pl.KeyAsInt(node.page)) + if key.Less(keyAtIndex) { + onePastMaxIndex = index + } else { + minIndex = index + 1 + } + } + if minIndex == slotCount { + // we didn't find it so return the next pointer + nextPtr := node.page.ReadNextPointer() + return nextPtr + } else { + slot := node.page.ReadPageSlot(int16(minIndex)) + ipl := slot.InternalPayload(node.page) + nextPtr := ipl.ValueAsPagePointer(node.page) + return nextPtr + } +} + +func (b *BTree) setRootNode(newRootNode bufferpool.PageID) error { + b.mu.Lock() + defer b.mu.Unlock() + + // get the header node + headerNode, err := b.fetchNode(bufferpool.PageID{ObjectID: b.objectID, Shard: b.shard, Page: 0}) + if err != nil { + return err + } + headerNode.takeWriteLatch() + defer headerNode.releaseWriteLatch() + defer b.unpin(headerNode) + + // root page pointer is in slot 0 + slot := headerNode.page.ReadPageSlot(0) + + ipl := slot.InternalPayload(headerNode.page) + + // update the root page pointer + ipl.PutPagePointer(headerNode.page, newRootNode) + + b.bufferpool.FlushPage(headerNode.page.ID()) + + b.rootNode = newRootNode + return nil +} + +func (b *BTree) handleRootNodeSplit(pivot Sortable, lhsPtr bufferpool.PageID, rhsPtr bufferpool.PageID) error { + // create a new root node + newRoot, err := b.newInternal() + if err != nil { + return err + } + newRoot.takeWriteLatch() + defer newRoot.releaseWriteLatch() + defer b.unpin(newRoot) + + // add the pivot key pointing to the old root page + b.insertInternalEntryAt(newRoot, 0, pivot, lhsPtr) + + // set the next ptr to point to newNode + newRoot.page.WriteNextPointer(rhsPtr) + + b.setRootNode(newRoot.page.ID()) + return nil +} + +func (b *BTree) compactLeafPage(node *BTreeNode) error { + return nil +} + +// this function handles overflow +func (b *BTree) writeLeafEntryInSlot(node *BTreeNode, slotNumber int16, keyBytes []byte, payloadBytes []byte) error { + + keyLength := len(keyBytes) + payloadChunkLength := len(payloadBytes) + + payloadTotalLength := payloadChunkLength + + flags := 0 + + if payloadChunkLength > bufferpool.MAX_PAYLOAD_CHUNK_SIZE_ON_PAGE { + // we need to overflow to a new page, so set the fact we have to overflow + flags = 1 + // cap the write size + payloadChunkLength = bufferpool.MAX_PAYLOAD_CHUNK_SIZE_ON_PAGE + } + + // double check we won't blow free space on page + onPageSize := node.page.ComputeLeafPayloadTotalLength(keyLength, payloadChunkLength) + if (onPageSize + bufferpool.PAGE_SLOT_LENGTH) > int32(node.page.FreeSpaceOnPage()) { + err := b.compactLeafPage(node) + if err != nil { + return err + } + // check again + if (onPageSize + bufferpool.PAGE_SLOT_LENGTH) > int32(node.page.FreeSpaceOnPage()) { + // this shouldn't happen - if it does we have a logic error + // in our assumptions about how many keys fit on a page + // or saggitarius is rising in scorpio, or somesuch + panic("page is full") + } + } + + // get the current freespace offset + freeSpaceOffset := node.page.ReadFreeSpaceOffset() + + // compute the new free space offset for this page + freeSpaceOffset -= int16(onPageSize) + offset := freeSpaceOffset + + if flags == 1 { + // we are going to overflow so allocate an overflow page + overflowPage, err := b.newOverflow() + if err != nil { + return err + } + overflowPage.takeWriteLatch() + defer overflowPage.releaseWriteLatch() + defer b.unpin(overflowPage) + + bytesRemaining := payloadTotalLength + lowWater := 0 + hiWater := payloadChunkLength + // write the data on this page + offset = node.page.WriteLeafPagePayloadHeader(offset, int16(keyLength), keyBytes, int8(flags), overflowPage.page.ID().Page, int32(payloadTotalLength)) + node.page.WriteLeafPagePayloadBytes(offset, int16(payloadChunkLength), payloadBytes[lowWater:hiWater]) + + // now we've written payloadChunkLen bytes of payload to node.page, now write to overflow page + bytesRemaining -= payloadChunkLength + + for bytesRemaining > 0 { + overflowFreeSpace := int(overflowPage.page.FreeSpaceOnPage()) + + var err error + var nextOverflowPage *BTreeNode + nextOverflowPtr := int64(0) + if bytesRemaining > overflowFreeSpace { + // we're gonna need another overflow page + nextOverflowPage, err = b.newOverflow() + if err != nil { + return err + } + nextOverflowPage.takeWriteLatch() + defer nextOverflowPage.releaseWriteLatch() + defer b.unpin(nextOverflowPage) + nextOverflowPtr = int64(nextOverflowPage.page.ID().Page) + } + + lowWater = hiWater + // set the payload chunk length to the free space on the page + // less the 2 byte chunk length + payloadChunkLength := overflowFreeSpace - 2 + hiWater += payloadChunkLength + + if nextOverflowPtr > 0 { + overflowPage.page.WriteNextPointer(bufferpool.PageID{ObjectID: b.objectID, Shard: b.shard, Page: nextOverflowPtr}) + } + + overflowPage.page.WriteLeafPagePayloadBytes(bufferpool.PAGE_SLOTS_START_OFFSET, int16(payloadChunkLength), payloadBytes[lowWater:hiWater]) + bytesRemaining -= payloadChunkLength + overflowPage = nextOverflowPage + } + + } else { + offset = node.page.WriteLeafPagePayloadHeader(offset, int16(keyLength), keyBytes, int8(flags), 0, int32(payloadTotalLength)) + node.page.WriteLeafPagePayloadBytes(offset, int16(payloadChunkLength), payloadBytes) + } + + // update the free space offset on this page + node.page.WriteFreeSpaceOffset(int16(freeSpaceOffset)) + + // make a slot and write it + slot := bufferpool.PageSlot{ + PayloadOffset: freeSpaceOffset, + } + node.page.WritePageSlot(slotNumber, slot) + return nil +} + +func (b *BTree) insertLeafEntryAt(node *BTreeNode, i int, key Sortable, tup *BTreeTuple) error { + if node.latchState() != bufferpool.Write { + panic("unexpected latch state") + } + + // get the slot count + slotCount := int(node.page.ReadSlotCount()) + + // move all the slots after where we are going to insert + for j := slotCount; j > i; j-- { + sl := node.page.ReadPageSlot(int16(j - 1)) + node.page.WritePageSlot(int16(j), sl) + } + + // put the payload together + valueData, err := tup.Bytes() + if err != nil { + return err + } + + // write to WAL here before we write to page!!! + + err = b.writeLeafEntryInSlot(node, int16(i), key.Bytes(), valueData) + if err != nil { + return err + } + + // update the slot count + slotCount++ + node.page.WriteSlotCount(int16(slotCount)) + + return nil +} + +func (b *BTree) writeInternalEntryInSlot(node *BTreeNode, slotNumber int16, keyBytes []byte, ptrValue int64) error { + + // get the current freespace offset + freeSpaceOffset := node.page.ReadFreeSpaceOffset() + + keyLength := len(keyBytes) + + onPageSize := node.page.ComputeInternalPayloadTotalLength(keyLength) + + // compute the new free space offset for this page + freeSpaceOffset -= int16(onPageSize) + + node.page.WriteInternalPagePayload(freeSpaceOffset, int16(keyLength), keyBytes, ptrValue) + + // update the free space offset on this page + node.page.WriteFreeSpaceOffset(int16(freeSpaceOffset)) + + // make a slot and write it + slot := bufferpool.PageSlot{ + PayloadOffset: freeSpaceOffset, + } + node.page.WritePageSlot(slotNumber, slot) + return nil +} + +func (b *BTree) insertInternalEntryAt(node *BTreeNode, i int, key Sortable, pageID bufferpool.PageID) error { + if node.latchState() != bufferpool.Write { + panic("unexpected latch state") + } + + // get the slot count + slotCount := int(node.page.ReadSlotCount()) + + // move all the slots after where we are going to insert + for j := slotCount; j > i; j-- { + sl := node.page.ReadPageSlot(int16(j - 1)) + node.page.WritePageSlot(int16(j), sl) + } + + err := b.writeInternalEntryInSlot(node, int16(i), key.Bytes(), pageID.Page) + if err != nil { + return err + } + + // update the slot count + slotCount++ + node.page.WriteSlotCount(int16(slotCount)) + + return nil +} + +func (b *BTree) updatePointerEntryAt(node *BTreeNode, i int, value bufferpool.PageID) error { + slot := node.page.ReadPageSlot(int16(i)) + ipl := slot.InternalPayload(node.page) + return ipl.PutPagePointer(node.page, value) +} + +func (b *BTree) splitNode(nodeToSplit *BTreeNode) (*BTreeNode, Sortable, *BTreeNode, error) { + // TODO(pok) handle the sitch when inserting the new key and it ends up as the min key in rhs + if nodeToSplit.isLeaf() { + return b.splitLeafNode(nodeToSplit) + } else { + return b.splitInternalNode(nodeToSplit) + } +} + +func (b *BTree) insertNonFull(node *BTreeNode, key Sortable, tup *BTreeTuple, forceExclusive bool) error { + if node.isLeaf() { + // fmt.Printf("leaf insert on page %v (%v)\n", node.page.ID(), tup) + + if node.latchState() != bufferpool.Write { + panic("unexpected latch state") + } + + defer node.releaseWriteLatch() + defer b.unpin(node) + + i, exists := node.findKey(key) + if exists { + return errors.Errorf("key violation") + } + + err := b.insertLeafEntryAt(node, i, key, tup) + if err != nil { + return err + } + + return nil + } else { + // its an internal node so follow the pointers + childPtr, err := node.findNextPointer(key, b.objectID, b.shard) + if err != nil { + return err + } + childNode, err := b.fetchNode(childPtr) + if err != nil { + return err + } + + // fmt.Printf("internal node search on page %v: key=%v, childptr=%v\n", node.page.ID(), key, childPtr) + + // we need to latch child node + if forceExclusive { + childNode.takeWriteLatch() + } else { + // we're not exclusive... + + // ...but given we're inserting, if the child node is a leaf + // node we need to take a write latch on it + if childNode.isLeaf() { + childNode.takeWriteLatch() + } else { + childNode.takeReadLatch() + } + } + + // is the next node full? if so we need to split + // we need to write to node (the parent), lhs and rhs + if b.isNodeFull(childNode) { + // we need to split, but if we only have a read latch on the + // parent, we need to bail and retry + if node.latchState() != bufferpool.Write { + // release latch on node + node.releaseAnyLatch() + // release latch on childNode + childNode.releaseAnyLatch() + return ErrNeedsExclusive + } + + slotCount := node.slotCount() + + lhs, pivot, rhs, err := b.splitNode(childNode) + if err != nil { + return err + } + + // find out where we put the new pointer + j, _ := node.findKey(pivot) + + // add the pivot key pointing to the lhs page + b.insertInternalEntryAt(node, j, pivot, lhs.page.ID()) + + // if not the last key seperator, adjust the pointer for the adjacent key seperator + // if it is the last one, set the next pointer + if j < slotCount { + b.updatePointerEntryAt(node, j+1, rhs.page.ID()) + } else { + //set the next ptr to point to the rhs page + node.page.WriteNextPointer(rhs.page.ID()) + } + + node.releaseWriteLatch() + b.unpin(node) + + // find out which node our key needs to go into + if key.Less(pivot) { + rhs.releaseWriteLatch() + b.unpin(rhs) + return b.insertNonFull(lhs, key, tup, forceExclusive) + } else { + lhs.releaseWriteLatch() + b.unpin(lhs) + return b.insertNonFull(rhs, key, tup, forceExclusive) + } + } else { + // given the child node was not full, we can unlatch the parent here + node.releaseAnyLatch() + b.unpin(node) + + // now do the insert on the child node + return b.insertNonFull(childNode, key, tup, forceExclusive) + } + } +} + +func (b *BTree) splitLeafNode(nodeToSplit *BTreeNode) (*BTreeNode, Sortable, *BTreeNode, error) { + // fmt.Printf("leaf split on page %v\n", nodeToSplit.page.ID()) + + // where we are going to split + slotCount := int(nodeToSplit.page.ReadSlotCount()) + splitPoint := slotCount / 2 + // get the node to split + lhs := nodeToSplit + // make a new node + rhs, err := b.newLeaf() + if err != nil { + return nil, nil, nil, err + } + rhs.takeWriteLatch() + + seperationKeySlot := lhs.page.ReadPageSlot(int16(splitPoint)) + pl := seperationKeySlot.KeyPayload(lhs.page) + seperationKey := Int(pl.KeyAsInt(lhs.page)) + + leftSlotCount := int(lhs.page.ReadSlotCount()) + + // find the split point + var rightSlotCount = leftSlotCount - splitPoint + + //copy the data from left to right + freeSpaceOffset := rhs.page.ReadFreeSpaceOffset() + for i := 0; i < rightSlotCount; i++ { + j := splitPoint + i + + // read the slot from the page + s := lhs.page.ReadPageSlot(int16(j)) + // read the chunk from the page + lbl := s.LeafPayload(lhs.page) + c := lbl.GetPayloadReader(lhs.page) + + // mod the freespace offset based on the size of the chunk + freeSpaceOffset -= int16(c.Length()) + // update the values in the slot + s.PayloadOffset = freeSpaceOffset + + // write the chunk + offset := rhs.page.WriteLeafPagePayloadHeader(int16(freeSpaceOffset), c.KeyLength, c.KeyBytes, c.Flags, c.OverflowPtr, c.PayloadTotalLength) + rhs.page.WriteLeafPagePayloadBytes(offset, c.PayloadChunkLength, c.PayloadChunkBytes) + //write the slot + rhs.page.WritePageSlot(int16(i), s) + + // update the free space offset + rhs.page.WriteFreeSpaceOffset(int16(freeSpaceOffset)) + } + + // set the new slotcounts + lhs.page.WriteSlotCount(int16(splitPoint)) + rhs.page.WriteSlotCount(int16(rightSlotCount)) + + // set sibling pointers + rightPtr := lhs.page.ReadNextPointer() + rhs.page.WriteNextPointer(rightPtr) + lhs.page.WriteNextPointer(rhs.page.ID()) + rhs.page.WritePrevPointer(lhs.page.ID()) + + return lhs, seperationKey, rhs, nil +} + +func (b *BTree) splitInternalNode(nodeToSplit *BTreeNode) (*BTreeNode, Sortable, *BTreeNode, error) { + // fmt.Printf("internal split on page %v\n", nodeToSplit.page.ID()) + + // where we are going to split + slotCount := int(nodeToSplit.page.ReadSlotCount()) + splitPoint := slotCount / 2 + + // get the node to split + lhs := nodeToSplit + // make a new node + rhs, err := b.newInternal() + if err != nil { + return nil, nil, nil, err + } + rhs.takeWriteLatch() + + seperationKeySlot := lhs.page.ReadPageSlot(int16(splitPoint - 1)) + pl := seperationKeySlot.KeyPayload(lhs.page) + seperationKey := Int(pl.KeyAsInt(lhs.page)) + ipl := seperationKeySlot.InternalPayload(lhs.page) + lhsNext := ipl.ValueAsPagePointer(lhs.page) + + leftSlotCount := int(lhs.page.ReadSlotCount()) + + // find the split point + var rightSlotCount = leftSlotCount - splitPoint + + //copy the data from left to right + freeSpaceOffset := rhs.page.ReadFreeSpaceOffset() + for i := 0; i < rightSlotCount; i++ { + j := splitPoint + i + + // read the slot from the page + s := lhs.page.ReadPageSlot(int16(j)) + // read the chunk from the page + ipl := s.InternalPayload(lhs.page) + c := ipl.InternalPageChunk(lhs.page) + + // mod the freespace offset based on the size of the chunk + freeSpaceOffset -= int16(c.Length()) + // update the values in the slot + s.PayloadOffset = freeSpaceOffset + + // write the chunk + rhs.page.WriteInternalPageChunk(int16(freeSpaceOffset), c) + //write the slot + rhs.page.WritePageSlot(int16(i), s) + + // update the free space offset + rhs.page.WriteFreeSpaceOffset(int16(freeSpaceOffset)) + } + + // set the new slotcounts + lhs.page.WriteSlotCount(int16(splitPoint - 1)) + rhs.page.WriteSlotCount(int16(rightSlotCount)) + + // set the next ptr pages + rhs.page.WriteNextPointer(lhs.page.ReadNextPointer()) + lhs.page.WriteNextPointer(lhsNext) + + return lhs, seperationKey, rhs, nil +} + +// func (b *BTree) Delete(x *BTreeNode, k int) { +// t := b.t +// i := 0 +// for i < x.slotCount() && k > x.keys[i].Key { +// i += 1 +// } +// if x.isLeaf { +// if i < x.slotCount() && x.keys[i].Key == k { +// x.keys = removePair(x.keys, i) +// } +// return +// } + +// if i < x.slotCount() && x.keys[i].Key == k { +// b.deleteInternalNode(x, k, i) +// } else if len(x.child[i].keys) >= t { +// b.Delete(x.child[i], k) +// } else { +// if i != 0 && i+2 < len(x.child) { +// if len(x.child[i-1].keys) >= t { +// b.deleteSibling(x, i, i-1) +// } else if len(x.child[i+1].keys) >= t { +// b.deleteSibling(x, i, i+1) +// } else { +// b.deleteMerge(x, i, i+1) +// } +// } else if i == 0 { +// if len(x.child[i+1].keys) >= t { +// b.deleteSibling(x, i, i+1) +// } else { +// b.deleteMerge(x, i, i+1) +// } +// } else if i+1 == len(x.child) { +// if len(x.child[i-1].keys) >= t { +// b.deleteSibling(x, i, i-1) +// } else { +// b.deleteMerge(x, i, i-1) +// } +// } +// b.Delete(x.child[i], k) +// } + +// } + +// func (b *BTree) deleteInternalNode(x *BTreeNode, k int, i int) { +// t := b.t +// if x.isLeaf { +// if x.keys[i].Key == k { +// x.keys = removePair(x.keys, i) +// return +// } +// return +// } + +// if len(x.child[i].keys) >= t { +// x.keys[i] = b.deletePredecessor(x.child[i]) +// return +// } else if len(x.child[i+1].keys) >= t { +// x.keys[i] = b.deleteSuccessor(x.child[i+1]) +// return +// } else { +// b.deleteMerge(x, i, i+1) +// b.deleteInternalNode(x.child[i], k, b.t-1) +// } +// } + +// func (b *BTree) deletePredecessor(x *BTreeNode) *Pair { +// if x.isLeaf { +// k := x.keys[x.slotCount()-1] +// x.keys = removePair(x.keys, x.slotCount()-1) +// return k +// } +// n := x.slotCount() - 1 +// if len(x.child[n].keys) >= b.t { +// b.deleteSibling(x, n, n+1) +// } else { +// b.deleteMerge(x, n, n+1) +// } +// return b.deletePredecessor(x.child[n]) +// } + +// func (b *BTree) deleteSuccessor(x *BTreeNode) *Pair { +// if x.isLeaf { +// k := x.keys[0] +// x.keys = removePair(x.keys, 0) +// return k +// } +// if len(x.child[1].keys) >= b.t { +// b.deleteSibling(x, 0, 1) +// } else { +// b.deleteMerge(x, 0, 1) +// } +// return b.deleteSuccessor(x.child[0]) +// } + +// func (b *BTree) deleteMerge(x *BTreeNode, i int, j int) { +// cnode := x.child[i] +// var new *BTreeNode + +// if j > i { +// rsnode := x.child[j] +// cnode.keys = append(cnode.keys, x.keys[i]) +// for q, k := range rsnode.keys { +// cnode.keys = append(cnode.keys, k) +// if len(rsnode.child) > 0 { +// cnode.child = append(cnode.child, rsnode.child[q]) +// } +// } +// if len(rsnode.child) > 0 { +// cnode.child = append(cnode.child, rsnode.child[len(rsnode.child)-1]) +// rsnode.child = removeNode(rsnode.child, len(rsnode.child)-1) +// } +// new = cnode +// x.keys = removePair(x.keys, i) +// x.child = removeNode(x.child, j) +// } else { +// lsnode := x.child[j] +// lsnode.keys = append(lsnode.keys, x.keys[j]) +// for q, k := range cnode.keys { +// lsnode.keys = append(lsnode.keys, k) +// if len(lsnode.child) > 0 { +// lsnode.child = append(lsnode.child, cnode.child[q]) +// } +// } +// new = lsnode +// x.keys = removePair(x.keys, j) +// x.child = removeNode(x.child, i) +// } + +// if x == b.Root && x.slotCount() == 0 { +// b.Root = new +// } +// } + +// func (b *BTree) deleteSibling(x *BTreeNode, i int, j int) { +// cnode := x.child[i] +// if i < j { +// rsnode := x.child[j] +// cnode.keys = append(cnode.keys, x.keys[i]) +// x.keys[i] = rsnode.keys[0] + +// if len(rsnode.child) > 0 { +// cnode.child = append(cnode.child, rsnode.child[0]) +// rsnode.child = removeNode(rsnode.child, 0) +// } +// rsnode.keys = removePair(rsnode.keys, 0) +// } else { +// lsnode := x.child[j] +// cnode.keys = insertPair(cnode.keys, 0, x.keys[i-1]) +// x.keys[i-1] = lsnode.keys[len(lsnode.keys)-1] +// lsnode.keys = removePair(lsnode.keys, len(lsnode.keys)-1) +// if len(lsnode.child) > 0 { +// insertNode(cnode.child, 0, nil) +// p := lsnode.child[len(lsnode.child)-1] +// lsnode.child = removeNode(lsnode.child, len(lsnode.child)-1) +// cnode.child = insertNode(cnode.child, 0, p) +// } +// } +// } + +// func removeNode(a []*BTreeNode, s int) []*BTreeNode { +// return append(a[:s], a[s+1:]...) +// } + +// func insertNode(a []*BTreeNode, index int, value *BTreeNode) []*BTreeNode { +// if len(a) == index { // nil or empty slice or after last element +// return append(a, value) +// } +// a = append(a[:index+1], a[index:]...) // index < len(a) +// a[index] = value +// return a +// } + +// func removePair(a []*Pair, s int) []*Pair { +// return append(a[:s], a[s+1:]...) +// } + +// func insertPair(a []*Pair, index int, value *Pair) []*Pair { +// if len(a) == index { // nil or empty slice or after last element +// return append(a, value) +// } +// a = append(a[:index+1], a[index:]...) // index < len(a) +// a[index] = value +// return a +// } + +func (b *BTree) Dump(l int) { + fmt.Printf("btree(keysPerLeafPage: %d, keysPerInternalPage: %d)\n", b.keysPerLeafPage, b.keysPerInternalPage) + + node, _ := b.fetchNode(b.rootNode) + defer b.bufferpool.UnpinPage(node.page.ID()) + b.nodeDump(node, l) +} + +func (b *BTree) nodeDump(node *BTreeNode, l int) { + fmt.Printf("%snode(%d) --> leafNode: %v, slotCount: %d\n", fmt.Sprintf("%*s", l, ""), node.page.ID(), node.isLeaf(), node.page.ReadSlotCount()) + + if node.isLeaf() { + keys := "" + sc := int16(node.slotCount()) + si := bufferpool.NewPageSlotIterator(node.page, 0) + slot := si.Next() + for slot != nil { + pl := slot.KeyPayload(node.page) + k := int(pl.KeyAsInt(node.page)) + keys += fmt.Sprintf("%d", k) + if si.Cursor() < sc { + keys += ", " + } + slot = si.Next() + } + fmt.Printf("%skeys [%s]\n", fmt.Sprintf("%*s", l+2, ""), keys) + } else { + fmt.Printf("%ssep-keys [\n", fmt.Sprintf("%*s", l+2, "")) + si := bufferpool.NewPageSlotIterator(node.page, 0) + slot := si.Next() + for slot != nil { + pl := slot.KeyPayload(node.page) + k := int(pl.KeyAsInt(node.page)) + fmt.Printf("%s<%d\n", fmt.Sprintf("%*s", l+4, ""), k) + ipl := slot.InternalPayload(node.page) + pn := bufferpool.PageID(ipl.ValueAsPagePointer(node.page)) + cn, _ := b.fetchNode(pn) + b.nodeDump(cn, l+6) + b.bufferpool.UnpinPage(cn.page.ID()) + slot = si.Next() + } + pn := node.page.ReadNextPointer() + if pn.Page != bufferpool.INVALID_PAGE { + fmt.Printf("%s>=(next)\n", fmt.Sprintf("%*s", l+4, "")) + cn, _ := b.fetchNode(pn) + b.nodeDump(cn, l+6) + b.bufferpool.UnpinPage(cn.page.ID()) + } else { + fmt.Printf("%s>=(next MISSING!)\n", fmt.Sprintf("%*s", l+4, "")) + } + fmt.Printf("%s]\n", fmt.Sprintf("%*s", l+2, "")) + } +} diff --git a/tstore/btree_test.go b/tstore/btree_test.go new file mode 100644 index 000000000..422304d21 --- /dev/null +++ b/tstore/btree_test.go @@ -0,0 +1,160 @@ +package tstore + +import ( + "fmt" + "math/rand" + "os" + "testing" + + "github.com/featurebasedb/featurebase/v3/bufferpool" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" +) + +func TestAddItemsToBTreeAndValidate(t *testing.T) { + diskManager := bufferpool.NewOnDiskDiskManager() + + objectId := int32(1) + shard := int32(0) + dataFile := fmt.Sprintf("ts-shard.%04d", shard) + + os.Remove(dataFile) + + diskManager.CreateOrOpenShard(objectId, shard, dataFile) + + bufferPool := bufferpool.NewBufferPool(100, diskManager) + + tableSchema := types.Schema{ + &types.PlannerColumn{ + ColumnName: "vtest", + Type: parser.NewDataTypeVarchar(50), + }, + } + + b, err := NewBTree(8, objectId, shard, tableSchema, bufferPool) + if err != nil { + t.Fatal(err) + } + + rowSchema := types.Schema{ + &types.PlannerColumn{ + ColumnName: "_id", + Type: parser.NewDataTypeID(), + }, + &types.PlannerColumn{ + ColumnName: "vtest", + Type: parser.NewDataTypeVarchar(50), + }, + } + + inserts := make([]int, 0) + for i := 1; i <= 300; i++ { + inserts = append(inserts, i) + } + rand.Shuffle(len(inserts), func(i, j int) { inserts[i], inserts[j] = inserts[j], inserts[i] }) + + rr := make(types.Row, 2) + for _, i := range inserts { + rr[0] = int64(i) + rr[1] = fmt.Sprintf("This is a test of things %d", i) + + tup := &BTreeTuple{ + TupleSchema: rowSchema, + Tuple: rr, + } + + // fmt.Printf("%v", tup) + + err = b.Insert(tup) + if err != nil { + t.Fatal(err) + } + } + + key, tuple := b.Search(nil, Int(33)) + + fmt.Printf("%v, %v\n\n", key, tuple) + + b.Dump(0) +} + +func TestAddItemsToBTreeAndValidate_VeryWide(t *testing.T) { + diskManager := bufferpool.NewOnDiskDiskManager() + + objectId := int32(1) + shard := int32(0) + dataFile := fmt.Sprintf("ts-shard.%04d", shard) + + os.Remove(dataFile) + + diskManager.CreateOrOpenShard(objectId, shard, dataFile) + + bufferPool := bufferpool.NewBufferPool(100, diskManager) + + tableSchema := make(types.Schema, 0) + + for i := 0; i < 3000; i++ { + tableSchema = append(tableSchema, &types.PlannerColumn{ + ColumnName: fmt.Sprintf("vtest%d", i+1), + Type: parser.NewDataTypeVarchar(4), + }) + } + + b, err := NewBTree(8, objectId, shard, tableSchema, bufferPool) + if err != nil { + t.Fatal(err) + } + + rowSchema := make(types.Schema, 0) + rowSchema = append(rowSchema, &types.PlannerColumn{ + ColumnName: "_id", + Type: parser.NewDataTypeID(), + }) + for i := 0; i < 3000; i++ { + rowSchema = append(rowSchema, &types.PlannerColumn{ + ColumnName: fmt.Sprintf("vtest%d", i+1), + Type: parser.NewDataTypeVarchar(4), + }) + } + + inserts := make([]int, 0) + for i := 1; i <= 300; i++ { + inserts = append(inserts, i) + } + rand.Seed(10) + rand.Shuffle(len(inserts), func(i, j int) { inserts[i], inserts[j] = inserts[j], inserts[i] }) + + rr := make(types.Row, 3001) + for j, i := range inserts { + rr[0] = int64(i) + + for j := 0; j < 3000; j++ { + rr[j+1] = fmt.Sprintf("%04d", j) + } + + tup := &BTreeTuple{ + TupleSchema: rowSchema, + Tuple: rr, + } + + fmt.Printf("[%d]row key %v\n\n", j, i) + + if j == 17 { + fmt.Printf("here\n") + } + + err = b.Insert(tup) + if err != nil { + t.Fatal(err) + } + + b.Dump(0) + fmt.Printf("\n\n--------------\n\n") + } + + key, tuple := b.Search(nil, Int(33)) + + fmt.Printf("%v, %v\n\n", key, tuple) + + b.Dump(0) +} diff --git a/tstore/btreenode.go b/tstore/btreenode.go new file mode 100644 index 000000000..ca0734ac4 --- /dev/null +++ b/tstore/btreenode.go @@ -0,0 +1,83 @@ +// Copyright 2023 Molecula Corp. All rights reserved. + +package tstore + +import "github.com/featurebasedb/featurebase/v3/bufferpool" + +type BTreeNode struct { + page *bufferpool.Page +} + +func (n *BTreeNode) slotCount() int { + return int(n.page.ReadSlotCount()) +} + +func (n *BTreeNode) isLeaf() bool { + return n.page.ReadPageType() == bufferpool.PAGE_TYPE_BTREE_LEAF +} + +func (n *BTreeNode) takeReadLatch() { + n.page.TakeReadLatch() +} + +func (n *BTreeNode) releaseAnyLatch() { + n.page.ReleaseAnyLatch() +} + +func (n *BTreeNode) releaseReadLatch() { + n.page.ReleaseReadLatch() +} + +func (n *BTreeNode) takeWriteLatch() { + n.page.TakeWriteLatch() +} + +func (n *BTreeNode) releaseWriteLatch() { + n.page.ReleaseWriteLatch() +} + +func (n *BTreeNode) latchState() bufferpool.PageLatchState { + return n.page.LatchState() +} + +func (n *BTreeNode) findKey(key Sortable) (int, bool) { + if n.latchState() == bufferpool.None { + panic("unexpected latch state") + } + + minIndex := 0 + onePastMaxIndex := int(n.page.ReadSlotCount()) + for onePastMaxIndex != minIndex { + index := (minIndex + onePastMaxIndex) / 2 + slot := n.page.ReadPageSlot(int16(index)) + pl := slot.KeyPayload(n.page) + keyAtIndex := Int(pl.KeyAsInt(n.page)) + if key.Equals(keyAtIndex) { + return index, true + } + if key.Less(keyAtIndex) { + onePastMaxIndex = index + } else { + minIndex = index + 1 + } + } + return minIndex, false +} + +func (n *BTreeNode) findNextPointer(key Sortable, objectID int32, shard int32) (bufferpool.PageID, error) { + if n.latchState() == bufferpool.None { + panic("unexpected latch state") + } + + slotCount := int(n.page.ReadSlotCount()) + + keyPosition, _ := n.findKey(key) + if keyPosition == slotCount { + // return the next pointer... + return n.page.ReadNextPointer(), nil + } + // else return the pointer at the position returned from findKey + slot := n.page.ReadPageSlot(int16(keyPosition)) + ipl := slot.InternalPayload(n.page) + return ipl.ValueAsPagePointer(n.page), nil +} diff --git a/tstore/btreetypes.go b/tstore/btreetypes.go new file mode 100644 index 000000000..f445d92cd --- /dev/null +++ b/tstore/btreetypes.go @@ -0,0 +1,188 @@ +// Copyright 2023 Molecula Corp. All rights reserved. + +package tstore + +import ( + "bytes" + "encoding/binary" + + "github.com/pkg/errors" + + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" +) + +const ( + KEY_SIZE_INT64 = 8 +) + +var ErrNeedsExclusive = errors.New("ErrNeedsExclusive") + +type BTreeTuple struct { + TupleSchema types.Schema + Tuple types.Row +} + +func (t *BTreeTuple) keyValue() Sortable { + kv, ok := t.Tuple[0].(int64) + if !ok { + return nil + } + return Int(kv) +} + +func NewBTreeTupleFromBytes(b []byte, schema types.Schema) *BTreeTuple { + t := &BTreeTuple{ + TupleSchema: schema, + Tuple: make(types.Row, len(schema)), + } + + rdr := bytes.NewReader(b) + for i, s := range schema { + switch s.Type.(type) { + case *parser.DataTypeVarchar: + var l int32 + binary.Read(rdr, binary.BigEndian, &l) + bvalue := make([]byte, l) + binary.Read(rdr, binary.BigEndian, &bvalue) + t.Tuple[i] = string(bvalue) + default: + panic("unexpected type") + } + } + return t +} + +func (b *BTreeTuple) Bytes() ([]byte, error) { + var valueBuf bytes.Buffer + for cidx, c := range b.TupleSchema { + // skip the key column + if cidx == 0 { + continue + } + rd := b.Tuple[cidx] + + switch ty := c.Type.(type) { + case *parser.DataTypeVarchar: + if rd == nil { + b := []byte{0, 0, 0, 0} + valueBuf.Write(b) + } else { + data, ok := rd.(string) + if !ok { + return []byte{}, errors.Errorf("unexpected type conversion '%T'", rd) + } + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, uint32(len(data))) + valueBuf.Write(b) + valueBuf.WriteString(data) + } + default: + return []byte{}, errors.Errorf("unexpected type '%T'", ty) + } + } + return valueBuf.Bytes(), nil +} + +type Equatable interface { + Equals(b Equatable) bool +} + +type Sortable interface { + Equatable + Less(b Sortable) bool + Length() int + Bytes() []byte +} + +type String string + +func (s String) Equals(other Equatable) bool { + if o, ok := other.(String); ok { + return s == o + } else { + return false + } +} + +func (s String) Less(other Sortable) bool { + if o, ok := other.(String); ok { + return s < o + } else { + return false + } +} + +func (i String) Length() int { + return len(i) +} + +func (i String) Bytes() []byte { + return []byte(i) +} + +type ByteSlice []byte + +// func (i ByteSlice) Hash() int { +// return int(i) +// } + +// func (i ByteSlice) Less(other Sortable) bool { +// if o, ok := other.(Int); ok { +// return i < o +// } else { +// return false +// } +// } + +// func (i ByteSlice) Equals(other Equatable) bool { +// if o, ok := other.(Int); ok { +// return i == o +// } else { +// return false +// } +// } + +func (i ByteSlice) Length() int { + return len(i) +} + +func (i ByteSlice) Bytes() []byte { + return i +} + +func (i ByteSlice) AsInt() int { + return int(binary.BigEndian.Uint32(i)) +} + +type Int int + +func (i Int) Hash() int { + return int(i) +} + +func (i Int) Less(other Sortable) bool { + if o, ok := other.(Int); ok { + return i < o + } else { + return false + } +} + +func (i Int) Equals(other Equatable) bool { + if o, ok := other.(Int); ok { + return i == o + } else { + return false + } +} + +func (i Int) Length() int { + return 4 +} + +func (i Int) Bytes() []byte { + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, uint32(i)) + return b +} diff --git a/wireprotocol/wireprimitives.go b/wireprotocol/wireprimitives.go index a0272ca4b..eb96d0116 100644 --- a/wireprotocol/wireprimitives.go +++ b/wireprotocol/wireprimitives.go @@ -36,6 +36,7 @@ const ( TYPE_IDSET int8 = 0x06 TYPE_STRING int8 = 0x07 TYPE_STRINGSET int8 = 0x08 + TYPE_VARCHAR int8 = 0x09 ) func ExpectToken(reader io.Reader, token int16) (int16, error) { @@ -50,6 +51,15 @@ func ExpectToken(reader io.Reader, token int16) (int16, error) { return tk, nil } +func ReadToken(reader io.Reader) (int16, error) { + var tk int16 + err := binary.Read(reader, binary.BigEndian, &tk) + if err != nil { + return 0, err + } + return tk, nil +} + // TOKEN_COLUMN_INFO message // length (bytes) // token 2 @@ -109,6 +119,10 @@ func WriteSchema(schema types.Schema) ([]byte, error) { case *parser.DataTypeStringSet: writeInt8(writer, TYPE_STRINGSET) + case *parser.DataTypeVarchar: + writeInt8(writer, TYPE_VARCHAR) + writeInt32(writer, int32(ty.Length)) + default: return []byte{}, errors.Errorf("unexpected type '%T'", s.Type) } @@ -179,6 +193,14 @@ func ReadSchema(reader io.Reader) (types.Schema, error) { case TYPE_STRINGSET: dataType = parser.NewDataTypeStringSet() + + case TYPE_VARCHAR: + var length int32 + err = binary.Read(reader, binary.BigEndian, &length) + if err != nil { + return nil, err + } + dataType = parser.NewDataTypeVarchar(int64(length)) } schema = append(schema, &types.PlannerColumn{ @@ -338,6 +360,18 @@ func WriteRow(row types.Row, schema types.Schema) ([]byte, error) { } } + case *parser.DataTypeVarchar: + if val == nil { + writeInt32(writer, 0) + } else { + v, ok := row[i].(string) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + writeInt32(writer, int32(len(v))) + writer.WriteString(v) + } + default: return []byte{}, errors.Errorf("unexpected type '%T'", s.Type) } @@ -483,6 +517,23 @@ func ReadRow(reader io.Reader, schema types.Schema) (types.Row, error) { row[idx] = set } + case *parser.DataTypeVarchar: + var len int32 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + bvalue := make([]byte, len) + err = binary.Read(reader, binary.BigEndian, &bvalue) + if err != nil { + return nil, err + } + row[idx] = string(bvalue) + } + default: return nil, errors.Errorf("unexpected type '%T'", s.Type) } @@ -558,6 +609,12 @@ func writeInt16(w io.Writer, i int16) { w.Write(b) } +func writeInt32(w io.Writer, i int32) { + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, uint32(i)) + w.Write(b) +} + func writeInt64(w io.Writer, i int64) { b := make([]byte, 8) binary.BigEndian.PutUint64(b, uint64(i))