scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-09 22:51:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

added varchar type; create table works; show columns works; wip on b+tree

Browse source
This commit is contained in:
pokeeffe-molecula 2023-03-01 12:45:22 -06:00
parent 6383a96ac5
commit cb72f666f8
46 changed files with 3431 additions and 534 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

134
api.go
View file

@ -15,6 +15,7 @@ import (
"math"
"net/url"
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
@ -29,11 +30,13 @@ import (
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/featurebasedb/featurebase/v3/rbf"
"github.com/featurebasedb/featurebase/v3/tstore"
"github.com/featurebasedb/featurebase/v3/wireprotocol"
"github.com/prometheus/client_golang/prometheus"
//"github.com/featurebasedb/featurebase/v3/pg"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/roaring"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
planner_types "github.com/featurebasedb/featurebase/v3/sql3/planner/types"
"github.com/featurebasedb/featurebase/v3/tracing"
"github.com/pkg/errors"
@ -263,9 +266,13 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
return nil, errors.Wrap(err, "validating api method")
}
// get the next indexID number
indexID := int32(len(api.holder.indexes) + 1)
// Populate the create index message.
ts := timestamp()
cim := &CreateIndexMessage{
IndexID: indexID,
Index: indexName,
CreatedAt: ts,
Owner: requestUserID,
@ -1727,6 +1734,14 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
}
}
// handle the tuple data
if len(req.Tuples) > 0 {
err := api.importTuples(ctx, tx, shard, indexName, req.Tuples)
if err != nil {
return err
}
}
if api.isComputeNode && !req.SuppressLog {
partition := disco.ShardToShardPartition(indexName, shard, disco.DefaultPartitionN)
msg := &computer.ImportRoaringShardMessage{
@ -1734,6 +1749,7 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
Partition: partition,
Shard: shard,
Views: make([]computer.RoaringUpdate, len(req.Views)),
Tuples: req.Tuples,
}
for i, view := range req.Views {
msg.Views[i] = computer.RoaringUpdate{
@ -1766,6 +1782,122 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
return nil
}
func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error {
// TODO wouldn't hurt to have consolidated logic somewhere for validating view names.
switch fieldType {
case FieldTypeSet, FieldTypeTime:
if viewUpdate.View == "" {
viewUpdate.View = "standard"
}
// add 'standard_' if we just have a time... this is how IDK works by default
if fieldType == FieldTypeTime && !strings.HasPrefix(viewUpdate.View, viewStandard) {
viewUpdate.View = fmt.Sprintf("%s_%s", viewStandard, viewUpdate.View)
}
case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp:
if viewUpdate.View == "" {
viewUpdate.View = "bsig_" + viewUpdate.Field
} else if viewUpdate.View != "bsig_"+viewUpdate.Field {
return NewBadRequestError(errors.Errorf("invalid view name (%s) for field %s of type %s", viewUpdate.View, viewUpdate.Field, fieldType))
}
}
return nil
}
func (api *API) importTuples(ctx context.Context, tx Tx, shard uint64, tableName string, tupleData []byte) error {
// get the table
index, err := api.Index(ctx, tableName)
if err != nil {
return err
}
// if the index id is 0 then we can't do an insert into the t-store
if index.ID == 0 {
return errors.Errorf("cannot insert into table '%s' because it does not have a non-zero object id", tableName)
}
basePath := index.TStorePath()
// open or create btreefile for this shard
dataFile := filepath.Join(basePath, fmt.Sprintf("ts-shard.%04d", shard))
api.holder.tstoredisk.CreateOrOpenShard(index.ID, int32(shard), dataFile)
// get the schema from the FeatureBase table in the form of a planner_types.Schema
// we just want the t-store types
fieldList := make([]*Field, 0)
for _, f := range index.fields {
if strings.EqualFold(f.options.Type, FieldTypeVarchar) {
fieldList = append(fieldList, f)
}
}
sort.Slice(fieldList, func(i, j int) bool {
return fieldList[i].CreatedAt() < fieldList[j].CreatedAt()
})
indexSchema := make(planner_types.Schema, len(fieldList))
for i, f := range fieldList {
indexSchema[i] = &planner_types.PlannerColumn{
ColumnName: f.name,
Type: parser.NewDataTypeVarchar(f.options.Length),
}
}
// create the b-tree we're going to use
b, err := tstore.NewBTree(tstore.KEY_SIZE_INT64, index.ID, int32(shard), indexSchema, api.holder.tstorepool)
if err != nil {
return err
}
// start to read the data for the import
rdr := bytes.NewReader(tupleData)
_, err = wireprotocol.ExpectToken(rdr, wireprotocol.TOKEN_SCHEMA_INFO)
if err != nil {
return err
}
// get the row schema from the import data
rowSchema, err := wireprotocol.ReadSchema(rdr)
if err != nil {
return err
}
// read rows until we get to the end
tk, err := wireprotocol.ReadToken(rdr)
if err != nil {
return err
}
for tk == wireprotocol.TOKEN_ROW {
rr, err := wireprotocol.ReadRow(rdr, rowSchema)
if err != nil {
return err
}
// make sure the key is at offset 0
s := rowSchema[0]
if !strings.EqualFold(s.ColumnName, "_id") {
return errors.Errorf("unexpected key column position ")
}
err = b.Insert(&tstore.BTreeTuple{
TupleSchema: rowSchema,
Tuple: rr,
})
if err != nil {
return err
}
tk, err = wireprotocol.ReadToken(rdr)
if err != nil {
return err
}
}
if tk != wireprotocol.TOKEN_DONE {
return errors.Errorf("unexpected token '%d'", tk)
}
return nil
}
// ImportValue is a wrapper around the common code in ImportValueWithTx, which
// currently just translates req.Clear into a clear ImportOption.
func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error {

182
batch/batch.go
View file

@ -16,6 +16,9 @@ import (
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/roaring"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
"github.com/featurebasedb/featurebase/v3/wireprotocol"
"github.com/pkg/errors"
)
@ -130,6 +133,9 @@ type Batch struct {
// values holds the values for each record of an int field
values map[string][]int64
// values holds the values for each record of an varchar field
tupleValues map[string][]interface{}
// boolValues is a map[fieldName][idsIndex]bool, which holds the values for
// each record of a bool field. It is a map of maps in order to accomodate
// nil values (they just aren't recorded in the map[int]).
@ -184,6 +190,8 @@ type Batch struct {
frags fragments
clearFrags fragments
tuplefrags tuplefragments
useShardTransactionalEndpoint bool
}
@ -257,6 +265,7 @@ func NewBatch(importer featurebase.Importer, size int, tbl *dax.Table, fields []
headerMap := make(map[string]*featurebase.FieldInfo, len(fields))
rowIDs := make(map[int][]uint64, len(fields))
values := make(map[string][]int64)
tupleValues := make(map[string][]interface{})
boolValues := make(map[string]map[int]bool)
boolNulls := make(map[string][]uint64)
tt := make(map[int]map[string][]int, len(fields))
@ -293,6 +302,8 @@ func NewBatch(importer featurebase.Importer, size int, tbl *dax.Table, fields []
rowIDs[i] = make([]uint64, 0, size)
case featurebase.FieldTypeBool:
boolValues[field.Name] = make(map[int]bool)
case featurebase.FieldTypeVarchar:
tupleValues[field.Name] = make([]interface{}, 0, size)
default:
return nil, errors.Errorf("field type '%s' is not currently supported through Batch", typ)
}
@ -309,6 +320,7 @@ func NewBatch(importer featurebase.Importer, size int, tbl *dax.Table, fields []
clearRowIDs: make(map[int]map[int]uint64),
rowIDSets: make(map[string][][]uint64),
values: values,
tupleValues: tupleValues,
boolValues: boolValues,
boolNulls: boolNulls,
nullIndices: make(map[string][]uint64),
@ -325,6 +337,7 @@ func NewBatch(importer featurebase.Importer, size int, tbl *dax.Table, fields []
frags: make(fragments),
clearFrags: make(fragments),
tuplefrags: make(tuplefragments),
}
if hasTime {
b.times = make([]QuantizedTime, 0, size)
@ -544,6 +557,9 @@ func (b *Batch) Add(rec Row) error {
case featurebase.FieldTypeBool:
// If we want to support bools as string values, we would do
// that here.
case featurebase.FieldTypeVarchar:
b.tupleValues[field.Name] = append(b.tupleValues[field.Name], val)
default:
// nil-extend
for len(b.rowIDs[i]) < curPos {
@ -647,6 +663,9 @@ func (b *Batch) Add(rec Row) error {
boolNulls = append(boolNulls, uint64(curPos))
b.boolNulls[field.Name] = boolNulls
case featurebase.FieldTypeVarchar:
b.tupleValues[field.Name] = append(b.tupleValues[field.Name], nil)
default:
// only append nil to rowIDs if this field already has
// rowIDs. Otherwise, this could be a []string or
@ -780,15 +799,22 @@ func (b *Batch) Import() error {
transTime := time.Now()
b.log.Printf("translating batch of %d took: %v", size, transTime.Sub(transStart))
var tuplefrags tuplefragments
frags, clearFrags, err := b.makeFragments(b.frags, b.clearFrags)
if err != nil {
return errors.Wrap(err, "making fragments (flush)")
}
if b.useShardTransactionalEndpoint {
frags, clearFrags, err = b.makeSingleValFragments(frags, clearFrags)
if err != nil {
return errors.Wrap(err, "making single val fragments")
}
tuplefrags, err = b.makeTupleStoreFragments((b.tuplefrags))
if err != nil {
return errors.Wrap(err, "making pvl fragments")
}
}
makeTime := time.Now()
@ -797,19 +823,24 @@ func (b *Batch) Import() error {
if b.splitBatchMode {
b.frags = frags
b.clearFrags = clearFrags
b.tuplefrags = tuplefrags
} else {
b.frags = make(fragments)
b.clearFrags = make(fragments)
b.tuplefrags = make(tuplefragments)
// create bitmaps out of each field in b.rowIDs and import. Also
// import int data.
if !b.useShardTransactionalEndpoint {
if len(tuplefrags) > 0 {
return errors.Wrap(errors.Errorf("t-store values are not supported in non-transactional imports"), "doing import")
}
err = b.doImport(frags, clearFrags)
if err != nil {
return errors.Wrap(err, "doing import")
}
b.log.Printf("importing fragments took %v", time.Since(makeTime))
} else {
err = b.doImportShardTransactional(frags, clearFrags)
err = b.doImportShardTransactional(frags, clearFrags, tuplefrags)
if err != nil {
return errors.Wrap(err, "doing shard transactional import")
}
@ -1143,7 +1174,7 @@ func (b *Batch) createFieldKeys(field *featurebase.FieldInfo, keys ...string) (m
return results, nil
}
func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error {
func (b *Batch) doImportShardTransactional(frags, clearFrags fragments, tuplefrags tuplefragments) error {
ctx := context.Background()
start := time.Now()
@ -1200,6 +1231,68 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error {
}
}
// handle tuple frags
for fragKey, tupleFrag := range tuplefrags {
request := getOrCreate(requests, fragKey)
// make a buffer for the tuple data
buf := new(bytes.Buffer)
// get the bytes for the schema
b, err := wireprotocol.WriteSchema(tupleFrag.schema)
if err != nil {
return errors.Wrap(err, "serializing tuple schema")
}
_, err = buf.Write(b)
if err != nil {
return errors.Wrap(err, "serializing tuple schema")
}
// now write each of the tuples
// make a row, we're going to reuse this
rrow := make(types.Row, len(tupleFrag.schema))
// build a map of columnNames to column indexes in the schema
colMap := make(map[string]int)
for i, sc := range tupleFrag.schema {
colMap[sc.ColumnName] = i
}
// iterate the tupleData - outside loop is rows
for id, trow := range tupleFrag.tupleData {
// write the _id column
rrow[0] = int64(id)
// now iterate the rest of the columns
for colName, tval := range trow {
j, ok := colMap[colName]
if !ok {
errors.Errorf("unexpected missing column '%s'", colName)
}
rrow[j] = tval
}
// write the row to the buffer
rb, err := wireprotocol.WriteRow(rrow, tupleFrag.schema)
if err != nil {
return errors.Wrap(err, "serializing tuple row")
}
_, err = buf.Write(rb)
if err != nil {
return errors.Wrap(err, "serializing tuple row")
}
}
// write done to the buffer
b = wireprotocol.WriteDone()
_, err = buf.Write(b)
if err != nil {
return errors.Wrap(err, "serializing tuples")
}
request.Tuples = buf.Bytes()
}
featurebase.SummaryBatchShardImportBuildRequestsSeconds.Observe(time.Since(start).Seconds())
start = time.Now()
eg := egpool.Group{PoolSize: 20}
@ -1695,6 +1788,69 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
return frags, clearFrags, nil
}
func (b *Batch) makeTupleStoreFragments(pvlfrags tuplefragments) (tuplefragments, error) {
shardWidth := b.shardWidth()
// -------------------------
// tuplestore fields
// -------------------------
// build a schema
tupleSchema := make(types.Schema, 0)
var idType parser.ExprDataType = parser.NewDataTypeID()
if b.tbl.StringKeys() {
idType = parser.NewDataTypeString()
}
tupleSchema = append(tupleSchema, &types.PlannerColumn{
ColumnName: "_id",
RelationName: string(b.tbl.Name),
AliasName: "",
Type: idType,
})
for fieldname := range b.tupleValues {
field := b.headerMap[fieldname]
switch field.Options.Type {
case featurebase.FieldTypeVarchar:
tupleSchema = append(tupleSchema, &types.PlannerColumn{
ColumnName: fieldname,
RelationName: string(b.tbl.Name),
AliasName: "",
Type: parser.NewDataTypeVarchar(field.Options.Length),
})
default:
continue
}
}
// for each of the varcharValues mapped, this is a column
for fieldname, varcharMap := range b.tupleValues {
field := b.headerMap[fieldname]
if field.Options.Type != featurebase.FieldTypeVarchar {
continue
}
// for each of the row values for this column
for pos, varcharVal := range varcharMap {
recID := b.ids[pos]
shard := recID / shardWidth
tf := pvlfrags.GetOrCreate(shard, tupleSchema)
_, ok := tf.tupleData[recID]
if !ok {
tf.tupleData[recID] = make(map[string]interface{})
}
tf.tupleData[recID][fieldname] = varcharVal
}
}
return pvlfrags, nil
}
type valsByIDsSortable struct {
ids []uint64
vals []int64
@ -1951,6 +2107,28 @@ func (b *Batch) reset() {
}
}
type tupleFragment struct {
// the schema of the fragment
schema types.Schema
// tupleData by id, by column name
tupleData map[uint64]map[string]interface{}
}
// map[shard]tupleFragment
type tuplefragments map[uint64]*tupleFragment
func (f tuplefragments) GetOrCreate(shard uint64, schema types.Schema) *tupleFragment {
tf, ok := f[shard]
if !ok {
tf = &tupleFragment{
schema: schema,
tupleData: make(map[uint64]map[string]interface{}),
}
f[shard] = tf
}
return tf
}
// map[shard][field][view]fragmentData
type fragments map[fragmentKey]map[string]*roaring.Bitmap

141
bufferpool/bufferpool.go
View file

@ -1,6 +1,8 @@
package bufferpool
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"sync"
@ -10,12 +12,30 @@ import (
type FrameID int
// PageID is the type for page id
type PageID int
type PageID struct {
ObjectID int32
Shard int32
Page int64
}
func (p PageID) Bytes() []byte {
var valueBuf bytes.Buffer
v := make([]byte, 4)
binary.BigEndian.PutUint32(v, uint32(p.ObjectID))
valueBuf.Write(v)
binary.BigEndian.PutUint32(v, uint32(p.Shard))
valueBuf.Write(v)
vp := make([]byte, 8)
binary.BigEndian.PutUint64(vp, uint64(p.Page))
valueBuf.Write(vp)
return valueBuf.Bytes()
}
var pageSyncPool = sync.Pool{
New: func() any {
pg := new(Page)
pg.id = PageID(INVALID_PAGE)
pg.latchState = None
pg.id = PageID{0, 0, INVALID_PAGE}
pg.isDirty = false
pg.pinCount = 0
return pg
@ -24,19 +44,26 @@ var pageSyncPool = sync.Pool{
// BufferPool represents a buffer pool of pages
type BufferPool struct {
// the underlying storage
diskManager DiskManager
// the actual pages in the buffer pool
pages []*Page
// the replacer that will elect replacements when buffer pool is full
replacer *ClockReplacer
// the actual frames in the buffer pool
frames []*Page
// the list of free frames
freeList []FrameID
framesMu sync.RWMutex
// the replacer that will elect replacements when buffer pool is full
replacer *ClockReplacer
// the map of frames to page ids to frame ids
// frame ids are the offset into pages
// if you ask the pool for page 673, this will know at
// what offset in pages page 673 will exist
pageTable map[PageID]FrameID
// frame ids are the offset into frames (above)
// if you ask the pool for page 1:1:673, this will know at
// what offset in pages page 1:1:673 will exist (or not)
pageTable map[PageID]FrameID
pageTableMu sync.RWMutex
}
// TODO(pok) implement a lazy writer
@ -44,9 +71,6 @@ type BufferPool struct {
// * increase size of cache if there is physical memory available
// * write out old pages and boot them from the cache to increase free list
// TODO(pok) implement a checkpoint that scans the pool and writes out dirty pages every
// minute or so
// NewBufferPool returns a buffer pool
func NewBufferPool(maxSize int, diskManager DiskManager) *BufferPool {
freeList := make([]FrameID, 0)
@ -58,7 +82,7 @@ func NewBufferPool(maxSize int, diskManager DiskManager) *BufferPool {
clockReplacer := NewClockReplacer(maxSize)
return &BufferPool{
diskManager: diskManager,
pages: pages,
frames: pages,
replacer: clockReplacer,
freeList: freeList,
pageTable: make(map[PageID]FrameID),
@ -70,7 +94,7 @@ func (b *BufferPool) Dump() {
fmt.Println()
fmt.Printf("------------------------------------------------------------------------------------------\n")
fmt.Printf("BUFFER POOL\n")
for _, p := range b.pages {
for _, p := range b.frames {
if p != nil {
p.Dump("")
}
@ -81,30 +105,40 @@ func (b *BufferPool) Dump() {
// FetchPage fetches the requested page from the buffer pool.
func (b *BufferPool) FetchPage(pageID PageID) (*Page, error) {
b.pageTableMu.Lock()
defer b.pageTableMu.Unlock()
b.framesMu.RLock()
// if it is in buffer pool already then just return it
if frameID, ok := b.pageTable[pageID]; ok {
page := b.pages[frameID]
page := b.frames[frameID]
page.pinCount++
b.replacer.Pin(frameID)
b.framesMu.RUnlock()
return page, nil
}
// we will need to write to frames, so unlock and take write lock
b.framesMu.RUnlock()
b.framesMu.Lock()
defer b.framesMu.Unlock()
// not in the buffer pool so try the free list or
// the replacer will vote a page off the island
frameID, isFromFreeList, err := b.getFrameID()
if err != nil {
b.framesMu.RUnlock()
return nil, err
}
if !isFromFreeList {
// if it didn't come from the freelist then
// remove page from current frame, writing it out if dirty
currentPage := b.pages[frameID]
currentPage := b.frames[frameID]
if currentPage != nil {
if currentPage.isDirty {
b.diskManager.WritePage(currentPage)
}
delete(b.pageTable, currentPage.id)
}
}
@ -116,8 +150,8 @@ func (b *BufferPool) FetchPage(pageID PageID) (*Page, error) {
}
page.pinCount = 1
b.pageTable[pageID] = frameID
pageSyncPool.Put(b.pages[frameID])
b.pages[frameID] = page
pageSyncPool.Put(b.frames[frameID])
b.frames[frameID] = page
b.replacer.Pin(frameID)
return page, nil
@ -125,8 +159,13 @@ func (b *BufferPool) FetchPage(pageID PageID) (*Page, error) {
// UnpinPage unpins the target page from the buffer pool
func (b *BufferPool) UnpinPage(pageID PageID) error {
b.framesMu.RLock()
b.pageTableMu.RLock()
defer b.framesMu.RUnlock()
defer b.pageTableMu.RUnlock()
if frameID, ok := b.pageTable[pageID]; ok {
page := b.pages[frameID]
page := b.frames[frameID]
page.DecPinCount()
if page.pinCount <= 0 {
@ -139,9 +178,15 @@ func (b *BufferPool) UnpinPage(pageID PageID) error {
}
// FlushPage Flushes the target page to disk
// This shold not be called during normal operation
func (b *BufferPool) FlushPage(pageID PageID) bool {
b.framesMu.RLock()
b.pageTableMu.RLock()
defer b.framesMu.RUnlock()
defer b.pageTableMu.RUnlock()
if frameID, ok := b.pageTable[pageID]; ok {
page := b.pages[frameID]
page := b.frames[frameID]
page.DecPinCount()
b.diskManager.WritePage(page)
@ -153,7 +198,12 @@ func (b *BufferPool) FlushPage(pageID PageID) bool {
}
// NewPage allocates a new page in the buffer pool with the disk manager help
func (b *BufferPool) NewPage() (*Page, error) {
func (b *BufferPool) NewPage(fileID int32, shard int32) (*Page, error) {
b.framesMu.Lock()
b.pageTableMu.Lock()
defer b.framesMu.Unlock()
defer b.pageTableMu.Unlock()
// get a free frame
frameID, isFromFreeList, err := b.getFrameID()
if err != nil {
@ -162,7 +212,7 @@ func (b *BufferPool) NewPage() (*Page, error) {
if !isFromFreeList {
// remove page from current frame
currentPage := b.pages[frameID]
currentPage := b.frames[frameID]
if currentPage != nil {
if currentPage.isDirty {
b.diskManager.WritePage(currentPage)
@ -173,20 +223,20 @@ func (b *BufferPool) NewPage() (*Page, error) {
}
// allocates new page
pageID, err := b.diskManager.AllocatePage()
pageID, err := b.diskManager.AllocatePage(fileID, shard)
if err != nil {
return nil, err
}
page := &Page{pageID, 1, false, [PAGE_SIZE]byte{}}
page.WritePageNumber(int32(pageID))
page := NewPage(pageID, 1)
page.WritePageNumber(pageID.Page)
page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
page.WriteNextPointer(int32(INVALID_PAGE))
page.WritePrevPointer(int32(INVALID_PAGE))
page.WriteNextPointer(PageID{0, 0, INVALID_PAGE})
page.WritePrevPointer(PageID{0, 0, INVALID_PAGE})
// update the frame table
b.pageTable[pageID] = frameID
pageSyncPool.Put(b.pages[frameID])
b.pages[frameID] = page
pageSyncPool.Put(b.frames[frameID])
b.frames[frameID] = page
return page, nil
}
@ -197,27 +247,32 @@ func (b *BufferPool) NewPage() (*Page, error) {
// and will copy the scratch page back over a real page later
func (b *BufferPool) ScratchPage() *Page {
page := &Page{
id: PageID(INVALID_PAGE),
id: PageID{0, 0, INVALID_PAGE},
pinCount: 0,
isDirty: false,
data: [PAGE_SIZE]byte{},
}
page.WritePageNumber(int32(INVALID_PAGE))
page.WritePageNumber(INVALID_PAGE)
page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
page.WriteNextPointer(int32(INVALID_PAGE))
page.WritePrevPointer(int32(INVALID_PAGE))
page.WriteNextPointer(PageID{0, 0, INVALID_PAGE})
page.WritePrevPointer(PageID{0, 0, INVALID_PAGE})
return page
}
// DeletePage deletes a page from the buffer pool
func (b *BufferPool) DeletePage(pageID PageID) error {
b.framesMu.Lock()
b.pageTableMu.Lock()
defer b.framesMu.Unlock()
defer b.pageTableMu.Unlock()
var frameID FrameID
var ok bool
if frameID, ok = b.pageTable[pageID]; !ok {
return nil
}
page := b.pages[frameID]
page := b.frames[frameID]
if page.pinCount > 0 {
return errors.New("pin count greater than 0")
@ -231,14 +286,6 @@ func (b *BufferPool) DeletePage(pageID PageID) error {
return nil
}
// FlushAllpages flushes all the pages in the buffer pool to disk
// Yeah, never call this unless you know what you are doing
func (b *BufferPool) FlushAllpages() {
for pageID := range b.pageTable {
b.FlushPage(pageID)
}
}
func (b *BufferPool) getFrameID() (FrameID, bool, error) {
if len(b.freeList) > 0 {
frameID, newFreeList := b.freeList[0], b.freeList[1:]
@ -250,12 +297,6 @@ func (b *BufferPool) getFrameID() (FrameID, bool, error) {
return victim, false, err
}
// OnDiskSize exposes the on disk size of the backing store
// behind this buffer pool
func (b *BufferPool) OnDiskSize() int64 {
return b.diskManager.FileSize()
}
// Close closes the buffer pool
func (b *BufferPool) Close() {
b.diskManager.Close()

5
bufferpool/diskmanager.go
View file

@ -4,17 +4,18 @@ package bufferpool
type DiskManager interface {
// reads a page from the disk
ReadPage(PageID) (*Page, error)
// writes a page to the disk
WritePage(*Page) error
// allocates a page
AllocatePage() (PageID, error)
AllocatePage(objectID int32, shard int32) (PageID, error)
// deallocates a page
DeallocatePage(PageID) error
// returns on disk file size
FileSize() int64
FileSize(objectID int32, shard int32) int64
// closes and does any clean up
Close()

33
bufferpool/inmemdiskmanager.go
View file

@ -1,3 +1,4 @@
// Copyright 2023 Molecula Corp. All rights reserved.
package bufferpool
import (
@ -12,12 +13,12 @@ import (
// that can spill to disk when a threshold is reached
type InMemDiskSpillingDiskManager struct {
// tracks the number of pages
numPages int
numPages int64
onDiskPages int
onDiskPages int64
// tracks the number of pages we can consume before spilling
thresholdPages int
thresholdPages int64
hasSpilled *struct{}
fd *os.File
@ -26,7 +27,7 @@ type InMemDiskSpillingDiskManager struct {
}
// NewInMemDiskSpillingDiskManager returns a in-memory version of disk manager
func NewInMemDiskSpillingDiskManager(thresholdPages int) *InMemDiskSpillingDiskManager {
func NewInMemDiskSpillingDiskManager(thresholdPages int64) *InMemDiskSpillingDiskManager {
dm := &InMemDiskSpillingDiskManager{
numPages: 0,
thresholdPages: thresholdPages,
@ -38,11 +39,11 @@ func NewInMemDiskSpillingDiskManager(thresholdPages int) *InMemDiskSpillingDiskM
// ReadPage reads a page from pages
func (d *InMemDiskSpillingDiskManager) ReadPage(pageID PageID) (*Page, error) {
// check we're not asking for page out of range
if pageID < 0 || int(pageID) >= d.numPages {
if pageID.Page < 0 || pageID.Page >= d.numPages {
return nil, errors.New("page not found")
}
// check that the offset is within range
offset := int(pageID) * PAGE_SIZE
offset := pageID.Page * PAGE_SIZE
var page = pageSyncPool.Get().(*Page)
// we have to do this stupid check because if -cpuprofile is set for go test, this
@ -54,7 +55,7 @@ func (d *InMemDiskSpillingDiskManager) ReadPage(pageID PageID) (*Page, error) {
// do the read
if d.hasSpilled == nil {
if offset+PAGE_SIZE > len(d.data) {
if offset+PAGE_SIZE > int64(len(d.data)) {
return nil, errors.New("offset out of range")
}
b := copy(page.data[:], d.data[offset:offset+PAGE_SIZE])
@ -75,10 +76,10 @@ func (d *InMemDiskSpillingDiskManager) ReadPage(pageID PageID) (*Page, error) {
// WritePage writes a page in memory to pages
func (d *InMemDiskSpillingDiskManager) WritePage(page *Page) error {
// make sure the offset is sensible
offset := int(page.ID()) * PAGE_SIZE
offset := page.ID().Page * PAGE_SIZE
// do the write
if d.hasSpilled == nil {
if offset+PAGE_SIZE > len(d.data) {
if offset+PAGE_SIZE > int64(len(d.data)) {
return errors.New("offset out of range")
}
copy(d.data[offset:], page.data[:])
@ -100,9 +101,9 @@ func (d *InMemDiskSpillingDiskManager) WritePage(page *Page) error {
}
// AllocatePage allocates a page and returns the page number
func (d *InMemDiskSpillingDiskManager) AllocatePage() (PageID, error) {
func (d *InMemDiskSpillingDiskManager) AllocatePage(objectID int32, shard int32) (PageID, error) {
d.numPages = d.numPages + 1
pageID := PageID(d.numPages - 1)
pageID := PageID{objectID, shard, int64(d.numPages - 1)}
if d.hasSpilled == nil {
// we have not spilled (yet), so make storage bigger
@ -113,16 +114,16 @@ func (d *InMemDiskSpillingDiskManager) AllocatePage() (PageID, error) {
if d.numPages > d.thresholdPages {
fileUUID, err := uuid.NewV4()
if err != nil {
return PageID(INVALID_PAGE), err
return PageID{objectID, shard, INVALID_PAGE}, err
}
// TODO(pok) we should try to tell the OS not to cache this file
d.fd, err = os.CreateTemp("", fmt.Sprintf("fb-ehash-%s", fileUUID.String()))
if err != nil {
return PageID(INVALID_PAGE), err
return PageID{objectID, shard, INVALID_PAGE}, err
}
_, err = d.fd.WriteAt(d.data, 0)
if err != nil {
return PageID(INVALID_PAGE), err
return PageID{objectID, shard, INVALID_PAGE}, err
}
d.data = []byte{}
d.hasSpilled = &struct{}{}
@ -135,7 +136,7 @@ func (d *InMemDiskSpillingDiskManager) AllocatePage() (PageID, error) {
size := int64(d.onDiskPages * PAGE_SIZE)
_, err = d.fd.WriteAt([]byte{0}, size-1)
if err != nil {
return PageID(INVALID_PAGE), err
return PageID{objectID, shard, INVALID_PAGE}, err
}
}
}
@ -149,7 +150,7 @@ func (d *InMemDiskSpillingDiskManager) DeallocatePage(pageID PageID) error {
return nil
}
func (d *InMemDiskSpillingDiskManager) FileSize() int64 {
func (d *InMemDiskSpillingDiskManager) FileSize(fileID int32, shard int32) int64 {
return int64(len(d.data))
}

267
bufferpool/ondiskdiskmanager.go Normal file
View file

@ -0,0 +1,267 @@
// Copyright 2023 Molecula Corp. All rights reserved.
package bufferpool
import (
"fmt"
"os"
"sync"
"github.com/pkg/errors"
)
type FileShardID struct {
ObjectId int32
Shard int32
}
// OnDiskDiskManager is a on disk implementation for a DiskManager interface
type OnDiskDiskManager struct {
mu sync.Mutex
files map[FileShardID]*os.File
}
// NewInMemDiskSpillingDiskManager returns a in-memory version of disk manager
func NewOnDiskDiskManager() *OnDiskDiskManager {
dm := &OnDiskDiskManager{
files: make(map[FileShardID]*os.File),
}
return dm
}
func (d *OnDiskDiskManager) CreateOrOpenShard(objectId int32, shard int32, dataFile string) error {
// serialize access here
d.mu.Lock()
defer d.mu.Unlock()
fileID := FileShardID{objectId, shard}
// do we have the file already open?
_, ok := d.files[fileID]
if ok {
return nil
}
var fd *os.File
// see if the file for this shard exists
_, err := os.Stat(dataFile)
if err != nil {
//create file
fd, err = os.OpenFile(dataFile, os.O_RDWR|os.O_CREATE, 0o600)
if err != nil {
return fmt.Errorf("open file: %w", err)
}
// write the root page
err = d.writeRootPage(fd, objectId, shard)
if err != nil {
return err
}
} else {
// open or create the file
fd, err = os.OpenFile(dataFile, os.O_RDWR, 0o600)
if err != nil {
return fmt.Errorf("open file: %w", err)
}
}
d.files[fileID] = fd
return nil
}
func (d *OnDiskDiskManager) writeRootPage(fd *os.File, objectId int32, shard int32) error {
headerPage := NewPage(PageID{objectId, shard, 0}, 0)
headerPage.WritePageNumber(0)
headerPage.WriteFreeSpaceOffset(int16(PAGE_SIZE))
headerPage.WriteNextPointer(PageID{0, 0, INVALID_PAGE})
headerPage.WritePrevPointer(PageID{0, 0, INVALID_PAGE})
headerPage.WritePageType(PAGE_TYPE_BTREE_HEADER)
// write a slot that points to the initial root page
rootPageID := PageID{objectId, shard, 1}
// get the free space offset
freeSpaceOffset := headerPage.ReadFreeSpaceOffset()
keyBytes := []byte{0, 0, 0, 0}
// build a chunk
chunk := InternalPageChunk{
KeyLength: int16(len(keyBytes)),
KeyBytes: keyBytes,
PtrValue: 1,
}
// compute the new free space offset
freeSpaceOffset -= int16(chunk.Length())
headerPage.WriteInternalPageChunk(freeSpaceOffset, chunk)
// update the free space offset
headerPage.WriteFreeSpaceOffset(int16(freeSpaceOffset))
// make a slot
slot := PageSlot{
PayloadOffset: freeSpaceOffset,
}
// write the slot
headerPage.WritePageSlot(0, slot)
// update the slot count
headerPage.WriteSlotCount(int16(1))
_, err := fd.WriteAt(headerPage.data[:], 0)
if err != nil {
return err
}
rootPage := NewPage(rootPageID, 0)
rootPage.WritePageNumber(1)
rootPage.WriteFreeSpaceOffset(int16(PAGE_SIZE))
rootPage.WriteNextPointer(PageID{0, 0, INVALID_PAGE})
rootPage.WritePrevPointer(PageID{0, 0, INVALID_PAGE})
rootPage.WritePageType(PAGE_TYPE_BTREE_LEAF)
_, err = fd.WriteAt(rootPage.data[:], PAGE_SIZE)
if err != nil {
return err
}
return nil
}
// ReadPage reads a page from disk
func (d *OnDiskDiskManager) ReadPage(pageID PageID) (*Page, error) {
d.mu.Lock()
defer d.mu.Unlock()
// do we have the file open
fileID := FileShardID{pageID.ObjectID, pageID.Shard}
fd, ok := d.files[fileID]
if !ok {
return nil, errors.Errorf("file id '%d' not open", pageID.ObjectID)
}
info, err := os.Stat(fd.Name())
if err != nil {
return nil, err
}
numPages := info.Size() / int64(PAGE_SIZE)
// check we're not asking for page out of range
if pageID.Page < 0 || pageID.Page >= numPages {
return nil, errors.Errorf("requested page number '%d' out of range", pageID.Page)
}
offset := pageID.Page * int64(PAGE_SIZE)
var page = pageSyncPool.Get().(*Page)
// we have to do this stupid check because if -cpuprofile is set for go test, this
// the previous line return a weird nil-ish thing...
if page == (*Page)(nil) {
page = pageSyncPool.New().(*Page)
}
page.id = pageID
// do the read
_, err = fd.ReadAt(page.data[:], offset)
if err != nil {
return nil, err
}
return page, nil
}
// WritePage writes a page in memory to pages
func (d *OnDiskDiskManager) WritePage(page *Page) error {
d.mu.Lock()
defer d.mu.Unlock()
// do we have the file open
fileID := FileShardID{page.id.ObjectID, page.id.Shard}
fd, ok := d.files[fileID]
if !ok {
return errors.Errorf("file id '%d' not open", page.id.ObjectID)
}
info, err := os.Stat(fd.Name())
if err != nil {
return err
}
numPages := info.Size() / int64(PAGE_SIZE)
// check we're not asking for page out of range
if page.id.Page < 0 || page.id.Page >= numPages {
return errors.Errorf("requested page number '%d' out of range", page.id.Page)
}
offset := page.id.Page * int64(PAGE_SIZE)
// do the write
_, err = fd.WriteAt(page.data[:], offset)
if err != nil {
return err
}
// yeah, nah
// err = d.fd.Sync()
// if err != nil {
// return err
// }
return nil
}
// AllocatePage allocates a page and returns the page number
func (d *OnDiskDiskManager) AllocatePage(objectId int32, shard int32) (PageID, error) {
d.mu.Lock()
defer d.mu.Unlock()
// do we have the file open
fileID := FileShardID{objectId, shard}
fd, ok := d.files[fileID]
if !ok {
return PageID{objectId, shard, INVALID_PAGE}, errors.Errorf("file id '%d' not open", objectId)
}
info, err := os.Stat(fd.Name())
if err != nil {
return PageID{objectId, shard, INVALID_PAGE}, err
}
numPages := info.Size() / int64(PAGE_SIZE)
numPages = numPages + 1
pageID := PageID{objectId, shard, numPages - 1}
size := numPages * PAGE_SIZE
_, err = fd.WriteAt([]byte{0}, size-1)
if err != nil {
return PageID{objectId, shard, INVALID_PAGE}, err
}
return pageID, nil
}
// DeallocatePage removes page from disk
func (d *OnDiskDiskManager) DeallocatePage(pageID PageID) error {
d.mu.Lock()
defer d.mu.Unlock()
// nothing to do right now
return nil
}
func (d *OnDiskDiskManager) FileSize(objectId int32, shard int32) int64 {
// TODO(pok) return correct file size
return 0
}
func (d *OnDiskDiskManager) Close() {
d.mu.Lock()
defer d.mu.Unlock()
for _, fd := range d.files {
fd.Close()
}
// empty the efiles
d.files = make(map[FileShardID]*os.File, 0)
}

624
bufferpool/page.go
View file

@ -4,17 +4,20 @@ import (
"encoding/binary"
"errors"
"fmt"
"sync"
)
const PAGE_SIZE int = 8192
const PAGE_SIZE int64 = 8192
const INVALID_PAGE int = -1
const INVALID_PAGE int64 = -1
const PAGE_TYPE_BTREE_OVERFLOW = 8
const PAGE_TYPE_BTREE_HEADER = 9
const PAGE_TYPE_BTREE_INTERNAL = 10
const PAGE_TYPE_BTREE_LEAF = 11
const PAGE_TYPE_HASH_TABLE = 12
// PAGE
// SLOTTED PAGE
// page size 8192 bytes
// byte aligned, big endian
@ -23,111 +26,146 @@ const PAGE_TYPE_HASH_TABLE = 12
// |----------------------------------------------------|
// | header |
// |====================================================|
// | 0 | 4 | pageNumber (int32) |
// | 4 | 2 | pageType (int16) |
// | 6 | 2 | slotCount (int16) |
// | 8 | 2 | localDepth (int16) |
// | 10 | 2 | freeSpaceOffset (int16) |
// | 12 | 4 | prevPointer (int32) |
// | 16 | 4 | nextPointer (int32) |
// | 0 | 8 | pageNumber (int64) |
// | 8 | 2 | pageType (int16) |
// | 10 | 2 | slotCount (int16) |
// | 12 | 2 | localDepth (int16) | // used only for hash tables
// | 14 | 2 | freeSpaceOffset (int16) |
// | 16 | 8 | prevPointer (int64) |
// | 24 | 8 | nextPointer (int64) |
// | 32 | 4 | checksum (int32) |
// | 36 | 16 | pageLSN (int64) |
// |====================================================|
// | <start of slot array 1..slotCount> |
// |----------------------------------------------------|
// | 20 | slotcount | slot entry is 2 int16 |
// | 52 | slotcount | slot entry is int16 |
// | | * slotwidth | values (payloadOffset, |
// | | * #slots | payloadLength) |
// |----------------------------------------------------|
// | <free space> |
// |----------------------------------------------------|
// | <payload starting at freeSpaceOffset> |
// | payload entries are keylength (int16), key bytes, |
// | payload length (int32), payload bytes |
// | see below... |
// |====================================================|
const PAGE_NUMBER_OFFSET = 0 // offset 0, length 4, end 4
const PAGE_TYPE_OFFSET = 4 // offset 4, length 2, end 6
const PAGE_SLOT_COUNT_OFFSET = 6 // offset 6, length 2, end 8
const PAGE_LOCAL_DEPTH_OFFSET = 8 // offset 8, length 2, end 10
const PAGE_FREE_SPACE_OFFSET = 10 // offset 10, length 2, end 12
const PAGE_PREV_POINTER_OFFSET = 12 // offset 12, length 4, end 16
const PAGE_NEXT_POINTER_OFFSET = 16 // offset 16, length 4, end 20
const PAGE_SLOTS_START_OFFSET = 20 // offset 20
// == internal payload chunk ==
// keyLength (int16)
// keyBytes
// ptrValue (int64) (page number)
// PAGE_SLOT_LENGTH is the size of the page slot key/value.
//
// key offset int16 //offset 0, length 2, end 2
// value offset int16 //offset 2, length 2, end 4
const PAGE_SLOT_LENGTH = 4
// == tuple payload chunk ==
// keyLength (int16)
// keyBytes
// flags int8
// overflowPtr (int64) [optional]
// rowPayloadTotalLen (int32) [optional]
// rowPayloadChunkLen (int16)
// rowPayloadChunkBytes
// Page represents a page on disk
// == row payload bytes ==
// writeTID (int64)
// schemaVersion (int16)
// redoPtr (int64)
// fieldOffsets (one for each field, int16, FF is null)
// offsets point to:
// *fieldData (one for each field)
// valueLen (int32) only used for variable length types
// valueBytes
const PAGE_NUMBER_OFFSET = 0 // offset 0, length 8, end 8
const PAGE_TYPE_OFFSET = 8 // offset 8, length 2, end 10
const PAGE_SLOT_COUNT_OFFSET = 10 // offset 10, length 2, end 12
const PAGE_LOCAL_DEPTH_OFFSET = 12 // offset 12, length 2, end 14
const PAGE_FREE_SPACE_OFFSET = 14 // offset 14, length 2, end 16
const PAGE_PREV_POINTER_OFFSET = 16 // offset 16, length 8, end 24
const PAGE_NEXT_POINTER_OFFSET = 24 // offset 24, length 8, end 32
const PAGE_CHECKSUM = 32 // offset 32, length 4, end 36
const PAGE_LSN = 36 // offset 36, length 16, end 52
const PAGE_SLOTS_START_OFFSET = 52 // offset 52
// PAGE_SLOT_LENGTH is the size of the page slot offset value.
const PAGE_SLOT_LENGTH = 2
// 1k is the max key size for now
// we can make this bigger later with overflow
const MAX_KEY_ON_PAGE_SIZE = 1024
// 768 bytes is the max payload on a page before we overflow
// this gives us 1792 bytes for key and payload
// available space on a page after header is 8144 ish
// this gives us room for 4 key/value @ 2036 bytes per page, so
// there is a little bit of wiggle room
const MAX_PAYLOAD_CHUNK_SIZE_ON_PAGE = 768
type PageLatchState int
const (
None PageLatchState = iota
Read
Write
)
// Page represents various types of data page on disk and in memory
type Page struct {
id PageID
pinCount int
isDirty bool
data [PAGE_SIZE]byte
mu sync.RWMutex
latchState PageLatchState
id PageID
pinCount int
isDirty bool
data [PAGE_SIZE]byte
}
type PageSlot struct {
KeyOffset int16
ValueOffset int16
func NewPage(pageID PageID, pinCount int) *Page {
return &Page{
id: pageID,
latchState: None,
pinCount: pinCount,
isDirty: false,
data: [PAGE_SIZE]byte{},
}
}
func (s *PageSlot) KeyBytes(page *Page) []byte {
offset := s.KeyOffset
keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
offset += 2
result := make([]byte, keyLen)
copy(result, page.data[offset:offset+keyLen])
return result
func (p *Page) TakeReadLatch() {
p.mu.RLock()
p.latchState = Read
}
func (s *PageSlot) KeyAsInt(page *Page) int32 {
return int32(binary.BigEndian.Uint32(page.data[s.KeyOffset+2:]))
func (p *Page) ReleaseReadLatch() {
p.mu.RUnlock()
p.latchState = None
}
func (s *PageSlot) ValueBytes(page *Page) []byte {
offset := s.ValueOffset
valueLen := int32(binary.BigEndian.Uint32(page.data[offset:]))
offset += 4
result := make([]byte, valueLen)
copy(result, page.data[offset:int32(offset)+valueLen])
return result
func (p *Page) TakeWriteLatch() {
p.mu.Lock()
p.latchState = Write
}
func (s *PageSlot) ValueAsPagePointer(page *Page) int32 {
return int32(binary.BigEndian.Uint32(page.data[s.ValueOffset+4:]))
func (p *Page) ReleaseWriteLatch() {
p.mu.Unlock()
p.latchState = None
}
type PageChunk struct {
KeyLength int16
KeyBytes []byte
// TODO(pok) ValueBytes can be up to int32 long
// this requires an overflow page mechanism, that is not implemented
// yet, so be aware of this when storing stuff...
ValueLength int32
ValueBytes []byte
func (p *Page) ReleaseAnyLatch() {
if p.latchState == Read {
p.ReleaseReadLatch()
} else if p.latchState == Write {
p.ReleaseWriteLatch()
}
}
func (pc *PageChunk) Length() int {
return 2 + len(pc.KeyBytes) + 4 + len(pc.ValueBytes)
func (p *Page) LatchState() PageLatchState {
return p.latchState
}
func (pc *PageChunk) ComputeKeyOffset(pageOffset int) int {
return pageOffset
}
func (pc *PageChunk) ComputeValueOffset(pageOffset int) int {
return pageOffset + 2 + len(pc.KeyBytes)
}
func (p *Page) WritePageNumber(pageNumber int32) {
p.id = PageID(pageNumber)
binary.BigEndian.PutUint32(p.data[PAGE_NUMBER_OFFSET:], uint32(pageNumber))
// routines to read and write from page header information
func (p *Page) WritePageNumber(pageNumber int64) {
p.id.Page = pageNumber
binary.BigEndian.PutUint64(p.data[PAGE_NUMBER_OFFSET:], uint64(pageNumber))
p.isDirty = true
}
func (p *Page) ReadPageNumber() int {
return int(binary.BigEndian.Uint32(p.data[PAGE_NUMBER_OFFSET:]))
func (p *Page) ReadPageNumber() int64 {
return int64(binary.BigEndian.Uint32(p.data[PAGE_NUMBER_OFFSET:]))
}
func (p *Page) WritePageType(pageType int16) {
@ -157,24 +195,6 @@ func (p *Page) ReadLocalDepth() int16 {
return int16(binary.BigEndian.Uint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:]))
}
func (p *Page) ReadSlot(slot int16) PageSlot {
offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
keyOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
offset += 2
valueOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
return PageSlot{
KeyOffset: keyOffset,
ValueOffset: valueOffset,
}
}
func (p *Page) WriteSlot(slot int16, value PageSlot) {
offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
binary.BigEndian.PutUint16(p.data[offset:], uint16(value.KeyOffset))
offset += 2
binary.BigEndian.PutUint16(p.data[offset:], uint16(value.ValueOffset))
}
func (p *Page) WriteFreeSpaceOffset(offset int16) {
binary.BigEndian.PutUint16(p.data[PAGE_FREE_SPACE_OFFSET:], uint16(offset))
p.isDirty = true
@ -184,106 +204,152 @@ func (p *Page) ReadFreeSpaceOffset() int16 {
return int16(binary.BigEndian.Uint16(p.data[PAGE_FREE_SPACE_OFFSET:]))
}
func (p *Page) WritePrevPointer(prevPointer int32) {
binary.BigEndian.PutUint32(p.data[PAGE_PREV_POINTER_OFFSET:], uint32(prevPointer))
func (p *Page) WritePrevPointer(prevPointer PageID) {
binary.BigEndian.PutUint64(p.data[PAGE_PREV_POINTER_OFFSET:], uint64(prevPointer.Page))
p.isDirty = true
}
func (p *Page) ReadPrevPointer() int {
return int(binary.BigEndian.Uint32(p.data[PAGE_PREV_POINTER_OFFSET:]))
func (p *Page) ReadPrevPointer() PageID {
page := int64(binary.BigEndian.Uint64(p.data[PAGE_PREV_POINTER_OFFSET:]))
return PageID{ObjectID: p.id.ObjectID, Shard: p.id.Shard, Page: page}
}
func (p *Page) WriteNextPointer(nextPointer int32) {
binary.BigEndian.PutUint32(p.data[PAGE_NEXT_POINTER_OFFSET:], uint32(nextPointer))
func (p *Page) WriteNextPointer(nextPointer PageID) {
binary.BigEndian.PutUint64(p.data[PAGE_NEXT_POINTER_OFFSET:], uint64(nextPointer.Page))
p.isDirty = true
}
func (p *Page) ReadNextPointer() int {
return int(binary.BigEndian.Uint32(p.data[PAGE_NEXT_POINTER_OFFSET:]))
func (p *Page) ReadNextPointer() PageID {
page := int64(binary.BigEndian.Uint64(p.data[PAGE_NEXT_POINTER_OFFSET:]))
return PageID{ObjectID: p.id.ObjectID, Shard: p.id.Shard, Page: page}
}
func (p *Page) WriteChunk(offset int16, chunk PageChunk) {
binary.BigEndian.PutUint16(p.data[offset:], uint16(chunk.KeyLength))
// read slots from pages
func (p *Page) ReadPageSlot(slot int16) PageSlot {
offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
keyOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
offset += 2
copy(p.data[offset:], chunk.KeyBytes)
offset += int16(len(chunk.KeyBytes))
binary.BigEndian.PutUint32(p.data[offset:], uint32(chunk.ValueLength))
offset += 4
copy(p.data[offset:], chunk.ValueBytes)
p.isDirty = true
}
func (p *Page) ReadChunk(offset int16) PageChunk {
keyLen := int16(binary.BigEndian.Uint16(p.data[offset:]))
offset += 2
keyBytes := make([]byte, keyLen)
copy(keyBytes, p.data[offset:offset+keyLen])
offset += keyLen
valueLen := int32(binary.BigEndian.Uint32(p.data[offset:]))
offset += 4
valueBytes := make([]byte, valueLen)
copy(valueBytes, p.data[offset:int32(offset)+valueLen])
return PageChunk{
KeyLength: keyLen,
KeyBytes: keyBytes,
ValueLength: valueLen,
ValueBytes: valueBytes,
return PageSlot{
PayloadOffset: keyOffset,
}
}
func (p *Page) FreeSpace() int16 {
freeSpaceOffset := p.ReadFreeSpaceOffset()
freespace := freeSpaceOffset - (p.ReadSlotCount()*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET)
return freespace
func (p *Page) WritePageSlot(slot int16, value PageSlot) {
offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
binary.BigEndian.PutUint16(p.data[offset:], uint16(value.PayloadOffset))
}
func (p *Page) WriteKeyValueInSlot(slotNumber int16, key []byte, value []byte) error {
// TODO(pok) deprecate this once we get b+tree working, this
// is just used in hash table right now
func (p *Page) PutKeyValueInPageSlot(slotNumber int16, keyBytes []byte, payloadBytes []byte) error {
freeSpaceOffset := p.ReadFreeSpaceOffset()
// build a chunk
chunk := PageChunk{
KeyLength: int16(len(key)),
KeyBytes: key,
ValueLength: int32(len(value)),
ValueBytes: value,
keyLength := len(keyBytes)
payloadChunkLength := len(payloadBytes)
// check for overflow
if payloadChunkLength > MAX_PAYLOAD_CHUNK_SIZE_ON_PAGE {
return errors.New("overflow")
}
// compute the new free space offset
freeSpaceOffset -= int16(chunk.Length())
totalPayloadSize := p.ComputeLeafPayloadTotalLength(keyLength, payloadChunkLength)
// check we won't blow free space on page
slotCount := p.ReadSlotCount()
slotEndOffset := slotCount*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET
// DEBUG!!
//fmt.Printf("freeSpaceOffset: %d, slotCount: %d, slotCount*4 + 4 + 20: %d, freeSpace: %d\n", freeSpaceOffset, slotCount, slotEndOffset, freeSpaceOffset-slotEndOffset)
if freeSpaceOffset-slotEndOffset <= 0 {
// check to make sure we don't blow free space
if p.FreeSpaceOnPage() < int16(totalPayloadSize) {
return errors.New("page is full")
}
keyOffset := chunk.ComputeKeyOffset(int(freeSpaceOffset))
valueOffset := chunk.ComputeValueOffset(int(freeSpaceOffset))
// compute the new free space offset
freeSpaceOffset -= int16(totalPayloadSize)
offset := freeSpaceOffset
p.WriteChunk(freeSpaceOffset, chunk)
// write the header
offset = p.WriteLeafPagePayloadHeader(offset, int16(keyLength), keyBytes, 0, 0, int32(payloadChunkLength))
// write the payload
p.WriteLeafPagePayloadBytes(offset, int16(payloadChunkLength), payloadBytes)
// update the free space offset
p.WriteFreeSpaceOffset(int16(freeSpaceOffset))
// make a slot
slot := PageSlot{
KeyOffset: int16(keyOffset),
ValueOffset: int16(valueOffset),
PayloadOffset: freeSpaceOffset,
}
// write the slot
p.WriteSlot(slotNumber, slot)
p.WritePageSlot(slotNumber, slot)
return nil
}
func (p *Page) WritePage(page *Page) {
func (p *Page) ComputeInternalPayloadTotalLength(keyLength int) int32 {
l := /*keyLength*/ 2 + keyLength + /*ptrValue*/ 8
return int32(l)
}
func (p *Page) WriteInternalPagePayload(offset int16, keyLen int16, keyBytes []byte, ptrValue int64) {
binary.BigEndian.PutUint16(p.data[offset:], uint16(keyLen))
offset += 2
copy(p.data[offset:], keyBytes)
offset += keyLen
binary.BigEndian.PutUint64(p.data[offset:], uint64(ptrValue))
p.isDirty = true
}
func (p *Page) ComputeLeafPayloadTotalLength(keyLength int, payloadLength int) int32 {
l := /*keyLength*/ 2 + keyLength + /*flags*/ 1 + /*payLoadLength*/ 2 + payloadLength
if l > MAX_PAYLOAD_CHUNK_SIZE_ON_PAGE {
l += 8 + 4 // add overflow ptr and total payload size
}
return int32(l)
}
func (p *Page) WriteLeafPagePayloadHeader(offset int16, keyLen int16, keyBytes []byte, flags int8, overflowPtr int64, payloadTotalLen int32) int16 {
binary.BigEndian.PutUint16(p.data[offset:], uint16(keyLen))
offset += 2
copy(p.data[offset:], keyBytes)
offset += keyLen
p.data[offset] = byte(flags)
offset += 1
// if we are overflowing write the overflow ptr
if flags == 1 {
binary.BigEndian.PutUint64(p.data[offset:], uint64(overflowPtr))
offset += 8
// total length
binary.BigEndian.PutUint32(p.data[offset:], uint32(payloadTotalLen))
offset += 4
}
p.isDirty = true
return offset
}
func (p *Page) WriteLeafPagePayloadBytes(offset int16, payloadChunkLength int16, payloadChunkBytes []byte) {
// chunk length
binary.BigEndian.PutUint16(p.data[offset:], uint16(payloadChunkLength))
offset += 2
// now copy the payload bytes
copy(p.data[offset:], payloadChunkBytes)
p.isDirty = true
}
func (p *Page) WriteInternalPageChunk(offset int16, chunk InternalPageChunk) {
binary.BigEndian.PutUint16(p.data[offset:], uint16(chunk.KeyLength))
offset += 2
copy(p.data[offset:], chunk.KeyBytes)
offset += int16(len(chunk.KeyBytes))
binary.BigEndian.PutUint64(p.data[offset:], uint64(chunk.PtrValue))
p.isDirty = true
}
func (p *Page) FreeSpaceOnPage() int16 {
freeSpaceOffset := p.ReadFreeSpaceOffset()
freespace := freeSpaceOffset - (p.ReadSlotCount()*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET)
return freespace
}
func (p *Page) CopyPageTo(page *Page) {
// copy everything but pageNumber & pageType
offset := PAGE_SLOT_COUNT_OFFSET
offset := int64(PAGE_SLOT_COUNT_OFFSET)
copy(page.data[offset:], p.data[offset:offset+PAGE_SIZE-offset])
}
@ -301,6 +367,211 @@ func (p *Page) DecPinCount() {
}
}
func (pg *Page) Dump(label string) {
indent := 0
if len(label) > 0 {
fmt.Printf("%s%s:\n", fmt.Sprintf("%*s", indent, ""), label)
indent += 4
}
pageType := pg.ReadPageType()
fmt.Printf("%sPAGE(%d) pageType: %d slotCount: %d, prevPtr: %d, nextPtr: %d\n", fmt.Sprintf("%*s", indent, ""), pg.ID(), pageType, pg.ReadSlotCount(), pg.ReadPrevPointer(), pg.ReadNextPointer())
fmt.Printf("%sKEYS: -->\n", fmt.Sprintf("%*s", indent, ""))
indent += 4
// get the keys off the page
keys := make([]int, 0)
pointers := make([]PageID, 0)
iter := NewPageSlotIterator(pg, 0)
for {
ps := iter.Next()
if ps == nil {
break
}
pl := ps.KeyPayload(pg)
keys = append(keys, int(pl.KeyAsInt(pg)))
if pageType == PAGE_TYPE_BTREE_INTERNAL {
ipl := ps.InternalPayload(pg)
pointers = append(pointers, ipl.ValueAsPagePointer(pg))
}
}
if pageType == PAGE_TYPE_BTREE_LEAF {
for _, key := range keys {
fmt.Printf("%s(%d)\n", fmt.Sprintf("%*s", indent, ""), key)
}
} else {
for idx, key := range keys {
ptr := pointers[idx]
fmt.Printf("%s(%d, %d)\n", fmt.Sprintf("%*s", indent, ""), key, ptr)
}
ptr := pg.ReadNextPointer()
fmt.Printf("%s(-->, %d)\n", fmt.Sprintf("%*s", indent, ""), ptr)
}
}
type KeyPayload struct {
BaseOffset int16
}
func (p *KeyPayload) KeyAsInt(page *Page) int32 {
return int32(binary.BigEndian.Uint32(page.data[p.BaseOffset+2:]))
}
func (p *KeyPayload) KeyBytes(page *Page) []byte {
offset := p.BaseOffset
keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
offset += 2
result := make([]byte, keyLen)
copy(result, page.data[offset:offset+keyLen])
return result
}
type InternalPayload struct {
BaseOffset int16
}
func (p *InternalPayload) ptrOffset(page *Page) int16 {
offset := p.BaseOffset
keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
offset += 2 + keyLen
return offset
}
func (p *InternalPayload) ValueAsPagePointer(page *Page) PageID {
offset := p.ptrOffset(page)
pageid := int64(binary.BigEndian.Uint64(page.data[offset:]))
return PageID{ObjectID: page.id.ObjectID, Shard: page.id.Shard, Page: pageid}
}
func (p *InternalPayload) PutPagePointer(page *Page, pagePtr PageID) error {
offset := p.ptrOffset(page)
binary.BigEndian.PutUint64(page.data[offset:], uint64(pagePtr.Page))
page.isDirty = true
return nil
}
func (l *InternalPayload) InternalPageChunk(page *Page) InternalPageChunk {
offset := l.BaseOffset
keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
offset += 2
keyBytes := make([]byte, keyLen)
copy(keyBytes, page.data[offset:offset+keyLen])
offset += keyLen
ptrValue := int64(binary.BigEndian.Uint64(page.data[offset:]))
return InternalPageChunk{
KeyLength: keyLen,
KeyBytes: keyBytes,
PtrValue: ptrValue,
}
}
type LeafPayload struct {
BaseOffset int16
}
func (l *LeafPayload) valueOffset(page *Page) int16 {
offset := l.BaseOffset
keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
offset += 2 + keyLen
return offset
}
func (l *LeafPayload) ValueLength(page *Page) int32 {
offset := l.valueOffset(page)
valueLen := int32(binary.BigEndian.Uint32(page.data[offset:]))
return valueLen
}
// this wil fail in overflow
func (l *LeafPayload) ValueAsBytes(page *Page) []byte {
offset := l.valueOffset(page)
valueLen := int32(binary.BigEndian.Uint32(page.data[offset:]))
offset += 4
result := make([]byte, valueLen)
copy(result, page.data[offset:int32(offset)+valueLen])
return result
}
func (l *LeafPayload) GetPayloadReader(page *Page) LeafPagePayLoadReader {
offset := l.BaseOffset
keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
offset += 2
keyBytes := make([]byte, keyLen)
copy(keyBytes, page.data[offset:offset+keyLen])
offset += keyLen
flags := int8(page.data[offset])
offset += 1
overflowPtr := int64(0)
payloadTotalLength := int32(0)
if flags == 1 {
overflowPtr = int64(binary.BigEndian.Uint64(page.data[offset:]))
offset += 8
payloadTotalLength = int32(binary.BigEndian.Uint32(page.data[offset:]))
offset += 4
}
valueLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
offset += 2
if payloadTotalLength == 0 {
payloadTotalLength = int32(valueLen)
}
valueBytes := make([]byte, valueLen)
copy(valueBytes, page.data[offset:int16(offset)+valueLen])
return LeafPagePayLoadReader{
KeyLength: keyLen,
KeyBytes: keyBytes,
Flags: flags,
OverflowPtr: overflowPtr,
PayloadTotalLength: payloadTotalLength,
PayloadChunkLength: valueLen,
PayloadChunkBytes: valueBytes,
}
}
type PageSlot struct {
PayloadOffset int16
}
func (s *PageSlot) KeyPayload(page *Page) KeyPayload {
return KeyPayload{BaseOffset: s.PayloadOffset}
}
func (s *PageSlot) InternalPayload(page *Page) InternalPayload {
return InternalPayload{BaseOffset: s.PayloadOffset}
}
func (s *PageSlot) LeafPayload(page *Page) LeafPayload {
return LeafPayload{BaseOffset: s.PayloadOffset}
}
type LeafPagePayLoadReader struct {
KeyLength int16
KeyBytes []byte
Flags int8
OverflowPtr int64
PayloadTotalLength int32
PayloadChunkLength int16
PayloadChunkBytes []byte
}
func (pc *LeafPagePayLoadReader) Length() int32 {
l := /*keyLength*/ 2 + pc.KeyLength + /*flags*/ 1 + /*payLoadLength*/ 2 + pc.PayloadChunkLength
if pc.Flags == 1 {
l += 8 + 4 // add overflow ptr and total payload size
}
return int32(l)
}
type InternalPageChunk struct {
KeyLength int16
KeyBytes []byte
PtrValue int64
}
func (pc *InternalPageChunk) Length() int {
return 2 + len(pc.KeyBytes) + 8
}
type PageSlotIterator struct {
page *Page
slotCount int16
@ -318,7 +589,7 @@ func NewPageSlotIterator(page *Page, fromSlot int16) *PageSlotIterator {
func (i *PageSlotIterator) Next() *PageSlot {
if i.cursor < i.slotCount {
s := i.page.ReadSlot(i.cursor)
s := i.page.ReadPageSlot(i.cursor)
i.cursor++
return &s
}
@ -328,44 +599,3 @@ func (i *PageSlotIterator) Next() *PageSlot {
func (i *PageSlotIterator) Cursor() int16 {
return i.cursor
}
func (pg *Page) Dump(label string) {
indent := 0
if len(label) > 0 {
fmt.Printf("%s%s:\n", fmt.Sprintf("%*s", indent, ""), label)
indent += 4
}
pageType := pg.ReadPageType()
fmt.Printf("%sPAGE(%d) pageType: %d slotCount: %d, prevPtr: %d, nextPtr: %d\n", fmt.Sprintf("%*s", indent, ""), pg.ID(), pageType, pg.ReadSlotCount(), pg.ReadPrevPointer(), pg.ReadNextPointer())
fmt.Printf("%sKEYS: -->\n", fmt.Sprintf("%*s", indent, ""))
indent += 4
// get the keys off the page
keys := make([]int, 0)
pointers := make([]int, 0)
iter := NewPageSlotIterator(pg, 0)
for {
ps := iter.Next()
if ps == nil {
break
}
keys = append(keys, int(ps.KeyAsInt(pg)))
if pageType == /*nodeTypeInternal*/ 10 {
pointers = append(pointers, int(ps.ValueAsPagePointer(pg)))
}
}
if pageType == /*nodeTypeLeaf*/ 11 {
for _, key := range keys {
fmt.Printf("%s(%d)\n", fmt.Sprintf("%*s", indent, ""), key)
}
} else {
for idx, key := range keys {
ptr := pointers[idx]
fmt.Printf("%s(%d, %d)\n", fmt.Sprintf("%*s", indent, ""), key, ptr)
}
ptr := pg.ReadNextPointer()
fmt.Printf("%s(-->, %d)\n", fmt.Sprintf("%*s", indent, ""), ptr)
}
}

1
cluster.go
View file

@ -914,6 +914,7 @@ type CreateShardMessage struct {
// CreateIndexMessage is an internal message indicating index creation.
type CreateIndexMessage struct {
IndexID int32
Index string
CreatedAt int64
Owner string

1
dax/computer/logmessage.go
View file

@ -231,6 +231,7 @@ type ImportRoaringShardMessage struct {
Partition int `json:"partition"`
Shard uint64 `json:"shard"`
Views []RoaringUpdate `json:"views"`
Tuples []byte `json:"tuples"`
}
// RoaringUpdate is identical to featurebase.RoaringUpdate, but we

5
dax/table.go
View file

@ -83,6 +83,7 @@ const (
BaseTypeStringSet = "stringset" // keyed set
BaseTypeStringSetQ = "stringsetq" // keyed set timequantum
BaseTypeTimestamp = "timestamp" //
BaseTypeVarchar = "varchar" //
DefaultPartitionN = 256
@ -684,7 +685,8 @@ func BaseTypeFromString(s string) (BaseType, error) {
BaseTypeString,
BaseTypeStringSet,
BaseTypeStringSetQ,
BaseTypeTimestamp:
BaseTypeTimestamp,
BaseTypeVarchar:
return BaseType(lowered), nil
default:
return "", errors.Errorf("invalid field type: %s", s)
@ -824,4 +826,5 @@ type FieldOptions struct {
TTL time.Duration `json:"ttl,omitempty"`
ForeignIndex string `json:"foreign-index,omitempty"`
TrackExistence bool `json:"track-existence"`
Length int64 `json:"length,omitempty"`
}

6
encoding/proto/proto.go
View file

@ -589,6 +589,7 @@ func (s Serializer) encodeIndexInfos(idxs []*pilosa.IndexInfo) []*pb.Index {
func (s Serializer) encodeIndexInfo(idx *pilosa.IndexInfo) *pb.Index {
return &pb.Index{
IndexID: idx.ID,
Name: idx.Name,
CreatedAt: idx.CreatedAt,
Options: s.encodeIndexMeta(&idx.Options),
@ -638,6 +639,7 @@ func (s Serializer) encodeFieldOptions(o *pilosa.FieldOptions) *pb.FieldOptions
ForeignIndex: o.ForeignIndex,
NoStandardView: o.NoStandardView,
TrackExistence: o.TrackExistence,
Length: o.Length,
}
}
@ -688,6 +690,7 @@ func (s Serializer) encodeCreateShardMessage(m *pilosa.CreateShardMessage) *pb.C
func (s Serializer) encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *pb.CreateIndexMessage {
return &pb.CreateIndexMessage{
IndexID: m.IndexID,
Index: m.Index,
CreatedAt: m.CreatedAt,
Owner: m.Owner,
@ -929,6 +932,7 @@ func (s Serializer) decodeIndexes(idxs []*pb.Index, m []*pilosa.IndexInfo) {
}
func (s Serializer) decodeIndex(idx *pb.Index, m *pilosa.IndexInfo) {
m.ID = idx.IndexID
m.Name = idx.Name
m.CreatedAt = idx.CreatedAt
m.Options = pilosa.IndexOptions{}
@ -963,6 +967,7 @@ func (s Serializer) decodeFieldOptions(options *pb.FieldOptions, m *pilosa.Field
s.decodeDecimal(options.Max, &m.Max)
m.Base = options.Base
m.Scale = options.Scale
m.Length = options.Length
m.BitDepth = uint64(options.BitDepth)
m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum)
ttlValue, err := time.ParseDuration(options.TTL)
@ -1032,6 +1037,7 @@ func (s Serializer) decodeCreateShardMessage(pb *pb.CreateShardMessage, m *pilos
}
func (s Serializer) decodeCreateIndexMessage(pb *pb.CreateIndexMessage, m *pilosa.CreateIndexMessage) {
m.IndexID = pb.IndexID
m.Index = pb.Index
m.CreatedAt = pb.CreatedAt
m.Owner = pb.Owner

48
extendiblehash/extendiblehash.go
View file

@ -19,10 +19,10 @@ type ExtendibleHashTable struct {
// NewExtendibleHashTable creates a new ExtendibleHashTable
func NewExtendibleHashTable(keyLength int, valueLength int, bufferPool *bufferpool.BufferPool) (*ExtendibleHashTable, error) {
bytesPerKV := keyLength + valueLength + bufferpool.PAGE_SLOT_LENGTH
keysPerPage := (bufferpool.PAGE_SIZE - bufferpool.PAGE_SLOTS_START_OFFSET) / bytesPerKV
keysPerPage := int(bufferpool.PAGE_SIZE-bufferpool.PAGE_SLOTS_START_OFFSET) / bytesPerKV
//create the root page
page, err := bufferPool.NewPage()
page, err := bufferPool.NewPage(0, 0)
if err != nil {
return nil, err
}
@ -53,8 +53,9 @@ func (e *ExtendibleHashTable) Get(key []byte) ([]byte, bool, error) {
index, found := e.findKey(page, key)
if found {
slot := page.ReadSlot(int16(index))
return slot.ValueBytes(page), true, nil
slot := page.ReadPageSlot(int16(index))
lpl := slot.LeafPayload(page)
return lpl.ValueAsBytes(page), true, nil
}
return []byte{}, false, nil
}
@ -99,7 +100,7 @@ func (e *ExtendibleHashTable) hashFunction(k Hashable) int {
func (e *ExtendibleHashTable) getPageID(key []byte) (bufferpool.PageID, error) {
hash := e.hashFunction(Key(key))
if hash > len(e.directory)-1 {
return 0, fmt.Errorf("hash (%d) out of the directory array bounds (%d)", hash, len(e.directory))
return bufferpool.PageID{ObjectID: 0, Shard: 0, Page: bufferpool.INVALID_PAGE}, fmt.Errorf("hash (%d) out of the directory array bounds (%d)", hash, len(e.directory))
}
id := e.directory[hash]
return bufferpool.PageID(id), nil
@ -111,8 +112,9 @@ func (e *ExtendibleHashTable) findKey(page *bufferpool.Page, key []byte) (int, b
for onePastMaxIndex != minIndex {
index := (minIndex + onePastMaxIndex) / 2
s := page.ReadSlot(int16(index))
keyAtIndex := s.KeyBytes(page)
s := page.ReadPageSlot(int16(index))
pl := s.KeyPayload(page)
keyAtIndex := pl.KeyBytes(page)
if bytes.Equal(keyAtIndex, key) {
return index, true
@ -135,11 +137,11 @@ func (e *ExtendibleHashTable) splitOnKey(page *bufferpool.Page, key []byte) erro
// scratch page for left
p0 := e.bufferPool.ScratchPage()
p0.WritePageNumber(int32(page.ID()))
p0.WritePageNumber(page.ID().Page)
p0.WritePageType(bufferpool.PAGE_TYPE_HASH_TABLE)
// allocate new page for split
p1, err := e.bufferPool.NewPage()
p1, err := e.bufferPool.NewPage(0, 0)
if err != nil {
return err
}
@ -160,19 +162,21 @@ func (e *ExtendibleHashTable) splitOnKey(page *bufferpool.Page, key []byte) erro
if slot == nil {
break
}
keyBytes := slot.KeyBytes(page)
pl := slot.KeyPayload(page)
lpl := slot.LeafPayload(page)
keyBytes := pl.KeyBytes(page)
k := string(keyBytes)
h := Key(k).Hash()
if h&hiBit > 0 {
sc := p1.ReadSlotCount()
p1.WriteKeyValueInSlot(sc, keyBytes, slot.ValueBytes(page))
p1.PutKeyValueInPageSlot(sc, keyBytes, lpl.ValueAsBytes(page))
// update the slot count
p1.WriteSlotCount(int16(sc + 1))
} else {
sc := p0.ReadSlotCount()
p0.WriteKeyValueInSlot(sc, keyBytes, slot.ValueBytes(page))
p0.PutKeyValueInPageSlot(sc, keyBytes, lpl.ValueAsBytes(page))
// update the slot count
p0.WriteSlotCount(int16(sc + 1))
}
@ -186,7 +190,7 @@ func (e *ExtendibleHashTable) splitOnKey(page *bufferpool.Page, key []byte) erro
}
// copy p1 back into page
p0.WritePage(page)
p0.CopyPageTo(page)
return nil
}
@ -194,7 +198,7 @@ func (e *ExtendibleHashTable) splitOnKey(page *bufferpool.Page, key []byte) erro
func (e *ExtendibleHashTable) cleanPage(page *bufferpool.Page) error {
scratch := e.bufferPool.ScratchPage()
// copy page number
scratch.WritePageNumber(int32(page.ID()))
scratch.WritePageNumber(page.ID().Page)
// set the page type
scratch.WritePageType(bufferpool.PAGE_TYPE_HASH_TABLE)
// copy local depth
@ -207,14 +211,16 @@ func (e *ExtendibleHashTable) cleanPage(page *bufferpool.Page) error {
if slot == nil {
break
}
scratch.WriteKeyValueInSlot(si.Cursor(), slot.KeyBytes(page), slot.ValueBytes(page))
pl := slot.KeyPayload(page)
lpl := slot.LeafPayload(page)
scratch.PutKeyValueInPageSlot(si.Cursor(), pl.KeyBytes(page), lpl.ValueAsBytes(page))
}
// update the slot count
scratch.WriteSlotCount(page.ReadSlotCount())
// write scratch back to page
scratch.WritePage(page)
scratch.CopyPageTo(page)
return nil
}
@ -222,7 +228,7 @@ func (e *ExtendibleHashTable) keyValueWillFit(page *bufferpool.Page, key, value
// will this k/v fit on the page?
slotLen := 4 // we need 2 len words for the slot
chunkLen := 6 + len(key) + len(value) // int16 len + int32 len + len of respective []byte
fs := page.FreeSpace()
fs := page.FreeSpaceOnPage()
return fs > (int16(slotLen) + int16(chunkLen))
}
@ -242,7 +248,7 @@ func (e *ExtendibleHashTable) putKeyValue(page *bufferpool.Page, key, value []by
slotCount := int(page.ReadSlotCount())
if found {
// we found the key, so we will update the value
err := page.WriteKeyValueInSlot(int16(newIndex), []byte(key), []byte(value))
err := page.PutKeyValueInPageSlot(int16(newIndex), []byte(key), []byte(value))
if err != nil {
return err
}
@ -252,10 +258,10 @@ func (e *ExtendibleHashTable) putKeyValue(page *bufferpool.Page, key, value []by
// TODO(pok) we should move all the slots in one fell swoop, because,... performance
// move all the slots after where we are going to insert
for j := slotCount; j > newIndex; j-- {
sl := page.ReadSlot(int16(j - 1))
page.WriteSlot(int16(j), sl)
sl := page.ReadPageSlot(int16(j - 1))
page.WritePageSlot(int16(j), sl)
}
err := page.WriteKeyValueInSlot(int16(newIndex), []byte(key), []byte(value))
err := page.PutKeyValueInPageSlot(int16(newIndex), []byte(key), []byte(value))
if err != nil {
return err
}

96
extendiblehash/extendiblehash_test.go
View file

@ -8,18 +8,19 @@ import (
"github.com/stretchr/testify/assert"
)
func makeDirectory() (*ExtendibleHashTable, error) {
func makeDirectory() (*ExtendibleHashTable, bufferpool.DiskManager, error) {
diskManager := bufferpool.NewInMemDiskSpillingDiskManager(128)
bufferPool := bufferpool.NewBufferPool(128, diskManager)
keySize := 12
valueSize := 20
return NewExtendibleHashTable(keySize, valueSize, bufferPool)
eht, err := NewExtendibleHashTable(keySize, valueSize, bufferPool)
return eht, diskManager, err
}
func TestHashTable_ExtendibleHash(t *testing.T) {
d, err := makeDirectory()
d, _, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
@ -36,7 +37,7 @@ func TestHashTable_ExtendibleHash(t *testing.T) {
}
func TestHashTable_GetPage(t *testing.T) {
d, err := makeDirectory()
d, _, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
@ -44,18 +45,18 @@ func TestHashTable_GetPage(t *testing.T) {
d.directory = make([]bufferpool.PageID, 16)
key := "478"
d.directory[14] = 2
d.directory[14] = bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 2}
pageID, err := d.getPageID([]byte(key))
if err != nil {
t.Fatal(err)
}
assert.Equal(t, 2, int(pageID))
assert.Equal(t, 2, int(pageID.Page))
}
func TestHashTable_GetPage_ShouldReturnError_WhenOffsetIsNotLimitedToDataSize(t *testing.T) {
d, err := makeDirectory()
d, _, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
@ -67,7 +68,7 @@ func TestHashTable_GetPage_ShouldReturnError_WhenOffsetIsNotLimitedToDataSize(t
}
func TestHashTable_GetPage_ShouldReturnError_WhenPageIDIsOutOfTheTable(t *testing.T) {
d, err := makeDirectory()
d, _, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
@ -79,24 +80,24 @@ func TestHashTable_GetPage_ShouldReturnError_WhenPageIDIsOutOfTheTable(t *testin
}
func TestHashTable_Get(t *testing.T) {
d, err := makeDirectory()
d, _, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
d.globalDepth = 4
d.directory = make([]bufferpool.PageID, 16)
d.directory[14] = 2
d.directory[14] = bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 2}
// force there to be two pages
page, err := d.bufferPool.NewPage() //1
page, err := d.bufferPool.NewPage(0, 0) //1
if err != nil {
t.Fatal(err)
}
page.WritePageType(bufferpool.PAGE_TYPE_HASH_TABLE)
d.bufferPool.FlushPage(page.ID())
page, err = d.bufferPool.NewPage() //2
page, err = d.bufferPool.NewPage(0, 0) //2
if err != nil {
t.Fatal(err)
}
@ -104,7 +105,7 @@ func TestHashTable_Get(t *testing.T) {
d.bufferPool.FlushPage(page.ID())
// now do the test
page, err = d.bufferPool.FetchPage(2)
page, err = d.bufferPool.FetchPage(bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 2})
if err != nil {
t.Fatal(err)
}
@ -113,7 +114,7 @@ func TestHashTable_Get(t *testing.T) {
key := "478"
value := "Hi"
page.WriteKeyValueInSlot(0, []byte(key), []byte(value))
page.PutKeyValueInPageSlot(0, []byte(key), []byte(value))
page.WriteSlotCount(int16(1))
result, _, err := d.Get([]byte(key))
@ -125,7 +126,7 @@ func TestHashTable_Get(t *testing.T) {
}
func TestHashTable_Get_ShouldHandleError(t *testing.T) {
d, err := makeDirectory()
d, _, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
@ -139,12 +140,12 @@ func TestHashTable_Get_ShouldHandleError(t *testing.T) {
}
func TestHashTable_Put(t *testing.T) {
d, err := makeDirectory()
d, _, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
page, err := d.bufferPool.FetchPage(0)
page, err := d.bufferPool.FetchPage(bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 0})
if err != nil {
t.Fatal(err)
}
@ -165,12 +166,12 @@ func TestHashTable_Put(t *testing.T) {
}
func TestHashTable_Put_ShouldIncreaseSize_WhenTableIsFull(t *testing.T) {
d, err := makeDirectory()
d, _, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
page, err := d.bufferPool.FetchPage(0)
page, err := d.bufferPool.FetchPage(bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 0})
if err != nil {
t.Fatal(err)
}
@ -192,12 +193,12 @@ func TestHashTable_Put_ShouldIncreaseSize_WhenTableIsFull(t *testing.T) {
}
func TestHashTable_PutShouldIncrementLD_WhenPageIsFull(t *testing.T) {
d, err := makeDirectory()
d, ds, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
page, err := d.bufferPool.FetchPage(0)
page, err := d.bufferPool.FetchPage(bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 0})
if err != nil {
t.Fatal(err)
}
@ -209,17 +210,17 @@ func TestHashTable_PutShouldIncrementLD_WhenPageIsFull(t *testing.T) {
d.Put([]byte("12345678"), []byte("Yolo !"))
assert.Equal(t, int64(8192*2), d.bufferPool.OnDiskSize())
assert.Equal(t, int64(8192*2), ds.FileSize(0, 0))
assert.Equal(t, 1, int(d.globalDepth))
p0, err := d.bufferPool.FetchPage(0)
p0, err := d.bufferPool.FetchPage(bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 0})
if err != nil {
t.Fatal(err)
}
defer d.bufferPool.UnpinPage(p0.ID())
assert.Equal(t, int16(1), p0.ReadLocalDepth())
p1, err := d.bufferPool.FetchPage(1)
p1, err := d.bufferPool.FetchPage(bufferpool.PageID{ObjectID: 0, Shard: 0, Page: 1})
if err != nil {
t.Fatal(err)
}
@ -228,7 +229,7 @@ func TestHashTable_PutShouldIncrementLD_WhenPageIsFull(t *testing.T) {
}
func TestHashTable_Put_INT(t *testing.T) {
d, err := makeDirectory()
d, _, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
@ -240,11 +241,44 @@ func TestHashTable_Put_INT(t *testing.T) {
}
}
assert.Equal(t, []bufferpool.PageID{0, 1, 2, 3, 4, 7, 6, 5, 13, 14, 12, 9, 8, 15, 10, 11, 28, 24, 21, 18, 4, 19, 29, 20, 27, 22, 25, 23, 17, 15, 16, 26}, d.directory)
assert.Equal(t, []bufferpool.PageID{
{ObjectID: 0, Shard: 0, Page: 0},
{ObjectID: 0, Shard: 0, Page: 1},
{ObjectID: 0, Shard: 0, Page: 2},
{ObjectID: 0, Shard: 0, Page: 3},
{ObjectID: 0, Shard: 0, Page: 4},
{ObjectID: 0, Shard: 0, Page: 7},
{ObjectID: 0, Shard: 0, Page: 6},
{ObjectID: 0, Shard: 0, Page: 5},
{ObjectID: 0, Shard: 0, Page: 13},
{ObjectID: 0, Shard: 0, Page: 15},
{ObjectID: 0, Shard: 0, Page: 11},
{ObjectID: 0, Shard: 0, Page: 9},
{ObjectID: 0, Shard: 0, Page: 8},
{ObjectID: 0, Shard: 0, Page: 14},
{ObjectID: 0, Shard: 0, Page: 10},
{ObjectID: 0, Shard: 0, Page: 12},
{ObjectID: 0, Shard: 0, Page: 28},
{ObjectID: 0, Shard: 0, Page: 25},
{ObjectID: 0, Shard: 0, Page: 21},
{ObjectID: 0, Shard: 0, Page: 18},
{ObjectID: 0, Shard: 0, Page: 4},
{ObjectID: 0, Shard: 0, Page: 20},
{ObjectID: 0, Shard: 0, Page: 29},
{ObjectID: 0, Shard: 0, Page: 19},
{ObjectID: 0, Shard: 0, Page: 27},
{ObjectID: 0, Shard: 0, Page: 24},
{ObjectID: 0, Shard: 0, Page: 23},
{ObjectID: 0, Shard: 0, Page: 22},
{ObjectID: 0, Shard: 0, Page: 17},
{ObjectID: 0, Shard: 0, Page: 14},
{ObjectID: 0, Shard: 0, Page: 16},
{ObjectID: 0, Shard: 0, Page: 26},
}, d.directory)
}
func TestHashTable_Put_SameKey_ALotOfTime(t *testing.T) {
d, err := makeDirectory()
d, ds, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
@ -260,11 +294,11 @@ func TestHashTable_Put_SameKey_ALotOfTime(t *testing.T) {
assert.Equal(t, "Yolo ! 9999", string(value))
assert.Equal(t, 1, len(d.directory))
assert.Equal(t, int64(8192), d.bufferPool.OnDiskSize())
assert.Equal(t, int64(8192), ds.FileSize(0, 0))
}
func TestHashTable_Put_Many_Keys(t *testing.T) {
d, err := makeDirectory()
d, _, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
@ -289,7 +323,7 @@ func TestHashTable_Put_Many_Keys(t *testing.T) {
func BenchmarkHashTable_Put_Many_Keys(b *testing.B) {
for i := 0; i < b.N; i++ {
d, err := makeDirectory()
d, _, err := makeDirectory()
if err != nil {
b.Fatal(err)
}
@ -317,7 +351,7 @@ func addToPage(page *bufferpool.Page, numberOfRecords int) error {
for i := 0; i < numberOfRecords; i++ {
//fmt.Printf("writing record %d\n", i+1)
itoa := strconv.Itoa(i)
err := page.WriteKeyValueInSlot(int16(i), []byte("key"+itoa), []byte("value foo bar"))
err := page.PutKeyValueInPageSlot(int16(i), []byte("key"+itoa), []byte("value foo bar"))
if err != nil {
return err
}

113
field.go
View file

@ -33,6 +33,8 @@ const (
// Default ranked field cache
DefaultCacheSize = 50000
DefaultVarcharLength = 50
bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64
maxInt = 1<<(bitsPerWord-1) - 1 // either 1<<31 - 1 or 1<<63 - 1
@ -47,6 +49,7 @@ const (
FieldTypeBool = "bool"
FieldTypeDecimal = "decimal"
FieldTypeTimestamp = "timestamp"
FieldTypeVarchar = "varchar"
)
type protected struct {
@ -141,6 +144,19 @@ func OptFieldForeignIndex(index string) FieldOption {
}
}
// OptFieldTypeNonRBFDefault is a functional option on FieldOptions
// used to set the field type and cache setting to the default values.
func OptFieldTypeNonRBFDefault() FieldOption {
return func(fo *FieldOptions) error {
if fo.Type != "" {
return errors.Errorf("field type is already set to: %s", fo.Type)
}
fo.Type = FieldTypeVarchar
fo.Length = DefaultVarcharLength
return nil
}
}
// OptFieldTypeDefault is a functional option on FieldOptions
// used to set the field type and cache setting to the default values.
func OptFieldTypeDefault() FieldOption {
@ -389,6 +405,20 @@ func OptFieldTrackExistence() FieldOption {
}
}
// OptFieldTypeVarchar is a functional option on FieldOptions
// used to specify the field as being type `varchar` and to
// provide any respective configuration values.
func OptFieldTypeVarchar(length int64) FieldOption {
return func(fo *FieldOptions) error {
if fo.Type != "" {
return errors.Errorf("field type is already set to: %s", fo.Type)
}
fo.Type = FieldTypeVarchar
fo.Length = length
return nil
}
}
// newField returns a new instance of field (without name validation).
func newField(holder *Holder, path, index, name string, opts ...FieldOption) (*Field, error) {
// Apply functional option.
@ -582,39 +612,50 @@ func (f *Field) Open() error {
return errors.Wrap(err, "creating field dir")
}
f.holder.Logger.Debugf("load available shards for index/field: %s/%s", f.index, f.name)
if strings.EqualFold(f.options.Type, FieldTypeVarchar) {
f.holder.Logger.Debugf("opening b-tree for index/field: %s/%s", f.index, f.name)
if err := f.loadAvailableShards(); err != nil {
return errors.Wrap(err, "loading available shards")
}
// Apply the field options loaded from etcd (or set via setOptions()).
f.holder.Logger.Debugf("apply options for index/field: %s/%s", f.index, f.name)
if err := f.applyOptions(f.options); err != nil {
return errors.Wrap(err, "applying options")
}
f.holder.Logger.Debugf("open views for index/field: %s/%s", f.index, f.name)
if err := f.openViews(); err != nil {
return errors.Wrap(err, "opening views")
}
// Apply the field-specific translateStore.
if err := f.applyTranslateStore(); err != nil {
return errors.Wrap(err, "applying translate store")
}
// If the field has a foreign index, make sure the index
// exists.
if f.options.ForeignIndex != "" {
if err := f.holder.checkForeignIndex(f); err != nil {
return errors.Wrap(err, "checking foreign index")
// Apply the field options loaded from etcd (or set via setOptions()).
f.holder.Logger.Debugf("apply options for index/field: %s/%s", f.index, f.name)
if err := f.applyOptions(f.options); err != nil {
return errors.Wrap(err, "applying options")
}
}
f.availableShardChan = make(chan struct{}, 1)
f.wg.Add(1)
go f.writeAvailableShards()
} else {
f.holder.Logger.Debugf("load available shards for index/field: %s/%s", f.index, f.name)
if err := f.loadAvailableShards(); err != nil {
return errors.Wrap(err, "loading available shards")
}
// Apply the field options loaded from etcd (or set via setOptions()).
f.holder.Logger.Debugf("apply options for index/field: %s/%s", f.index, f.name)
if err := f.applyOptions(f.options); err != nil {
return errors.Wrap(err, "applying options")
}
f.holder.Logger.Debugf("open views for index/field: %s/%s", f.index, f.name)
if err := f.openViews(); err != nil {
return errors.Wrap(err, "opening views")
}
// Apply the field-specific translateStore.
if err := f.applyTranslateStore(); err != nil {
return errors.Wrap(err, "applying translate store")
}
// If the field has a foreign index, make sure the index
// exists.
if f.options.ForeignIndex != "" {
if err := f.holder.checkForeignIndex(f); err != nil {
return errors.Wrap(err, "checking foreign index")
}
}
f.availableShardChan = make(chan struct{}, 1)
f.wg.Add(1)
go f.writeAvailableShards()
}
return nil
}(); err != nil {
f.unprotectedClose()
@ -883,6 +924,9 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.TTL = 0
f.options.Keys = false
f.options.ForeignIndex = ""
case FieldTypeVarchar:
f.options.Type = FieldTypeVarchar
f.options.Length = opt.Length
default:
return errors.New("invalid field type")
}
@ -2195,6 +2239,7 @@ type FieldOptions struct {
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
ForeignIndex string `json:"foreignIndex"`
TTL time.Duration `json:"ttl,omitempty"`
Length int64 `json:"length,omitempty"`
}
// newFieldOptions returns a new instance of FieldOptions
@ -2360,6 +2405,14 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
}{
o.Type,
})
case FieldTypeVarchar:
return json.Marshal(struct {
Type string `json:"type"`
Length int64 `json:"length"`
}{
o.Type,
o.Length,
})
}
return nil, errors.Errorf("invalid field type: '%s'", o.Type)
}

2
handler.go
View file

@ -464,6 +464,8 @@ type ImportRoaringShardRequest struct {
Remote bool
Views []RoaringUpdate
Tuples []byte
// SuppressLog requests we not write to the write log. Typically
// that would be because this request is being replayed from a
// write log.

24
holder.go
View file

@ -10,9 +10,11 @@ import (
"path/filepath"
"runtime"
"sort"
"strings"
"sync"
"time"
"github.com/featurebasedb/featurebase/v3/bufferpool"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/featurebasedb/featurebase/v3/logger"
@ -43,6 +45,9 @@ const (
// DataframesDir is the directory where we store the dataframe files (currently Apache Arrow)
DataframesDir = "dataframes"
// TStoreDir is the directory where we store the t-store files
TStoreDir = "tstore"
)
func init() {
@ -143,6 +148,10 @@ type Holder struct {
// snapshotter/writelogger; then the Controller should only start directing
// queries to that computer once it has completed applying the snapshot.
directiveApplied bool
// t-store
tstorepool *bufferpool.BufferPool
tstoredisk *bufferpool.OnDiskDiskManager
}
// HolderOpts holds information about the holder which other things might want
@ -254,6 +263,8 @@ type HolderConfig struct {
Sharder disco.Sharder
CacheFlushInterval time.Duration
Logger logger.Logger
TStoreBufferPool *bufferpool.BufferPool
TStoreDiskManager *bufferpool.OnDiskDiskManager
StorageConfig *storage.Config
RBFConfig *rbfcfg.Config
@ -266,6 +277,7 @@ type HolderConfig struct {
// need to override these; that's usually handled by server options
// such as OptServerOpenTranslateStore.
func DefaultHolderConfig() *HolderConfig {
dm := bufferpool.NewOnDiskDiskManager()
return &HolderConfig{
PartitionN: disco.DefaultPartitionN,
OpenTranslateStore: OpenInMemTranslateStore,
@ -280,6 +292,8 @@ func DefaultHolderConfig() *HolderConfig {
Logger: logger.NopLogger,
StorageConfig: storage.NewDefaultConfig(),
RBFConfig: rbfcfg.NewDefaultConfig(),
TStoreDiskManager: dm,
TStoreBufferPool: bufferpool.NewBufferPool(1024, dm),
}
}
@ -328,6 +342,8 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
sharder: cfg.Sharder,
Schemator: cfg.Schemator,
Logger: cfg.Logger,
tstorepool: cfg.TStoreBufferPool,
tstoredisk: cfg.TStoreDiskManager,
Opts: HolderOpts{StorageBackend: cfg.StorageConfig.Backend},
Auditor: NewAuditor(),
@ -494,6 +510,7 @@ func (h *Holder) Open() error {
// from the API, which already comes from etcd. In that case, this logic
// could be removed, and the createdAt on the index struct could be
// removed.
index.ID = cim.IndexID
index.createdAt = cim.CreatedAt
index.owner = cim.Owner
index.description = cim.Meta.Description
@ -724,6 +741,7 @@ func (h *Holder) schema(ctx context.Context, includeViews bool) ([]*IndexInfo, e
}
di := &IndexInfo{
ID: cim.IndexID,
Name: cim.Index,
CreatedAt: cim.CreatedAt,
Owner: cim.Owner,
@ -992,6 +1010,7 @@ func (h *Holder) createIndex(cim *CreateIndexMessage, broadcast bool) (*Index, e
index.keys = cim.Meta.Keys
index.trackExistence = cim.Meta.TrackExistence
index.ID = cim.IndexID
index.createdAt = cim.CreatedAt
index.owner = cim.Owner
index.description = cim.Meta.Description
@ -1035,6 +1054,7 @@ func (h *Holder) createIndexWithPartitions(cim *CreateIndexMessage, translatePar
index.keys = cim.Meta.Keys
index.trackExistence = cim.Meta.TrackExistence
index.ID = cim.IndexID
index.createdAt = cim.CreatedAt
index.translatePartitions = translatePartitions
@ -1475,7 +1495,9 @@ func (s *holderSyncer) setTranslateReadOnlyFlags(snap *disco.ClusterSnapshot) {
}
for _, field := range index.Fields() {
field.TranslateStore().SetReadOnly(!isPrimaryFieldTranslator)
if !strings.EqualFold(field.options.Type, FieldTypeVarchar) {
field.TranslateStore().SetReadOnly(!isPrimaryFieldTranslator)
}
}
}
s.Cluster.mu.RUnlock()

7
http_handler.go
View file

@ -2072,6 +2072,8 @@ func fieldOptionsToFunctionalOpts(opt fieldOptions) []FieldOption {
fos = append(fos, OptFieldTypeMutex(*opt.CacheType, *opt.CacheSize))
case FieldTypeBool:
fos = append(fos, OptFieldTypeBool())
case FieldTypeVarchar:
fos = append(fos, OptFieldTypeVarchar(*opt.Length))
}
if opt.Keys != nil {
if *opt.Keys {
@ -2193,6 +2195,7 @@ type fieldOptions struct {
ForeignIndex *string `json:"foreignIndex,omitempty"`
TTL *string `json:"ttl,omitempty"`
Base *int64 `json:"base,omitempty"`
Length *int64 `json:"length,omitempty"`
}
func (o *fieldOptions) validate() error {
@ -2309,6 +2312,10 @@ func (o *fieldOptions) validate() error {
} else if o.ForeignIndex != nil {
return NewBadRequestError(errors.New("bool field cannot be a foreign key"))
}
case FieldTypeVarchar:
if o.Length == nil {
return NewBadRequestError(errors.New("varchar field requires a length argument"))
}
default:
return errors.Errorf("invalid field type: %s", o.Type)
}

70
index.go
View file

@ -10,6 +10,7 @@ import (
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
@ -26,6 +27,7 @@ import (
// Index represents a container for fields.
type Index struct {
mu sync.RWMutex
ID int32
createdAt int64
owner string
description string
@ -112,6 +114,11 @@ func (i *Index) DataframesPath() string {
return filepath.Join(i.path, DataframesDir)
}
// TStorePath returns the path of the t-store files specific to an index
func (i *Index) TStorePath() string {
return filepath.Join(i.path, TStoreDir)
}
// Name returns name of the index.
func (i *Index) Name() string { return i.name }
@ -180,6 +187,7 @@ func (i *Index) OpenWithSchema(idx *disco.Index) error {
return errors.Wrap(err, "decoding create index message")
}
i.createdAt = cim.CreatedAt
i.ID = cim.IndexID
i.trackExistence = cim.Meta.TrackExistence
i.keys = cim.Meta.Keys
@ -206,6 +214,12 @@ func (i *Index) open(idx *disco.Index) (err error) {
return errors.Wrap(err, "creating dataframes directory")
}
// Ensure the t-store path exists
i.holder.Logger.Debugf("ensure t-store path exists: %s", i.TStorePath())
if err := os.MkdirAll(i.TStorePath(), 0o750); err != nil {
return errors.Wrap(err, "creating tstore directory")
}
i.closing = make(chan struct{})
// fmt.Printf("new channel %p for index %p\n", i.closing, i)
@ -240,6 +254,12 @@ func (i *Index) open(idx *disco.Index) (err error) {
return errors.Wrap(err, "setting field bitDepths")
}
// if the table has store fields then open the b-tree files
if i.hasTStoreFields() {
// TODO (pok) kick off recovery here
i.holder.Logger.Debugf("open t-store index: %s", i.name)
}
if i.trackExistence {
if err := i.openExistenceField(); err != nil {
return errors.Wrap(err, "opening existence field")
@ -353,13 +373,18 @@ fileLoop:
// openField opens the field directory, initializes the field, and adds it to
// the in-memory map of fields maintained by Index.
func (i *Index) openField(mu *sync.Mutex, cfm *CreateFieldMessage, file string) (*Field, error) {
var err error
var fld *Field
mu.Lock()
fld, err := i.newField(i.fieldPath(filepath.Base(file)), filepath.Base(file))
if strings.EqualFold(cfm.Meta.Type, FieldTypeVarchar) {
fld, err = i.newNonRBFField(i.fieldPath(filepath.Base(file)), filepath.Base(file))
} else {
fld, err = i.newField(i.fieldPath(filepath.Base(file)), filepath.Base(file))
}
mu.Unlock()
if err != nil {
return nil, errors.Wrapf(ErrName, "'%s'", file)
}
// Pass holder through to the field for use in looking
// up a foreign index.
fld.holder = i.holder
@ -432,6 +457,16 @@ func (i *Index) setFieldBitDepths() error {
return nil
}
func (i *Index) hasTStoreFields() bool {
for _, f := range i.fields {
switch f.Type() {
case FieldTypeVarchar:
return true
}
}
return false
}
// Close closes the index and its fields.
func (i *Index) Close() error {
i.mu.Lock()
@ -933,11 +968,22 @@ func (i *Index) createField(cfm *CreateFieldMessage) (*Field, error) {
return nil, ErrInvalidCacheType
}
// Initialize field.
f, err := i.newField(i.fieldPath(cfm.Field), cfm.Field)
if err != nil {
return nil, errors.Wrap(err, "initializing")
var err error
var f *Field
// initialize non-rbf field
if strings.EqualFold(cfm.Meta.Type, FieldTypeVarchar) {
f, err = i.newNonRBFField(i.fieldPath(cfm.Field), cfm.Field)
if err != nil {
return nil, errors.Wrap(err, "initializing")
}
} else {
// initialize rbf field
f, err = i.newField(i.fieldPath(cfm.Field), cfm.Field)
if err != nil {
return nil, errors.Wrap(err, "initializing")
}
}
f.createdAt = cfm.CreatedAt
f.owner = cfm.Owner
@ -978,6 +1024,17 @@ func (i *Index) newField(path, name string) (*Field, error) {
return f, nil
}
func (i *Index) newNonRBFField(path, name string) (*Field, error) {
f, err := newField(i.holder, path, i.name, name, OptFieldTypeNonRBFDefault())
if err != nil {
return nil, err
}
f.idx = i
f.broadcaster = i.broadcaster
f.serializer = i.serializer
return f, nil
}
// DeleteField removes a field from the index.
func (i *Index) DeleteField(name string) error {
i.mu.Lock()
@ -1047,6 +1104,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// IndexInfo represents schema information for an index.
type IndexInfo struct {
ID int32 `json:"id"`
Name string `json:"name"`
CreatedAt int64 `json:"createdAt,omitempty"`
UpdatedAt int64 `json:"updatedAt"`

331
pb/private.pb.go
View file

@ -103,6 +103,7 @@ type FieldOptions struct {
TimeUnit string `protobuf:"bytes,19,opt,name=TimeUnit,proto3" json:"TimeUnit,omitempty"`
TTL string `protobuf:"bytes,20,opt,name=TTL,proto3" json:"TTL,omitempty"`
TrackExistence bool `protobuf:"varint,21,opt,name=TrackExistence,proto3" json:"TrackExistence,omitempty"`
Length int64 `protobuf:"varint,22,opt,name=Length,proto3" json:"Length,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -260,6 +261,13 @@ func (m *FieldOptions) GetTrackExistence() bool {
return false
}
func (m *FieldOptions) GetLength() int64 {
if m != nil {
return m.Length
}
return 0
}
type ImportResponse struct {
Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@ -650,6 +658,7 @@ type CreateIndexMessage struct {
Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"`
CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
Owner string `protobuf:"bytes,5,opt,name=Owner,proto3" json:"Owner,omitempty"`
IndexID int32 `protobuf:"varint,6,opt,name=IndexID,proto3" json:"IndexID,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -716,6 +725,13 @@ func (m *CreateIndexMessage) GetOwner() string {
return ""
}
func (m *CreateIndexMessage) GetIndexID() int32 {
if m != nil {
return m.IndexID
}
return 0
}
type CreateFieldMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
@ -1146,6 +1162,7 @@ type Index struct {
CreatedAt int64 `protobuf:"varint,2,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
Options *IndexMeta `protobuf:"bytes,5,opt,name=Options,proto3" json:"Options,omitempty"`
Fields []*Field `protobuf:"bytes,4,rep,name=Fields,proto3" json:"Fields,omitempty"`
IndexID int32 `protobuf:"varint,6,opt,name=IndexID,proto3" json:"IndexID,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1212,6 +1229,13 @@ func (m *Index) GetFields() []*Field {
return nil
}
func (m *Index) GetIndexID() int32 {
if m != nil {
return m.IndexID
}
return 0
}
type URI struct {
Scheme string `protobuf:"bytes,1,opt,name=Scheme,proto3" json:"Scheme,omitempty"`
Host string `protobuf:"bytes,2,opt,name=Host,proto3" json:"Host,omitempty"`
@ -2959,117 +2983,119 @@ func init() {
func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) }
var fileDescriptor_d2a91b51c7bdc125 = []byte{
// 1750 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcd, 0x6e, 0x24, 0x49,
0x11, 0xa6, 0x7e, 0xdc, 0x3f, 0xd1, 0x6e, 0x4f, 0x3b, 0xd7, 0x6b, 0x6a, 0xbc, 0x83, 0xd5, 0x93,
0xa0, 0x99, 0x66, 0x24, 0x8c, 0xf0, 0x1e, 0x16, 0xb1, 0x97, 0x1d, 0x77, 0x7b, 0x96, 0x66, 0x77,
0x7e, 0x36, 0xed, 0x99, 0x23, 0x28, 0x5d, 0x9d, 0xd8, 0xa5, 0xa9, 0xae, 0x6a, 0xaa, 0xaa, 0x3d,
0xdd, 0x7b, 0x40, 0x02, 0x09, 0xc1, 0x85, 0x3b, 0xe2, 0xc0, 0x5b, 0xf0, 0x0e, 0x5c, 0x90, 0x78,
0x04, 0x34, 0xdc, 0x78, 0x0a, 0x94, 0x91, 0x99, 0x55, 0xd9, 0xed, 0xb2, 0xbd, 0x58, 0x7b, 0xab,
0xf8, 0x22, 0x2b, 0xf2, 0x8b, 0x9f, 0x8c, 0x8c, 0x2a, 0xe8, 0xce, 0xb2, 0xe8, 0x92, 0x17, 0xe2,
0x60, 0x96, 0xa5, 0x45, 0x4a, 0xdc, 0xd9, 0xd9, 0xde, 0xe6, 0x6c, 0x7e, 0x16, 0x47, 0xa1, 0x42,
0x68, 0x04, 0xed, 0x71, 0x32, 0x11, 0x8b, 0xe7, 0xa2, 0xe0, 0x84, 0x80, 0xff, 0x85, 0x58, 0xe6,
0x81, 0xd7, 0x77, 0x06, 0x2d, 0x86, 0xcf, 0xe4, 0x11, 0x6c, 0x9d, 0x66, 0x3c, 0x7c, 0x7b, 0xbc,
0x88, 0xf2, 0x42, 0x24, 0xa1, 0x08, 0x7c, 0xd4, 0xae, 0xa1, 0xa4, 0x0f, 0x9d, 0x91, 0xc8, 0xc3,
0x2c, 0x9a, 0x15, 0x51, 0x9a, 0x04, 0x1b, 0x7d, 0x67, 0xd0, 0x66, 0x36, 0x44, 0xff, 0xeb, 0xc1,
0xe6, 0xb3, 0x48, 0xc4, 0x93, 0x97, 0x28, 0xe7, 0x72, 0xbb, 0xd3, 0xe5, 0x4c, 0x04, 0x2d, 0x5c,
0x8b, 0xcf, 0xe4, 0x01, 0xb4, 0x87, 0x3c, 0xbc, 0x10, 0xa8, 0xf0, 0x50, 0x51, 0x01, 0xa5, 0xf6,
0x24, 0xfa, 0x5a, 0xf1, 0xe8, 0xb2, 0x0a, 0x90, 0x14, 0x4e, 0xa3, 0xa9, 0xf8, 0x6a, 0xce, 0x93,
0x62, 0x3e, 0x35, 0x14, 0x2c, 0x88, 0xec, 0x42, 0xe3, 0x65, 0x3c, 0x79, 0x1e, 0x25, 0x41, 0xbb,
0xef, 0x0c, 0x3c, 0xa6, 0x25, 0x83, 0xf3, 0x45, 0x00, 0x15, 0xce, 0x17, 0x65, 0x40, 0x3a, 0xab,
0x01, 0x79, 0x91, 0x9e, 0x14, 0x3c, 0x99, 0xf0, 0x6c, 0xf2, 0x26, 0x12, 0xef, 0x82, 0x4d, 0x15,
0x90, 0x55, 0x54, 0xbe, 0x7b, 0xc4, 0x73, 0x11, 0x74, 0xd1, 0x22, 0x3e, 0x93, 0x3d, 0x68, 0x1d,
0x45, 0xc5, 0x48, 0xcc, 0x8a, 0x8b, 0x60, 0xab, 0xef, 0x0c, 0x7c, 0x56, 0xca, 0x64, 0x07, 0x36,
0x4e, 0x42, 0x1e, 0x8b, 0xe0, 0x1e, 0xbe, 0xa0, 0x04, 0x42, 0x61, 0xf3, 0x59, 0x9a, 0x89, 0xe8,
0x3c, 0xc1, 0x34, 0x05, 0x3d, 0x74, 0x6a, 0x05, 0x23, 0xdf, 0x03, 0x4f, 0xba, 0xb4, 0xdd, 0x77,
0x06, 0x9d, 0xc3, 0xce, 0xc1, 0xec, 0xec, 0x60, 0x24, 0xc2, 0x68, 0xca, 0x63, 0x26, 0x71, 0x54,
0xf3, 0x45, 0x40, 0xea, 0xd4, 0x7c, 0x21, 0x39, 0xc9, 0x10, 0xbd, 0x4e, 0xa2, 0x22, 0xf8, 0x00,
0xad, 0x97, 0x32, 0xe9, 0x81, 0x77, 0x7a, 0xfa, 0x65, 0xb0, 0x83, 0xb0, 0x7c, 0xac, 0x29, 0x87,
0x0f, 0xeb, 0xca, 0x81, 0x52, 0xd8, 0x1a, 0x4f, 0x67, 0x69, 0x56, 0x30, 0x91, 0xcf, 0xd2, 0x24,
0x17, 0xd2, 0xd6, 0x71, 0x96, 0x05, 0x8e, 0xb2, 0x75, 0x9c, 0x65, 0xf4, 0xb7, 0xd0, 0x3b, 0x8a,
0xd3, 0xf0, 0xed, 0x88, 0x17, 0x9c, 0x89, 0xdf, 0xcc, 0x45, 0x5e, 0xc8, 0x28, 0x28, 0x47, 0xd5,
0x3a, 0x25, 0x48, 0x14, 0x2b, 0x27, 0x70, 0x15, 0x8a, 0x82, 0x8c, 0x30, 0xc6, 0x5f, 0x25, 0x1a,
0x9f, 0x31, 0x8a, 0x17, 0x3c, 0x9b, 0x60, 0x75, 0xf8, 0x4c, 0x09, 0x12, 0xc5, 0x9d, 0xb0, 0xa2,
0x7c, 0xa6, 0x04, 0x3a, 0x86, 0x6d, 0x6b, 0x7f, 0x4d, 0x73, 0x17, 0x1a, 0x2c, 0x7d, 0x37, 0x1e,
0xe5, 0x81, 0xd3, 0xf7, 0x06, 0x3e, 0xd3, 0x12, 0x96, 0x5e, 0x1a, 0xcf, 0xa7, 0x89, 0x54, 0xb9,
0xa8, 0xaa, 0x00, 0x7a, 0x1f, 0x36, 0xb0, 0x0e, 0xa5, 0x97, 0xd5, 0xbb, 0xf2, 0x91, 0xfe, 0xce,
0x81, 0xf6, 0x73, 0xbe, 0x40, 0x22, 0x39, 0xf9, 0x04, 0x5a, 0xa6, 0x4a, 0x70, 0x51, 0xe7, 0xf0,
0x23, 0x99, 0x91, 0x72, 0xc1, 0x81, 0xd1, 0x1e, 0x27, 0x45, 0xb6, 0x64, 0xe5, 0xe2, 0xbd, 0x4f,
0xa1, 0xbb, 0xa2, 0x92, 0x3b, 0xbd, 0x15, 0x4b, 0x13, 0xcf, 0xb7, 0x62, 0x29, 0xbd, 0xbc, 0xe4,
0xf1, 0x5c, 0x60, 0x94, 0x7c, 0xa6, 0x84, 0x9f, 0xb9, 0x3f, 0x75, 0xe8, 0x1b, 0x20, 0xc3, 0x4c,
0xf0, 0x42, 0xe0, 0x26, 0xcf, 0x45, 0x9e, 0xf3, 0x73, 0x71, 0x5b, 0xac, 0x3d, 0x3b, 0xd6, 0x65,
0x5c, 0x5d, 0x2b, 0xae, 0xf4, 0x09, 0x90, 0x91, 0x88, 0x45, 0x21, 0x74, 0x0f, 0xb9, 0xc1, 0xae,
0x8c, 0x83, 0x26, 0x71, 0xfb, 0x62, 0xf2, 0x10, 0x7c, 0xd9, 0x91, 0x70, 0xb7, 0xce, 0x61, 0x57,
0x86, 0xa8, 0x6c, 0x53, 0x0c, 0x55, 0x98, 0x10, 0x34, 0x37, 0x79, 0x5a, 0x20, 0x57, 0x8f, 0x55,
0x80, 0x34, 0xfb, 0xf2, 0x5d, 0x22, 0x32, 0x5d, 0x1c, 0x4a, 0xa0, 0x7f, 0x2d, 0x39, 0xa0, 0x57,
0xdf, 0x30, 0x10, 0x2b, 0x45, 0xf7, 0x03, 0xcd, 0xcc, 0x43, 0x66, 0x3d, 0xc9, 0xcc, 0x6e, 0x6a,
0x75, 0xe4, 0xfc, 0x6f, 0x46, 0xee, 0x0f, 0x0e, 0x90, 0xd7, 0xb3, 0xc9, 0x3a, 0xb9, 0x67, 0x75,
0x94, 0x91, 0x69, 0xe7, 0x70, 0x57, 0x6e, 0x7f, 0x55, 0xcb, 0xea, 0x9c, 0x7c, 0x0c, 0x0d, 0x65,
0x5d, 0x07, 0xf5, 0x5e, 0x49, 0x5d, 0xc1, 0x4c, 0xab, 0xe9, 0xa7, 0xd0, 0xb1, 0x60, 0xec, 0x8d,
0xaa, 0xa7, 0xab, 0xe8, 0x68, 0x49, 0x3a, 0xf1, 0xa6, 0xac, 0xb6, 0x36, 0x53, 0x02, 0xfd, 0xcc,
0x54, 0xc4, 0x5d, 0x03, 0x4c, 0x43, 0xf8, 0x48, 0x59, 0x78, 0x7a, 0xc9, 0xa3, 0x98, 0x9f, 0xc5,
0xff, 0x57, 0xd1, 0xae, 0xe4, 0x2a, 0x80, 0x26, 0xbe, 0x3b, 0x1e, 0xe9, 0x83, 0x6f, 0x44, 0x3a,
0x87, 0xaa, 0x87, 0xbc, 0xe0, 0x53, 0xa1, 0xad, 0xe1, 0x73, 0x99, 0x62, 0xf7, 0xc6, 0x14, 0x4b,
0xff, 0x23, 0xf1, 0x4e, 0xde, 0x96, 0x1e, 0xfa, 0x2f, 0x85, 0x9b, 0x13, 0x4f, 0x7f, 0x04, 0x8d,
0x93, 0xf0, 0x42, 0x4c, 0x39, 0xf9, 0x3e, 0x34, 0x91, 0xb9, 0xc8, 0x75, 0x1b, 0x68, 0x97, 0x35,
0xce, 0x8c, 0x46, 0x56, 0x84, 0xf6, 0xaf, 0x8e, 0xe6, 0xca, 0x56, 0xee, 0x7a, 0x8d, 0x3d, 0x86,
0xa6, 0xe6, 0x8b, 0x55, 0x76, 0xe5, 0x10, 0x19, 0x2d, 0x79, 0x08, 0x0d, 0xf4, 0x2e, 0x0f, 0xfc,
0x8a, 0x08, 0x22, 0x4c, 0x2b, 0xe8, 0x31, 0x78, 0xaf, 0xd9, 0x58, 0x56, 0x02, 0xb2, 0x37, 0x34,
0xb4, 0x24, 0xc9, 0xfd, 0x3c, 0xcd, 0x0b, 0x1d, 0x7b, 0x7c, 0x96, 0xd8, 0xab, 0x34, 0x53, 0x07,
0xb3, 0xcb, 0xf0, 0x99, 0xfe, 0xc9, 0x01, 0xff, 0x45, 0x3a, 0x11, 0x64, 0x0b, 0xdc, 0xf1, 0x48,
0x1b, 0x71, 0xc7, 0x23, 0x72, 0x1f, 0xed, 0xeb, 0x78, 0x37, 0xe5, 0xfe, 0xaf, 0xd9, 0x98, 0xe1,
0x9e, 0x0f, 0xa0, 0x3d, 0xce, 0x5f, 0x65, 0xd1, 0x94, 0x67, 0x4b, 0x3d, 0x97, 0x54, 0x00, 0x76,
0xa5, 0x42, 0x96, 0xb4, 0xaf, 0xd2, 0x8e, 0x02, 0x79, 0x08, 0xcd, 0xcf, 0xd9, 0xab, 0xa1, 0x34,
0xb9, 0xb1, 0x6a, 0xd2, 0xe0, 0xf4, 0x33, 0xe8, 0x49, 0x26, 0xb8, 0xde, 0x54, 0xd6, 0x2e, 0x34,
0x24, 0x56, 0x32, 0xd3, 0x52, 0xb5, 0x89, 0x6b, 0x6d, 0x42, 0x9f, 0x29, 0x0b, 0xc7, 0x97, 0x22,
0x29, 0xac, 0xda, 0x44, 0x19, 0x0d, 0x74, 0x99, 0x12, 0xc8, 0x03, 0xe5, 0xb5, 0x76, 0xaf, 0x25,
0xb9, 0x48, 0x99, 0x21, 0x4a, 0x97, 0x00, 0x86, 0xc9, 0x3c, 0x2f, 0xd7, 0x3a, 0x75, 0x6b, 0x09,
0x35, 0xe5, 0xa3, 0xbb, 0x0f, 0x48, 0xbd, 0x42, 0x98, 0x29, 0xac, 0x1f, 0x56, 0x85, 0xa5, 0xf2,
0x79, 0xaf, 0xcc, 0xbb, 0xda, 0xa3, 0x2a, 0xaf, 0x0b, 0xe8, 0x58, 0x78, 0x6d, 0x8d, 0x3d, 0x2e,
0x8b, 0xc3, 0xad, 0x8c, 0x21, 0xa2, 0x8d, 0x69, 0xf5, 0xcd, 0xdd, 0x98, 0x46, 0xba, 0xa5, 0xdc,
0xb0, 0xd3, 0x00, 0xee, 0xad, 0x1e, 0x78, 0x73, 0xcb, 0xae, 0xc3, 0xb7, 0x6c, 0xf5, 0x47, 0x07,
0xba, 0xc3, 0x78, 0x9e, 0x17, 0x22, 0x2b, 0x63, 0xda, 0xd6, 0x40, 0x99, 0xda, 0x0a, 0xa8, 0xcf,
0x2e, 0xd9, 0x87, 0x0d, 0x19, 0x71, 0x75, 0xb8, 0xed, 0x44, 0x28, 0xd8, 0xca, 0x84, 0x7f, 0x5d,
0x26, 0xe8, 0x1b, 0x68, 0x1d, 0x9d, 0x8c, 0x3f, 0xcf, 0xd2, 0xf9, 0xac, 0xd6, 0x63, 0x33, 0xfe,
0xba, 0xd6, 0xf8, 0xdb, 0x53, 0xa3, 0x9c, 0xf2, 0x0a, 0xa7, 0xb7, 0x9e, 0x9a, 0xde, 0x7c, 0x8d,
0xf0, 0x05, 0x3d, 0x81, 0x6d, 0xe5, 0xae, 0xec, 0x38, 0x77, 0x69, 0x8b, 0x66, 0x6e, 0xf2, 0xaa,
0xb9, 0x49, 0x1a, 0x55, 0x5d, 0xf7, 0xdb, 0x34, 0xfa, 0x4f, 0x17, 0xb6, 0x99, 0xc8, 0xa3, 0xaf,
0xc5, 0x38, 0xc9, 0x8b, 0x6c, 0x1e, 0x9a, 0x8b, 0xe3, 0x17, 0xe9, 0x99, 0xce, 0x85, 0xc7, 0x94,
0x70, 0xf3, 0x29, 0x21, 0x14, 0x9a, 0x76, 0x13, 0xb0, 0x17, 0x18, 0x05, 0x79, 0x02, 0xcd, 0x93,
0x74, 0x9e, 0x85, 0x65, 0xe5, 0x63, 0xe7, 0x56, 0xfb, 0x2b, 0x05, 0x33, 0x0b, 0xc8, 0x17, 0x40,
0x4e, 0x33, 0x9e, 0xe4, 0x31, 0x97, 0x94, 0xcc, 0x6b, 0xad, 0x6a, 0x20, 0xb3, 0xb4, 0x2b, 0x16,
0x6a, 0x5e, 0x23, 0x07, 0xf6, 0x11, 0x0e, 0x9a, 0xc8, 0x6f, 0xcb, 0xf0, 0xd3, 0xe7, 0xc4, 0x3e,
0xe4, 0x9f, 0xac, 0x55, 0x68, 0xd0, 0xc0, 0x57, 0xb6, 0xf1, 0x32, 0xb7, 0x15, 0x6c, 0x75, 0x1d,
0xfd, 0xbd, 0x03, 0x9b, 0x36, 0x9b, 0x5b, 0xda, 0x45, 0x99, 0x3e, 0xf7, 0xf6, 0xf9, 0xce, 0xa4,
0xcf, 0xaf, 0x9b, 0xa5, 0x37, 0xec, 0x99, 0x2f, 0x85, 0xef, 0x5e, 0x13, 0x9c, 0x3b, 0xd1, 0xe9,
0x43, 0xe7, 0x15, 0xcf, 0x8a, 0x48, 0x1a, 0xd3, 0xf7, 0xf4, 0x06, 0xb3, 0x21, 0x2a, 0xe0, 0xfe,
0x95, 0x22, 0x1a, 0xa6, 0xd3, 0x99, 0xac, 0xd6, 0x3b, 0x15, 0x93, 0x6c, 0xd3, 0x59, 0x96, 0x66,
0x26, 0x02, 0x28, 0xd0, 0x23, 0x68, 0x9d, 0xa6, 0xb3, 0x34, 0x4e, 0xcf, 0x97, 0xb7, 0xb4, 0x8c,
0x00, 0x9a, 0xea, 0x6a, 0x50, 0x2d, 0xaa, 0xcd, 0x8c, 0x48, 0x3f, 0x90, 0xf5, 0x1e, 0xf2, 0x38,
0x9c, 0xc7, 0xbc, 0x10, 0xf8, 0x45, 0x80, 0xe0, 0x97, 0x29, 0x9f, 0xa8, 0xae, 0xa0, 0x8f, 0x16,
0xfd, 0x95, 0x2e, 0x40, 0x8e, 0xee, 0x58, 0x57, 0xd0, 0xd3, 0xd0, 0x9e, 0xb5, 0x94, 0x44, 0x7e,
0x02, 0x1d, 0x6b, 0xb5, 0x3d, 0xc0, 0x59, 0x30, 0xb3, 0xd7, 0xd0, 0xbf, 0x3b, 0x2b, 0xef, 0x5c,
0xb9, 0x73, 0xf5, 0x56, 0x97, 0x2a, 0x48, 0x2d, 0xa6, 0x25, 0xe9, 0xfa, 0xf1, 0x22, 0x8c, 0xe7,
0xb9, 0x54, 0xe9, 0x0b, 0xb7, 0x04, 0xa4, 0xeb, 0xf2, 0xe3, 0x30, 0x9d, 0x9b, 0xe1, 0xc6, 0x88,
0xf2, 0x33, 0x72, 0x24, 0xf8, 0x24, 0x8e, 0x12, 0x81, 0xf5, 0xe2, 0xb1, 0x52, 0x26, 0x4f, 0x54,
0x8f, 0x35, 0x85, 0xbe, 0xb3, 0x46, 0x1c, 0x75, 0xaa, 0xf3, 0xe6, 0x94, 0x40, 0x6f, 0x5d, 0x45,
0x77, 0x80, 0xa8, 0x0a, 0x78, 0x7a, 0x96, 0x66, 0xe6, 0xb6, 0xa5, 0x43, 0xd3, 0x5c, 0x64, 0xf4,
0x6f, 0xbb, 0xc4, 0xab, 0xc8, 0xba, 0x76, 0x64, 0xe9, 0x2f, 0x61, 0x4b, 0xcf, 0x76, 0x22, 0xc3,
0x82, 0x96, 0x01, 0x60, 0x22, 0x4c, 0xe5, 0x98, 0x68, 0xbe, 0xe3, 0x2a, 0x40, 0xda, 0xc1, 0x41,
0xd7, 0xdc, 0x4e, 0x5a, 0xc2, 0xd9, 0x28, 0x3a, 0x4f, 0xc4, 0x04, 0x6f, 0x0c, 0x8f, 0x69, 0x89,
0xfe, 0xd9, 0x85, 0x1d, 0x35, 0x74, 0x26, 0xe7, 0x22, 0x2f, 0xaa, 0x6d, 0x70, 0xac, 0xc6, 0xfe,
0x5f, 0x8e, 0xd5, 0x78, 0x03, 0x3c, 0x82, 0xad, 0x61, 0x2c, 0x78, 0x56, 0x71, 0x50, 0x1b, 0xad,
0xa1, 0xf2, 0xdc, 0x20, 0xa2, 0xaf, 0x67, 0x35, 0x84, 0xda, 0x10, 0x39, 0x82, 0x96, 0x76, 0xcd,
0x34, 0xc4, 0x47, 0x78, 0x4b, 0xd5, 0xb0, 0x31, 0xf3, 0x6d, 0xae, 0xbf, 0x3a, 0x8d, 0xb8, 0xf7,
0x12, 0xba, 0x2b, 0xaa, 0x9a, 0xaf, 0xce, 0x81, 0xfd, 0xd5, 0xd9, 0x39, 0x24, 0xd6, 0xb8, 0xac,
0xad, 0xdb, 0x5f, 0xa2, 0x43, 0xf8, 0xb0, 0x8e, 0x40, 0x4e, 0x9e, 0x80, 0x27, 0x89, 0xaa, 0x61,
0x38, 0xb8, 0x8e, 0x28, 0x93, 0x8b, 0xe8, 0xdf, 0x1c, 0x1d, 0x54, 0xa1, 0xf5, 0xe6, 0xef, 0xc1,
0xc7, 0xb6, 0x91, 0x87, 0xa5, 0x91, 0xb5, 0x65, 0x07, 0xa5, 0xa3, 0x72, 0xf5, 0xde, 0x57, 0xd0,
0xaa, 0x73, 0xcf, 0x57, 0xee, 0xfd, 0x78, 0xd5, 0xbd, 0xfb, 0xd7, 0x31, 0xcb, 0x6d, 0x2f, 0x0f,
0x60, 0x57, 0xdd, 0xa6, 0x23, 0x5e, 0xf0, 0x5f, 0x67, 0x7c, 0x2a, 0x6e, 0xbc, 0x52, 0x8f, 0x7a,
0xff, 0x78, 0xbf, 0xef, 0xfc, 0xeb, 0xfd, 0xbe, 0xf3, 0xef, 0xf7, 0xfb, 0xce, 0x5f, 0xfe, 0xb3,
0xff, 0x9d, 0xb3, 0x06, 0xfe, 0x9e, 0xfb, 0xf8, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x0d,
0x5d, 0xdb, 0xc1, 0x13, 0x00, 0x00,
// 1781 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdd, 0x6e, 0x23, 0x49,
0x15, 0xa6, 0x7f, 0xe2, 0x9f, 0xe3, 0x38, 0xe3, 0xd4, 0x66, 0x43, 0x4f, 0x76, 0x88, 0x3c, 0x05,
0x9a, 0x31, 0x23, 0x11, 0x44, 0xf6, 0x62, 0x11, 0x7b, 0xb3, 0x13, 0x3b, 0xb3, 0x98, 0x9d, 0xbf,
0xad, 0x64, 0xe6, 0x12, 0x54, 0x69, 0x17, 0x49, 0x6b, 0xda, 0xdd, 0xa6, 0xbb, 0x9d, 0xb1, 0xf7,
0x02, 0x89, 0x95, 0x10, 0xdc, 0x70, 0x8f, 0x40, 0x82, 0xa7, 0xe0, 0x1d, 0xb8, 0x41, 0xe2, 0x11,
0xd0, 0xf0, 0x22, 0xa8, 0x4e, 0x55, 0x75, 0x97, 0x3d, 0x3d, 0xce, 0x32, 0xda, 0xbb, 0x3e, 0xdf,
0x29, 0x9f, 0xfa, 0xce, 0xa9, 0xaf, 0x4f, 0x9d, 0x36, 0x74, 0x67, 0x59, 0x74, 0xcd, 0x0b, 0x71,
0x34, 0xcb, 0xd2, 0x22, 0x25, 0xee, 0xec, 0xe2, 0x60, 0x7b, 0x36, 0xbf, 0x88, 0xa3, 0x50, 0x21,
0x34, 0x82, 0xf6, 0x38, 0x99, 0x88, 0xc5, 0x13, 0x51, 0x70, 0x42, 0xc0, 0xff, 0x42, 0x2c, 0xf3,
0xc0, 0xeb, 0x3b, 0x83, 0x16, 0xc3, 0x67, 0x72, 0x0f, 0x76, 0xce, 0x33, 0x1e, 0xbe, 0x3a, 0x5d,
0x44, 0x79, 0x21, 0x92, 0x50, 0x04, 0x3e, 0x7a, 0xd7, 0x50, 0xd2, 0x87, 0xce, 0x48, 0xe4, 0x61,
0x16, 0xcd, 0x8a, 0x28, 0x4d, 0x82, 0xad, 0xbe, 0x33, 0x68, 0x33, 0x1b, 0xa2, 0x5f, 0xfb, 0xb0,
0xfd, 0x28, 0x12, 0xf1, 0xe4, 0x19, 0xda, 0xb9, 0xdc, 0xee, 0x7c, 0x39, 0x13, 0x41, 0x0b, 0xd7,
0xe2, 0x33, 0xb9, 0x03, 0xed, 0x21, 0x0f, 0xaf, 0x04, 0x3a, 0x3c, 0x74, 0x54, 0x40, 0xe9, 0x3d,
0x8b, 0xbe, 0x52, 0x3c, 0xba, 0xac, 0x02, 0x24, 0x85, 0xf3, 0x68, 0x2a, 0xbe, 0x9c, 0xf3, 0xa4,
0x98, 0x4f, 0x0d, 0x05, 0x0b, 0x22, 0xfb, 0xd0, 0x78, 0x16, 0x4f, 0x9e, 0x44, 0x49, 0xd0, 0xee,
0x3b, 0x03, 0x8f, 0x69, 0xcb, 0xe0, 0x7c, 0x11, 0x40, 0x85, 0xf3, 0x45, 0x59, 0x90, 0xce, 0x6a,
0x41, 0x9e, 0xa6, 0x67, 0x05, 0x4f, 0x26, 0x3c, 0x9b, 0xbc, 0x8c, 0xc4, 0xeb, 0x60, 0x5b, 0x15,
0x64, 0x15, 0x95, 0xbf, 0x3d, 0xe1, 0xb9, 0x08, 0xba, 0x18, 0x11, 0x9f, 0xc9, 0x01, 0xb4, 0x4e,
0xa2, 0x62, 0x24, 0x66, 0xc5, 0x55, 0xb0, 0xd3, 0x77, 0x06, 0x3e, 0x2b, 0x6d, 0xb2, 0x07, 0x5b,
0x67, 0x21, 0x8f, 0x45, 0x70, 0x0b, 0x7f, 0xa0, 0x0c, 0x42, 0x61, 0xfb, 0x51, 0x9a, 0x89, 0xe8,
0x32, 0xc1, 0x63, 0x0a, 0x7a, 0x98, 0xd4, 0x0a, 0x46, 0xbe, 0x07, 0x9e, 0x4c, 0x69, 0xb7, 0xef,
0x0c, 0x3a, 0xc7, 0x9d, 0xa3, 0xd9, 0xc5, 0xd1, 0x48, 0x84, 0xd1, 0x94, 0xc7, 0x4c, 0xe2, 0xe8,
0xe6, 0x8b, 0x80, 0xd4, 0xb9, 0xf9, 0x42, 0x72, 0x92, 0x25, 0x7a, 0x91, 0x44, 0x45, 0xf0, 0x01,
0x46, 0x2f, 0x6d, 0xd2, 0x03, 0xef, 0xfc, 0xfc, 0x71, 0xb0, 0x87, 0xb0, 0x7c, 0xac, 0x91, 0xc3,
0x87, 0xb5, 0x72, 0xd8, 0x87, 0xc6, 0x63, 0x91, 0x5c, 0x16, 0x57, 0xc1, 0xbe, 0xaa, 0xa8, 0xb2,
0x28, 0x85, 0x9d, 0xf1, 0x74, 0x96, 0x66, 0x05, 0x13, 0xf9, 0x2c, 0x4d, 0x72, 0x21, 0xf7, 0x38,
0xcd, 0xb2, 0xc0, 0x51, 0x7b, 0x9c, 0x66, 0x19, 0xfd, 0x2d, 0xf4, 0x4e, 0xe2, 0x34, 0x7c, 0x35,
0xe2, 0x05, 0x67, 0xe2, 0x37, 0x73, 0x91, 0x17, 0xb2, 0x3a, 0xaa, 0x00, 0x6a, 0x9d, 0x32, 0x24,
0x8a, 0x8a, 0x0a, 0x5c, 0x85, 0xa2, 0x21, 0x2b, 0x8f, 0xe7, 0xa2, 0x04, 0x80, 0xcf, 0x58, 0xdd,
0x2b, 0x9e, 0x4d, 0x50, 0x35, 0x3e, 0x53, 0x86, 0x44, 0x71, 0x27, 0x54, 0x9a, 0xcf, 0x94, 0x41,
0xc7, 0xb0, 0x6b, 0xed, 0xaf, 0x69, 0xee, 0x43, 0x83, 0xa5, 0xaf, 0xc7, 0xa3, 0x3c, 0x70, 0xfa,
0xde, 0xc0, 0x67, 0xda, 0x42, 0x49, 0xa6, 0xf1, 0x7c, 0x9a, 0x48, 0x97, 0x8b, 0xae, 0x0a, 0xa0,
0xb7, 0x61, 0x0b, 0xf5, 0x29, 0xb3, 0xac, 0x7e, 0x2b, 0x1f, 0xe9, 0xef, 0x1c, 0x68, 0x3f, 0xe1,
0x0b, 0x24, 0x92, 0x93, 0x4f, 0xa0, 0x65, 0xd4, 0x83, 0x8b, 0x3a, 0xc7, 0x1f, 0xc9, 0x93, 0x2a,
0x17, 0x1c, 0x19, 0xef, 0x69, 0x52, 0x64, 0x4b, 0x56, 0x2e, 0x3e, 0xf8, 0x14, 0xba, 0x2b, 0x2e,
0xb9, 0xd3, 0x2b, 0xb1, 0x34, 0xf5, 0x7c, 0x25, 0x96, 0x32, 0xcb, 0x6b, 0x1e, 0xcf, 0x05, 0x56,
0xc9, 0x67, 0xca, 0xf8, 0x99, 0xfb, 0x53, 0x87, 0xbe, 0x04, 0x32, 0xcc, 0x04, 0x2f, 0x04, 0x6e,
0xf2, 0x44, 0xe4, 0x39, 0xbf, 0x14, 0x37, 0xd5, 0xda, 0xb3, 0x6b, 0x5d, 0xd6, 0xd5, 0xb5, 0xea,
0x4a, 0x1f, 0x00, 0x19, 0x89, 0x58, 0x14, 0x42, 0xf7, 0x96, 0x0d, 0x71, 0xe9, 0x5f, 0x1d, 0x43,
0xe2, 0xe6, 0xc5, 0xe4, 0x2e, 0xf8, 0xb2, 0x53, 0xe1, 0x6e, 0x9d, 0xe3, 0xae, 0x2c, 0x51, 0xd9,
0xbe, 0x18, 0xba, 0xf0, 0x40, 0x30, 0xdc, 0xe4, 0x61, 0x81, 0x5c, 0x3d, 0x56, 0x01, 0x32, 0xec,
0xb3, 0xd7, 0x89, 0xc8, 0xb4, 0x38, 0x94, 0x41, 0x02, 0x68, 0x62, 0x98, 0xf1, 0x28, 0x68, 0xf4,
0x9d, 0xc1, 0x16, 0x33, 0x26, 0xfd, 0x4b, 0xc9, 0x0e, 0xf3, 0xfd, 0x86, 0x25, 0x5a, 0x91, 0xe3,
0x0f, 0x34, 0x67, 0x0f, 0x39, 0xf7, 0x24, 0x67, 0xbb, 0x0d, 0xd6, 0xd1, 0xf6, 0xbf, 0x11, 0x6d,
0xfa, 0x7b, 0x07, 0xc8, 0x8b, 0xd9, 0x64, 0x9d, 0xdc, 0xa3, 0x3a, 0xca, 0xc8, 0xb4, 0x73, 0xbc,
0x2f, 0xb7, 0x7f, 0xdb, 0xcb, 0xea, 0x92, 0xbc, 0x0f, 0x0d, 0x15, 0x5d, 0x97, 0xfb, 0x56, 0x49,
0x5d, 0xc1, 0x4c, 0xbb, 0xe9, 0xa7, 0xd0, 0xb1, 0x60, 0xec, 0xa6, 0xea, 0x16, 0x50, 0xd5, 0xd1,
0x96, 0x4c, 0xe2, 0x65, 0xa9, 0xc3, 0x36, 0x53, 0x06, 0xfd, 0xcc, 0x68, 0xe5, 0x7d, 0x0b, 0x4c,
0x43, 0xf8, 0x48, 0x45, 0x78, 0x78, 0xcd, 0xa3, 0x98, 0x5f, 0xc4, 0xff, 0x97, 0x9c, 0x57, 0xce,
0x2a, 0x80, 0x26, 0xfe, 0x76, 0x3c, 0xd2, 0x2d, 0xc1, 0x98, 0x74, 0x0e, 0x55, 0x77, 0x79, 0xca,
0xa7, 0x42, 0x47, 0xc3, 0xe7, 0xf2, 0x88, 0xdd, 0x8d, 0x47, 0x2c, 0xf3, 0x8f, 0xc4, 0x6b, 0x79,
0xbf, 0x7a, 0x98, 0xbf, 0x34, 0x36, 0x1f, 0x3c, 0xfd, 0x11, 0x34, 0xce, 0xc2, 0x2b, 0x31, 0xe5,
0xe4, 0xfb, 0x5a, 0xa3, 0x22, 0xd7, 0x0d, 0xa2, 0x5d, 0xaa, 0x9f, 0x19, 0x0f, 0xfd, 0xbb, 0xa3,
0x93, 0xad, 0xa5, 0xb9, 0xb2, 0x95, 0xbb, 0xae, 0xb1, 0xfb, 0xd0, 0xd4, 0x7c, 0x51, 0x65, 0x6f,
0xbd, 0x5e, 0xc6, 0x4b, 0xee, 0x42, 0x03, 0xb3, 0xcb, 0x03, 0xbf, 0x22, 0x82, 0x08, 0xd3, 0x8e,
0x0d, 0x2f, 0xd4, 0x29, 0x78, 0x2f, 0xd8, 0x58, 0x6a, 0x04, 0xf3, 0x32, 0x04, 0xb5, 0x25, 0x69,
0xff, 0x3c, 0xcd, 0x0b, 0x7d, 0x2a, 0xf8, 0x2c, 0xb1, 0xe7, 0x69, 0xa6, 0x5e, 0xe6, 0x2e, 0xc3,
0x67, 0xfa, 0x47, 0x07, 0xfc, 0xa7, 0xe9, 0x44, 0x90, 0x1d, 0x70, 0xc7, 0x23, 0x1d, 0xc4, 0x1d,
0x8f, 0xc8, 0x6d, 0x8c, 0xaf, 0x4f, 0xa2, 0x29, 0x99, 0xbd, 0x60, 0x63, 0x86, 0x7b, 0xde, 0x81,
0xf6, 0x38, 0x7f, 0x9e, 0x45, 0x53, 0x9e, 0x2d, 0xf5, 0x8c, 0x53, 0x01, 0xd8, 0xc9, 0x0a, 0x29,
0x76, 0x5f, 0x09, 0x02, 0x0d, 0x72, 0x17, 0x9a, 0x9f, 0xb3, 0xe7, 0x43, 0x19, 0x72, 0x6b, 0x35,
0xa4, 0xc1, 0xe9, 0x67, 0xd0, 0x93, 0x4c, 0x70, 0xbd, 0xd1, 0xdc, 0x3e, 0x34, 0x24, 0x56, 0x32,
0xd3, 0x56, 0xb5, 0x89, 0x6b, 0x6d, 0x42, 0x1f, 0xa9, 0x08, 0xa7, 0xd7, 0x22, 0x29, 0x2c, 0xd5,
0xa2, 0x8d, 0x01, 0xba, 0x4c, 0x19, 0xe4, 0x8e, 0xca, 0x5a, 0xa7, 0xd7, 0x92, 0x5c, 0xa4, 0xcd,
0x10, 0xa5, 0x4b, 0x00, 0xc3, 0x64, 0x9e, 0x97, 0x6b, 0x9d, 0xba, 0xb5, 0x84, 0x1a, 0x61, 0xe9,
0xbe, 0x04, 0xd2, 0xaf, 0x10, 0x66, 0x24, 0xf7, 0xc3, 0x4a, 0x72, 0xea, 0xa4, 0x6f, 0x95, 0x8a,
0x50, 0x7b, 0x54, 0xc2, 0xbb, 0x82, 0x8e, 0x85, 0xd7, 0xaa, 0xef, 0x7e, 0x29, 0x1b, 0xb7, 0x0a,
0x86, 0x88, 0x0e, 0x66, 0xc4, 0xb3, 0xb1, 0x83, 0xd3, 0x48, 0x37, 0x9b, 0x0d, 0x3b, 0x0d, 0xe0,
0xd6, 0x6a, 0x2b, 0x30, 0x37, 0xf3, 0x3a, 0x7c, 0xc3, 0x56, 0x7f, 0x70, 0xa0, 0x3b, 0x8c, 0xe7,
0x79, 0x21, 0xb2, 0xb2, 0xa6, 0x6d, 0x0d, 0x94, 0x47, 0x5b, 0x01, 0xf5, 0xa7, 0x4b, 0x0e, 0x61,
0x4b, 0x56, 0x5c, 0xbd, 0xf6, 0xf6, 0x41, 0x28, 0xd8, 0x3a, 0x09, 0xff, 0x5d, 0x27, 0x41, 0x5f,
0x42, 0xeb, 0xe4, 0x6c, 0xfc, 0x79, 0x96, 0xce, 0x67, 0xb5, 0x19, 0x9b, 0x51, 0xda, 0xb5, 0x46,
0xe9, 0x9e, 0x1a, 0x0b, 0x55, 0x56, 0x38, 0x09, 0xf6, 0xd4, 0x24, 0xe8, 0x6b, 0x84, 0x2f, 0xe8,
0x19, 0xec, 0xaa, 0x74, 0x65, 0x2f, 0x7a, 0x9f, 0x86, 0x69, 0x66, 0x2d, 0xaf, 0x9a, 0xb5, 0x64,
0x50, 0xd5, 0x8f, 0xbf, 0xcd, 0xa0, 0xff, 0x72, 0x61, 0x97, 0x89, 0x3c, 0xfa, 0x4a, 0x8c, 0x93,
0xbc, 0xc8, 0xe6, 0xa1, 0xb9, 0x52, 0x7e, 0x91, 0x5e, 0xe8, 0xb3, 0xf0, 0x98, 0x32, 0x36, 0xbf,
0x25, 0x84, 0x42, 0xd3, 0x6e, 0x02, 0xf6, 0x02, 0xe3, 0x20, 0x0f, 0xa0, 0x79, 0x96, 0xce, 0xb3,
0xb0, 0x54, 0x3e, 0xf6, 0x74, 0xb5, 0xbf, 0x72, 0x30, 0xb3, 0x80, 0x7c, 0x01, 0xe4, 0x3c, 0xe3,
0x49, 0x1e, 0x73, 0x49, 0xc9, 0xfc, 0xac, 0x55, 0x0d, 0x71, 0x96, 0x77, 0x25, 0x42, 0xcd, 0xcf,
0xc8, 0x91, 0xfd, 0x0a, 0x07, 0x4d, 0xe4, 0xb7, 0x63, 0xf8, 0xe9, 0xf7, 0xc4, 0x7e, 0xc9, 0x3f,
0x59, 0x53, 0x28, 0xb6, 0xdb, 0xce, 0xf1, 0x2e, 0x5e, 0xf3, 0xb6, 0x83, 0xad, 0xae, 0xa3, 0x5f,
0x3b, 0xb0, 0x6d, 0xb3, 0xb9, 0xa1, 0x5d, 0x94, 0xc7, 0xe7, 0xde, 0x3c, 0x13, 0x9a, 0xe3, 0xf3,
0xeb, 0xe6, 0xef, 0x2d, 0x7b, 0x4e, 0x4c, 0xe1, 0xbb, 0xef, 0x28, 0xce, 0x7b, 0xd1, 0xe9, 0x43,
0xe7, 0x39, 0xcf, 0x8a, 0x48, 0x06, 0xd3, 0x37, 0xf8, 0x16, 0xb3, 0x21, 0x2a, 0xe0, 0xf6, 0x5b,
0x22, 0x1a, 0xa6, 0xd3, 0x99, 0x54, 0xeb, 0x7b, 0x89, 0x49, 0xb6, 0xe9, 0x2c, 0x4b, 0x33, 0x53,
0x01, 0x34, 0xe8, 0x09, 0xb4, 0xce, 0xd3, 0x59, 0x1a, 0xa7, 0x97, 0xcb, 0x1b, 0x5a, 0x46, 0x00,
0x4d, 0x75, 0x35, 0xa8, 0x16, 0xd5, 0x66, 0xc6, 0xa4, 0x1f, 0x48, 0xbd, 0x87, 0x3c, 0x0e, 0xe7,
0x31, 0x2f, 0x04, 0x7e, 0x45, 0x20, 0xf8, 0x38, 0xe5, 0x13, 0xd5, 0x15, 0xf4, 0xab, 0x45, 0x7f,
0xa5, 0x05, 0xc8, 0x31, 0x1d, 0xeb, 0x0a, 0x7a, 0x18, 0xda, 0x53, 0x98, 0xb2, 0xc8, 0x4f, 0xa0,
0x63, 0xad, 0xb6, 0x47, 0x3b, 0x0b, 0x66, 0xf6, 0x1a, 0xfa, 0x0f, 0x67, 0xe5, 0x37, 0x6f, 0xdd,
0xb9, 0x7a, 0xab, 0x6b, 0x55, 0xa4, 0x16, 0xd3, 0x96, 0x4c, 0xfd, 0x74, 0x11, 0xc6, 0xf3, 0x5c,
0xba, 0xf4, 0x85, 0x5b, 0x02, 0x32, 0x75, 0xf9, 0xa1, 0x99, 0xce, 0xcd, 0xd8, 0x63, 0x4c, 0xf9,
0x49, 0x3a, 0x12, 0x7c, 0x12, 0x47, 0x89, 0x40, 0xbd, 0x78, 0xac, 0xb4, 0xc9, 0x03, 0xd5, 0x63,
0x8d, 0xd0, 0xf7, 0xd6, 0x88, 0xa3, 0x4f, 0x75, 0xde, 0x9c, 0x12, 0xe8, 0xad, 0xbb, 0xe8, 0x1e,
0x10, 0xa5, 0x80, 0x87, 0x17, 0x69, 0x66, 0x6e, 0x5b, 0x3a, 0x34, 0xcd, 0x45, 0x56, 0xff, 0xa6,
0x4b, 0xbc, 0xaa, 0xac, 0x6b, 0x57, 0x96, 0xfe, 0x12, 0x76, 0xf4, 0xd4, 0x27, 0x32, 0x14, 0xb4,
0x2c, 0x00, 0x13, 0x61, 0x2a, 0x07, 0x48, 0xf3, 0xed, 0x57, 0x01, 0x32, 0x0e, 0x8e, 0xc0, 0xe6,
0x76, 0xd2, 0x16, 0xce, 0x46, 0xd1, 0x65, 0x22, 0x26, 0x78, 0x63, 0x78, 0x4c, 0x5b, 0xf4, 0x4f,
0x2e, 0xec, 0xa9, 0x71, 0x34, 0xb9, 0x14, 0x79, 0x51, 0x6d, 0x83, 0x03, 0x37, 0xf6, 0xff, 0x72,
0xe0, 0xc6, 0x1b, 0xe0, 0x1e, 0xec, 0x0c, 0x63, 0xc1, 0xb3, 0x8a, 0x83, 0xda, 0x68, 0x0d, 0x95,
0xef, 0x0d, 0x22, 0xfa, 0x7a, 0x56, 0xe3, 0xa9, 0x0d, 0x91, 0x13, 0x68, 0xe9, 0xd4, 0x4c, 0x43,
0xbc, 0x87, 0xb7, 0x54, 0x0d, 0x1b, 0x33, 0xf9, 0xe6, 0xfa, 0x4b, 0xd5, 0x98, 0x07, 0xcf, 0xa0,
0xbb, 0xe2, 0xaa, 0xf9, 0x52, 0x1d, 0xd8, 0x5f, 0xaa, 0x9d, 0x63, 0x62, 0x0d, 0xd2, 0x3a, 0xba,
0xfd, 0xf5, 0x3a, 0x84, 0x0f, 0xeb, 0x08, 0xe4, 0xe4, 0x01, 0x78, 0x92, 0xa8, 0x1a, 0x93, 0x83,
0x77, 0x11, 0x65, 0x72, 0x11, 0xfd, 0x9b, 0xa3, 0x8b, 0x2a, 0xb4, 0xdf, 0xfc, 0xe3, 0xf0, 0xb1,
0x1d, 0xe4, 0x6e, 0x19, 0x64, 0x6d, 0xd9, 0x51, 0x99, 0xa8, 0x5c, 0x7d, 0xf0, 0x25, 0xb4, 0xea,
0xd2, 0xf3, 0x55, 0x7a, 0x3f, 0x5e, 0x4d, 0xef, 0xf6, 0xbb, 0x98, 0xe5, 0x76, 0x96, 0x47, 0xb0,
0xaf, 0x6e, 0xd3, 0x11, 0x2f, 0xf8, 0xaf, 0x33, 0x3e, 0x15, 0x1b, 0xaf, 0xd4, 0x93, 0xde, 0x3f,
0xdf, 0x1c, 0x3a, 0xff, 0x7e, 0x73, 0xe8, 0xfc, 0xe7, 0xcd, 0xa1, 0xf3, 0xe7, 0xff, 0x1e, 0x7e,
0xe7, 0xa2, 0x81, 0x7f, 0xf5, 0x7d, 0xfc, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xe2, 0xbe, 0x5d,
0xe0, 0x0d, 0x14, 0x00, 0x00,
}
func (m *IndexMeta) Marshal() (dAtA []byte, err error) {
@ -3150,6 +3176,13 @@ func (m *FieldOptions) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.Length != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.Length))
i--
dAtA[i] = 0x1
i--
dAtA[i] = 0xb0
}
if m.TrackExistence {
i--
if m.TrackExistence {
@ -3639,6 +3672,11 @@ func (m *CreateIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.IndexID != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.IndexID))
i--
dAtA[i] = 0x30
}
if len(m.Owner) > 0 {
i -= len(m.Owner)
copy(dAtA[i:], m.Owner)
@ -4042,6 +4080,11 @@ func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.IndexID != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.IndexID))
i--
dAtA[i] = 0x30
}
if m.Options != nil {
{
size, err := m.Options.MarshalToSizedBuffer(dAtA[:i])
@ -5660,6 +5703,9 @@ func (m *FieldOptions) Size() (n int) {
if m.TrackExistence {
n += 3
}
if m.Length != 0 {
n += 2 + sovPrivate(uint64(m.Length))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -5837,6 +5883,9 @@ func (m *CreateIndexMessage) Size() (n int) {
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
if m.IndexID != 0 {
n += 1 + sovPrivate(uint64(m.IndexID))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -6027,6 +6076,9 @@ func (m *Index) Size() (n int) {
l = m.Options.Size()
n += 1 + l + sovPrivate(uint64(l))
}
if m.IndexID != 0 {
n += 1 + sovPrivate(uint64(m.IndexID))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -7299,6 +7351,25 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error {
}
}
m.TrackExistence = bool(v != 0)
case 22:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Length", wireType)
}
m.Length = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Length |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -8448,6 +8519,25 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error {
}
m.Owner = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IndexID", wireType)
}
m.IndexID = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.IndexID |= int32(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -9564,6 +9654,25 @@ func (m *Index) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IndexID", wireType)
}
m.IndexID = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.IndexID |= int32(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])

3
pb/private.proto
View file

@ -28,6 +28,7 @@ message FieldOptions {
string TimeUnit = 19;
string TTL = 20;
bool TrackExistence = 21;
int64 Length = 22;
}
message ImportResponse {
@ -70,6 +71,7 @@ message CreateIndexMessage {
IndexMeta Meta = 2;
int64 CreatedAt = 3;
string Owner = 5;
int32 IndexID = 6;
}
message CreateFieldMessage {
@ -117,6 +119,7 @@ message Index {
int64 CreatedAt = 2;
IndexMeta Options = 5;
repeated Field Fields = 4;
int32 IndexID = 6;
}
message URI {

2
pb/public.pb.go
View file

@ -575,6 +575,7 @@ func (m *KeyList) GetKeys() []string {
type ExtractedTableValue struct {
// Types that are valid to be assigned to Value:
//
// *ExtractedTableValue_IDs
// *ExtractedTableValue_Keys
// *ExtractedTableValue_BSIValue
@ -715,6 +716,7 @@ func (*ExtractedTableValue) XXX_OneofWrappers() []interface{} {
type ExtractedTableColumn struct {
// Types that are valid to be assigned to KeyOrID:
//
// *ExtractedTableColumn_Key
// *ExtractedTableColumn_ID
KeyOrID isExtractedTableColumn_KeyOrID `protobuf_oneof:"KeyOrID"`

11
schema.go
View file

@ -202,6 +202,7 @@ func FieldInfoToField(fi *FieldInfo) *dax.Field {
var epoch time.Time
var foreignIndex string
var timeQuantum dax.TimeQuantum
var length int64
fo := &fi.Options
@ -245,6 +246,9 @@ func FieldInfoToField(fi *FieldInfo) *dax.Field {
fieldType = dax.BaseTypeIDSetQ
}
timeQuantum = dax.TimeQuantum(fo.TimeQuantum)
case FieldTypeVarchar:
fieldType = dax.BaseTypeVarchar
length = fo.Length
default:
panic(fmt.Sprintf("unhandled featurebase field type: %s", fo.Type))
}
@ -265,6 +269,7 @@ func FieldInfoToField(fi *FieldInfo) *dax.Field {
TTL: fo.TTL,
ForeignIndex: foreignIndex,
TrackExistence: fo.TrackExistence,
Length: length,
},
}
}
@ -360,6 +365,7 @@ func FieldToFieldInfo(fld *dax.Field) *FieldInfo {
Min: min,
Max: max,
Scale: fld.Options.Scale,
Length: fld.Options.Length,
Keys: fld.StringKeys(),
NoStandardView: fld.Options.NoStandardView,
CacheType: fld.Options.CacheType,
@ -491,6 +497,11 @@ func FieldOptionsFromField(fld *dax.Field) ([]FieldOption, error) {
opts = append(opts,
OptFieldTypeTimestamp(fld.Options.Epoch, fld.Options.TimeUnit),
)
case dax.BaseTypeVarchar:
opts = append(opts,
OptFieldTypeVarchar(fld.Options.Length),
)
default:
return nil, errors.Errorf("unsupport field type: %s", fld.Type)
}

10
sql3/errors.go
View file

@ -46,6 +46,9 @@ const (
// decimal
ErrDecimalScaleExpected errors.Code = "ErrDecimalScaleExpected"
// varchar
ErrVarcharLengthExpected errors.Code = "ErrVarcharLengthExpected"
ErrInvalidCast errors.Code = "ErrInvalidCast"
ErrInvalidTypeCoercion errors.Code = "ErrInvalidTypeCoercion"
@ -507,6 +510,13 @@ func NewErrDecimalScaleExpected(line, col int) error {
)
}
func NewErrVarcharLengthExpected(line, col int) error {
return errors.New(
ErrVarcharLengthExpected,
fmt.Sprintf("[%d:%d] varchar length expected", line, col),
)
}
func NewErrInvalidTimeUnit(line, col int, unit string) error {
return errors.New(
ErrInvalidTimeUnit,

21
sql3/parser/ast.go
View file

@ -1787,11 +1787,10 @@ func (i *Variable) VarName() string {
}
type Type struct {
Name *Ident // type name
Lparen Pos // position of left paren (optional)
Precision *IntegerLit // precision (optional)
Scale *IntegerLit // scale (optional)
Rparen Pos // position of right paren (optional)
Name *Ident // type name
Lparen Pos // position of left paren (optional)
Modifier *IntegerLit // scale, length etc. (optional)
Rparen Pos // position of right paren (optional)
}
// Clone returns a deep copy of t.
@ -1801,20 +1800,14 @@ func (t *Type) Clone() *Type {
}
other := *t
other.Name = t.Name.Clone()
other.Precision = t.Precision.Clone()
other.Scale = t.Scale.Clone()
other.Modifier = t.Modifier.Clone()
return &other
}
// String returns the string representation of the type.
func (t *Type) String() string {
if t.Precision != nil && t.Scale != nil {
return fmt.Sprintf("%s(%s,%s)", t.Name.Name, t.Precision.String(), t.Scale.String())
} else if t.Precision != nil {
return fmt.Sprintf("%s(%s)", t.Name.Name, t.Precision.String())
} else if t.Scale != nil {
// I'm not sure how you're supposed to tell this from the t.Precision case.
return fmt.Sprintf("%s(%s)", t.Name.Name, t.Scale.String())
if t.Modifier != nil {
return fmt.Sprintf("%s(%s)", t.Name.Name, t.Modifier.String())
}
return t.Name.Name
}

6
sql3/parser/ast_test.go
View file

@ -347,7 +347,7 @@ func OLDTestCreateTableStatement_String(t *testing.T) {
Name: &parser.Ident{Name: "foo"},
Columns: []*parser.ColumnDefinition{{
Name: &parser.Ident{Name: "bar"},
Type: &parser.Type{Name: &parser.Ident{Name: "DECIMAL"}, Precision: &parser.IntegerLit{Value: "100"}},
Type: &parser.Type{Name: &parser.Ident{Name: "DECIMAL"}, Modifier: &parser.IntegerLit{Value: "100"}},
}},
Constraints: []parser.Constraint{
&parser.PrimaryKeyConstraint{
@ -375,7 +375,7 @@ func OLDTestCreateTableStatement_String(t *testing.T) {
Name: &parser.Ident{Name: "foo"},
Columns: []*parser.ColumnDefinition{{
Name: &parser.Ident{Name: "bar"},
Type: &parser.Type{Name: &parser.Ident{Name: "DECIMAL"}, Precision: &parser.IntegerLit{Value: "100"}, Scale: &parser.IntegerLit{Value: "200"}},
Type: &parser.Type{Name: &parser.Ident{Name: "DECIMAL"}, Modifier: &parser.IntegerLit{Value: "200"}},
}},
Constraints: []parser.Constraint{
&parser.ForeignKeyConstraint{
@ -385,7 +385,7 @@ func OLDTestCreateTableStatement_String(t *testing.T) {
ForeignColumns: []*parser.Ident{{Name: "y"}, {Name: "z"}},
},
},
}, `CREATE TABLE "foo" ("bar" DECIMAL(100,200), CONSTRAINT "fk" FOREIGN KEY ("a", "b") REFERENCES "x" ("y", "z"))`)
}, `CREATE TABLE "foo" ("bar" DECIMAL(200), CONSTRAINT "fk" FOREIGN KEY ("a", "b") REFERENCES "x" ("y", "z"))`)
AssertStatementStringer(t, &parser.CreateTableStatement{
Name: &parser.Ident{Name: "foo"},

28
sql3/parser/astdatatype.go
View file

@ -18,7 +18,8 @@ func IsValidTypeName(typeName string) bool {
dax.BaseTypeString,
dax.BaseTypeStringSet,
dax.BaseTypeStringSetQ,
dax.BaseTypeTimestamp:
dax.BaseTypeTimestamp,
dax.BaseTypeVarchar:
return true
default:
return false
@ -54,6 +55,7 @@ func (*DataTypeString) exprDataType() {}
func (*DataTypeStringSet) exprDataType() {}
func (*DataTypeStringSetQuantum) exprDataType() {}
func (*DataTypeTimestamp) exprDataType() {}
func (*DataTypeVarchar) exprDataType() {}
type DataTypeVoid struct {
}
@ -202,6 +204,30 @@ func (d *DataTypeDecimal) TypeInfo() map[string]interface{} {
}
}
type DataTypeVarchar struct {
Length int64
}
func NewDataTypeVarchar(length int64) *DataTypeVarchar {
return &DataTypeVarchar{
Length: length,
}
}
func (d *DataTypeVarchar) BaseTypeName() string {
return dax.BaseTypeVarchar
}
func (d *DataTypeVarchar) TypeDescription() string {
return fmt.Sprintf("%s(%d)", dax.BaseTypeVarchar, d.Length)
}
func (d *DataTypeVarchar) TypeInfo() map[string]interface{} {
return map[string]interface{}{
"length": d.Length,
}
}
type DataTypeID struct {
}

4
sql3/parser/parser.go
View file

@ -1769,10 +1769,10 @@ func (p *Parser) parseType() (_ *Type, err error) {
return &typ, err
}
// Optionally parse scale.
// Optionally parse modifier.
if p.peek() == LP {
typ.Lparen, _, _ = p.scan()
if typ.Scale, err = p.parseIntegerLiteral("scale"); err != nil {
if typ.Modifier, err = p.parseIntegerLiteral("scale"); err != nil {
return &typ, err
}

8
sql3/parser/parser_test.go
View file

@ -997,10 +997,10 @@ func TestParser_ParseStatement(t *testing.T) {
{
Name: &parser.Ident{NamePos: pos(29), Name: "col2"},
Type: &parser.Type{
Name: &parser.Ident{NamePos: pos(34), Name: "DECIMAL"},
Lparen: pos(41),
Scale: &parser.IntegerLit{ValuePos: pos(42), Value: "2"},
Rparen: pos(43),
Name: &parser.Ident{NamePos: pos(34), Name: "DECIMAL"},
Lparen: pos(41),
Modifier: &parser.IntegerLit{ValuePos: pos(42), Value: "2"},
Rparen: pos(43),
},
},
},

17
sql3/parser/walk.go
View file

@ -716,22 +716,13 @@ func walk(v Visitor, node Node) (_ Node, err error) {
if err := walkIdent(v, &n.Name); err != nil {
return node, err
}
if n.Precision != nil {
if p, err := walk(v, n.Precision); err != nil {
return node, err
} else if p != nil {
n.Precision = p.(*IntegerLit)
} else {
n.Precision = nil
}
}
if n.Scale != nil {
if scale, err := walk(v, n.Scale); err != nil {
if n.Modifier != nil {
if scale, err := walk(v, n.Modifier); err != nil {
return node, err
} else if scale != nil {
n.Scale = scale.(*IntegerLit)
n.Modifier = scale.(*IntegerLit)
} else {
n.Scale = nil
n.Modifier = nil
}
}
}

4
sql3/planner/compilebulkinsert.go
View file

@ -114,7 +114,7 @@ func (p *ExecutionPlanner) compileBulkInsertStatement(ctx context.Context, stmt
for _, m := range stmt.Columns {
for idx, fld := range tbl.Fields {
if strings.EqualFold(string(fld.Name), m.Name) {
options.targetColumns = append(options.targetColumns, newQualifiedRefPlanExpression(tableName, strings.ToLower(m.Name), idx, fieldSQLDataType(pilosa.FieldToFieldInfo(fld))))
options.targetColumns = append(options.targetColumns, newQualifiedRefPlanExpression(tableName, strings.ToLower(m.Name), idx, FieldSQLDataType(pilosa.FieldToFieldInfo(fld))))
break
}
}
@ -348,7 +348,7 @@ func (p *ExecutionPlanner) analyzeBulkInsertStatement(ctx context.Context, stmt
for _, fld := range tbl.Fields {
if strings.EqualFold(cm.Name, string(fld.Name)) {
found = true
colDataType := fieldSQLDataType(pilosa.FieldToFieldInfo(fld))
colDataType := FieldSQLDataType(pilosa.FieldToFieldInfo(fld))
// if we have transforms check that type and target colum ref are assignment compatible
// else check that the map expressions type and target column ref are assignment compatible

18
sql3/planner/compilecreatetable.go
View file

@ -93,6 +93,7 @@ func (p *ExecutionPlanner) compileColumn(ctx context.Context, col *parser.Column
var cacheType string = pilosa.DefaultCacheType
var cacheSize uint32 = pilosa.DefaultCacheSize
var scale int64
var length int64
min, max := pql.MinMax(0)
var epoch = pilosa.DefaultEpoch
var timeUnit string = pilosa.TimeUnitSeconds
@ -190,11 +191,11 @@ func (p *ExecutionPlanner) compileColumn(ctx context.Context, col *parser.Column
case dax.BaseTypeDecimal:
// if we don't have a scale, it's an error
if col.Type.Scale == nil {
if col.Type.Modifier == nil {
return nil, sql3.NewErrDecimalScaleExpected(col.Type.Name.NamePos.Line, col.Type.Name.NamePos.Column)
}
// get the scale value
scale, err = strconv.ParseInt(col.Type.Scale.Value, 10, 64)
scale, err = strconv.ParseInt(col.Type.Modifier.Value, 10, 64)
if err != nil {
return nil, err
}
@ -235,6 +236,19 @@ func (p *ExecutionPlanner) compileColumn(ctx context.Context, col *parser.Column
case dax.BaseTypeTimestamp:
column.fos = append(column.fos, pilosa.OptFieldTypeTimestamp(epoch, timeUnit))
case dax.BaseTypeVarchar:
// if we don't have a length, it's an error
if col.Type.Modifier == nil {
return nil, sql3.NewErrVarcharLengthExpected(col.Type.Name.NamePos.Line, col.Type.Name.NamePos.Column)
}
// get the modifier value
length, err = strconv.ParseInt(col.Type.Modifier.Value, 10, 64)
if err != nil {
return nil, err
}
column.fos = append(column.fos, pilosa.OptFieldTypeVarchar(length))
}
return column, nil
}

8
sql3/planner/compileinsert.go
View file

@ -40,7 +40,7 @@ func (p *ExecutionPlanner) compileInsertStatement(ctx context.Context, stmt *par
for idx, field := range tbl.Fields {
if strings.EqualFold(colName, string(field.Name)) {
targetColumns = append(targetColumns, newQualifiedRefPlanExpression(tableName, colName, idx, fieldSQLDataType(pilosa.FieldToFieldInfo(field))))
targetColumns = append(targetColumns, newQualifiedRefPlanExpression(tableName, colName, idx, FieldSQLDataType(pilosa.FieldToFieldInfo(field))))
break
}
}
@ -50,7 +50,7 @@ func (p *ExecutionPlanner) compileInsertStatement(ctx context.Context, stmt *par
if strings.EqualFold("_exists", string(field.Name)) {
continue
}
targetColumns = append(targetColumns, newQualifiedRefPlanExpression(tableName, string(field.Name), idx, fieldSQLDataType(pilosa.FieldToFieldInfo(field))))
targetColumns = append(targetColumns, newQualifiedRefPlanExpression(tableName, string(field.Name), idx, FieldSQLDataType(pilosa.FieldToFieldInfo(field))))
}
}
@ -94,7 +94,7 @@ func (p *ExecutionPlanner) analyzeInsertStatement(ctx context.Context, stmt *par
if strings.EqualFold("_exists", string(field.Name)) {
continue
}
typeNames = append(typeNames, fieldSQLDataType(pilosa.FieldToFieldInfo(field)))
typeNames = append(typeNames, FieldSQLDataType(pilosa.FieldToFieldInfo(field)))
}
// Make sure (implicit) insert list and expression list have the same
// number of items.
@ -131,7 +131,7 @@ func (p *ExecutionPlanner) analyzeInsertStatement(ctx context.Context, stmt *par
columnFound := false
for _, field := range tbl.Fields {
if strings.EqualFold(colName, string(field.Name)) {
typeName = fieldSQLDataType(pilosa.FieldToFieldInfo(field))
typeName = FieldSQLDataType(pilosa.FieldToFieldInfo(field))
columnFound = true
break
}

2
sql3/planner/compileselect.go
View file

@ -588,7 +588,7 @@ func (p *ExecutionPlanner) analyzeSource(ctx context.Context, source parser.Sour
TableName: objectName,
ColumnName: string(fld.Name),
ColumnIndex: i,
Datatype: fieldSQLDataType(pilosa.FieldToFieldInfo(fld)),
Datatype: FieldSQLDataType(pilosa.FieldToFieldInfo(fld)),
}
source.OutputColumns = append(source.OutputColumns, soc)
}

21
sql3/planner/compileshow.go
View file

@ -94,9 +94,9 @@ func (p *ExecutionPlanner) compileShowTablesStatement(ctx context.Context, stmt
columns := []types.PlanExpression{
&qualifiedRefPlanExpression{
tableName: "fb_tables",
columnName: string(dax.PrimaryKeyFieldName),
columnName: "onject_id",
columnIndex: 0,
dataType: parser.NewDataTypeString(),
dataType: parser.NewDataTypeInt(),
},
&qualifiedRefPlanExpression{
tableName: "fb_tables",
@ -203,33 +203,38 @@ func (p *ExecutionPlanner) compileShowColumnsStatement(ctx context.Context, stmt
dataType: parser.NewDataTypeInt(),
}, &qualifiedRefPlanExpression{
tableName: "fb_table_columns",
columnName: "min",
columnName: "length",
columnIndex: 8,
dataType: parser.NewDataTypeInt(),
}, &qualifiedRefPlanExpression{
tableName: "fb_table_columns",
columnName: "max",
columnName: "min",
columnIndex: 9,
dataType: parser.NewDataTypeInt(),
}, &qualifiedRefPlanExpression{
tableName: "fb_table_columns",
columnName: "timeunit",
columnName: "max",
columnIndex: 10,
dataType: parser.NewDataTypeInt(),
}, &qualifiedRefPlanExpression{
tableName: "fb_table_columns",
columnName: "timeunit",
columnIndex: 11,
dataType: parser.NewDataTypeString(),
}, &qualifiedRefPlanExpression{
tableName: "fb_table_columns",
columnName: "epoch",
columnIndex: 11,
columnIndex: 12,
dataType: parser.NewDataTypeInt(),
}, &qualifiedRefPlanExpression{
tableName: "fb_table_columns",
columnName: "timequantum",
columnIndex: 12,
columnIndex: 13,
dataType: parser.NewDataTypeString(),
}, &qualifiedRefPlanExpression{
tableName: "fb_table_columns",
columnName: "ttl",
columnIndex: 13,
columnIndex: 14,
dataType: parser.NewDataTypeString(),
}}

22
sql3/planner/expressiontypes.go
View file

@ -13,7 +13,7 @@ import (
)
// takes a *pilosa.FieldInfo and returns a sql data type
func fieldSQLDataType(f *pilosa.FieldInfo) parser.ExprDataType {
func FieldSQLDataType(f *pilosa.FieldInfo) parser.ExprDataType {
// This is special handling for the primary key (_id) field. The normal
// handling below was not well suited to this field because there isn't a
// `pilosa.FieldTypeID` to compare against. One option would have been to
@ -68,6 +68,9 @@ func fieldSQLDataType(f *pilosa.FieldInfo) parser.ExprDataType {
case pilosa.FieldTypeTimestamp:
return parser.NewDataTypeTimestamp()
case pilosa.FieldTypeVarchar:
return parser.NewDataTypeVarchar(f.Options.Length)
default:
return parser.NewDataTypeVoid()
}
@ -81,10 +84,10 @@ func dataTypeFromParserType(typ *parser.Type) (parser.ExprDataType, error) {
return parser.NewDataTypeBool(), nil
case dax.BaseTypeDecimal:
if typ.Scale == nil {
if typ.Modifier == nil {
return nil, sql3.NewErrDecimalScaleExpected(typ.Name.NamePos.Line, typ.Name.NamePos.Column)
}
scale, err := strconv.Atoi(typ.Scale.Value)
scale, err := strconv.Atoi(typ.Modifier.Value)
if err != nil {
return nil, err
}
@ -358,6 +361,19 @@ func typesAreAssignmentCompatible(targetType parser.ExprDataType, sourceType par
case *parser.DataTypeString:
switch sourceType.(type) {
case *parser.DataTypeString:
return true
case *parser.DataTypeVarchar:
return true
default:
return false
}
case *parser.DataTypeVarchar:
switch rhs := sourceType.(type) {
case *parser.DataTypeVarchar:
//if lhs length is >= rhs length, we're good
return lhs.Length >= rhs.Length
case *parser.DataTypeString:
return true
default:

6
sql3/planner/opfeaturebasecolumns.go
View file

@ -87,6 +87,11 @@ func (p *PlanOpFeatureBaseColumns) Schema() types.Schema {
ColumnName: "scale",
Type: parser.NewDataTypeInt(),
},
&types.PlannerColumn{
RelationName: "fb_table_columns",
ColumnName: "length",
Type: parser.NewDataTypeInt(),
},
&types.PlannerColumn{
RelationName: "fb_table_columns",
ColumnName: "min",
@ -156,6 +161,7 @@ func (i *showColumnsRowIter) Next(ctx context.Context) (types.Row, error) {
fields[i.rowIndex].Options.CacheType,
fields[i.rowIndex].Options.CacheSize,
fields[i.rowIndex].Options.Scale,
fields[i.rowIndex].Options.Length,
fields[i.rowIndex].Options.Min.ToInt64(0),
fields[i.rowIndex].Options.Max.ToInt64(0),
fields[i.rowIndex].Options.TimeUnit,

5
sql3/planner/opfeaturebasetables.go
View file

@ -9,7 +9,6 @@ import (
"time"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
)
@ -55,8 +54,8 @@ func (p *PlanOpFeatureBaseTables) Schema() types.Schema {
return types.Schema{
&types.PlannerColumn{
RelationName: "fb_tables",
ColumnName: string(dax.PrimaryKeyFieldName),
Type: parser.NewDataTypeString(),
ColumnName: "object_id",
Type: parser.NewDataTypeInt(),
},
&types.PlannerColumn{
RelationName: "fb_tables",

8
sql3/planner/opinsert.go
View file

@ -421,6 +421,14 @@ func (i *insertRowIter) Next(ctx context.Context) (types.Row, error) {
return nil, sql3.NewErrInternalf("unsupported timestamp type: %T", eval)
}
case pilosa.FieldTypeVarchar:
switch v := eval.(type) {
case string:
row.Values[posVals[idx]] = v
default:
return nil, sql3.NewErrInternalf("unexpected varchar type '%T'", v)
}
default:
row.Values[posVals[idx]] = eval
}

4
sql3/planner/oppqldistinctscan.go
View file

@ -98,7 +98,7 @@ func (p *PlanOpPQLDistinctScan) Schema() types.Schema {
result = append(result, &types.PlannerColumn{
ColumnName: string(fld.Name),
RelationName: p.tableName,
Type: fieldSQLDataType(pilosa.FieldToFieldInfo(fld)),
Type: FieldSQLDataType(pilosa.FieldToFieldInfo(fld)),
})
break
}
@ -158,7 +158,7 @@ func (i *distinctScanRowIter) Next(ctx context.Context) (types.Row, error) {
for _, fld := range table.Fields {
if strings.EqualFold(i.column, string(fld.Name)) {
i.columnDataType = fieldSQLDataType(pilosa.FieldToFieldInfo(fld))
i.columnDataType = FieldSQLDataType(pilosa.FieldToFieldInfo(fld))
break
}
}

4
sql3/planner/oppqltablescan.go
View file

@ -109,7 +109,7 @@ func (p *PlanOpPQLTableScan) Schema() types.Schema {
result = append(result, &types.PlannerColumn{
ColumnName: string(fld.Name),
RelationName: p.tableName,
Type: fieldSQLDataType(pilosa.FieldToFieldInfo(fld)),
Type: FieldSQLDataType(pilosa.FieldToFieldInfo(fld)),
})
break
}
@ -198,7 +198,7 @@ func (i *tableScanRowIter) Next(ctx context.Context) (types.Row, error) {
columnIdx: idx,
srcColumnIdx: -1,
columnName: string(fld.Name),
dataType: fieldSQLDataType(pilosa.FieldToFieldInfo(fld)),
dataType: FieldSQLDataType(pilosa.FieldToFieldInfo(fld)),
}
break
}

3
sql3/planner/planoptimizer.go
View file

@ -28,6 +28,7 @@ type OptimizerFunc func(context.Context, *ExecutionPlanner, types.PlanOperator,
// a list of optimzer rules; order can be important important
var optimizerFunctions = []OptimizerFunc{
// fix expression references for having
removeUnusedExtractColumnReferences,
@ -1391,7 +1392,7 @@ func fixFieldRefIndexesForHaving(ctx context.Context, scope *OptimizerScope, a *
*percentilePlanExpression:
for i, col := range schema {
if strings.EqualFold(typedExpr.String(), col.ColumnName) {
e := newQualifiedRefPlanExpression("", "", i, typedExpr.Type())
e := newQualifiedRefPlanExpression("", col.ColumnName, i, typedExpr.Type())
return e, false, nil
}
}

1172
tstore/btree.go Normal file
View file

File diff suppressed because it is too large Load diff

160
tstore/btree_test.go Normal file
View file

@ -0,0 +1,160 @@
package tstore
import (
"fmt"
"math/rand"
"os"
"testing"
"github.com/featurebasedb/featurebase/v3/bufferpool"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
)
func TestAddItemsToBTreeAndValidate(t *testing.T) {
diskManager := bufferpool.NewOnDiskDiskManager()
objectId := int32(1)
shard := int32(0)
dataFile := fmt.Sprintf("ts-shard.%04d", shard)
os.Remove(dataFile)
diskManager.CreateOrOpenShard(objectId, shard, dataFile)
bufferPool := bufferpool.NewBufferPool(100, diskManager)
tableSchema := types.Schema{
&types.PlannerColumn{
ColumnName: "vtest",
Type: parser.NewDataTypeVarchar(50),
},
}
b, err := NewBTree(8, objectId, shard, tableSchema, bufferPool)
if err != nil {
t.Fatal(err)
}
rowSchema := types.Schema{
&types.PlannerColumn{
ColumnName: "_id",
Type: parser.NewDataTypeID(),
},
&types.PlannerColumn{
ColumnName: "vtest",
Type: parser.NewDataTypeVarchar(50),
},
}
inserts := make([]int, 0)
for i := 1; i <= 300; i++ {
inserts = append(inserts, i)
}
rand.Shuffle(len(inserts), func(i, j int) { inserts[i], inserts[j] = inserts[j], inserts[i] })
rr := make(types.Row, 2)
for _, i := range inserts {
rr[0] = int64(i)
rr[1] = fmt.Sprintf("This is a test of things %d", i)
tup := &BTreeTuple{
TupleSchema: rowSchema,
Tuple: rr,
}
// fmt.Printf("%v", tup)
err = b.Insert(tup)
if err != nil {
t.Fatal(err)
}
}
key, tuple := b.Search(nil, Int(33))
fmt.Printf("%v, %v\n\n", key, tuple)
b.Dump(0)
}
func TestAddItemsToBTreeAndValidate_VeryWide(t *testing.T) {
diskManager := bufferpool.NewOnDiskDiskManager()
objectId := int32(1)
shard := int32(0)
dataFile := fmt.Sprintf("ts-shard.%04d", shard)
os.Remove(dataFile)
diskManager.CreateOrOpenShard(objectId, shard, dataFile)
bufferPool := bufferpool.NewBufferPool(100, diskManager)
tableSchema := make(types.Schema, 0)
for i := 0; i < 3000; i++ {
tableSchema = append(tableSchema, &types.PlannerColumn{
ColumnName: fmt.Sprintf("vtest%d", i+1),
Type: parser.NewDataTypeVarchar(4),
})
}
b, err := NewBTree(8, objectId, shard, tableSchema, bufferPool)
if err != nil {
t.Fatal(err)
}
rowSchema := make(types.Schema, 0)
rowSchema = append(rowSchema, &types.PlannerColumn{
ColumnName: "_id",
Type: parser.NewDataTypeID(),
})
for i := 0; i < 3000; i++ {
rowSchema = append(rowSchema, &types.PlannerColumn{
ColumnName: fmt.Sprintf("vtest%d", i+1),
Type: parser.NewDataTypeVarchar(4),
})
}
inserts := make([]int, 0)
for i := 1; i <= 300; i++ {
inserts = append(inserts, i)
}
rand.Seed(10)
rand.Shuffle(len(inserts), func(i, j int) { inserts[i], inserts[j] = inserts[j], inserts[i] })
rr := make(types.Row, 3001)
for j, i := range inserts {
rr[0] = int64(i)
for j := 0; j < 3000; j++ {
rr[j+1] = fmt.Sprintf("%04d", j)
}
tup := &BTreeTuple{
TupleSchema: rowSchema,
Tuple: rr,
}
fmt.Printf("[%d]row key %v\n\n", j, i)
if j == 17 {
fmt.Printf("here\n")
}
err = b.Insert(tup)
if err != nil {
t.Fatal(err)
}
b.Dump(0)
fmt.Printf("\n\n--------------\n\n")
}
key, tuple := b.Search(nil, Int(33))
fmt.Printf("%v, %v\n\n", key, tuple)
b.Dump(0)
}

83
tstore/btreenode.go Normal file
View file

@ -0,0 +1,83 @@
// Copyright 2023 Molecula Corp. All rights reserved.
package tstore
import "github.com/featurebasedb/featurebase/v3/bufferpool"
type BTreeNode struct {
page *bufferpool.Page
}
func (n *BTreeNode) slotCount() int {
return int(n.page.ReadSlotCount())
}
func (n *BTreeNode) isLeaf() bool {
return n.page.ReadPageType() == bufferpool.PAGE_TYPE_BTREE_LEAF
}
func (n *BTreeNode) takeReadLatch() {
n.page.TakeReadLatch()
}
func (n *BTreeNode) releaseAnyLatch() {
n.page.ReleaseAnyLatch()
}
func (n *BTreeNode) releaseReadLatch() {
n.page.ReleaseReadLatch()
}
func (n *BTreeNode) takeWriteLatch() {
n.page.TakeWriteLatch()
}
func (n *BTreeNode) releaseWriteLatch() {
n.page.ReleaseWriteLatch()
}
func (n *BTreeNode) latchState() bufferpool.PageLatchState {
return n.page.LatchState()
}
func (n *BTreeNode) findKey(key Sortable) (int, bool) {
if n.latchState() == bufferpool.None {
panic("unexpected latch state")
}
minIndex := 0
onePastMaxIndex := int(n.page.ReadSlotCount())
for onePastMaxIndex != minIndex {
index := (minIndex + onePastMaxIndex) / 2
slot := n.page.ReadPageSlot(int16(index))
pl := slot.KeyPayload(n.page)
keyAtIndex := Int(pl.KeyAsInt(n.page))
if key.Equals(keyAtIndex) {
return index, true
}
if key.Less(keyAtIndex) {
onePastMaxIndex = index
} else {
minIndex = index + 1
}
}
return minIndex, false
}
func (n *BTreeNode) findNextPointer(key Sortable, objectID int32, shard int32) (bufferpool.PageID, error) {
if n.latchState() == bufferpool.None {
panic("unexpected latch state")
}
slotCount := int(n.page.ReadSlotCount())
keyPosition, _ := n.findKey(key)
if keyPosition == slotCount {
// return the next pointer...
return n.page.ReadNextPointer(), nil
}
// else return the pointer at the position returned from findKey
slot := n.page.ReadPageSlot(int16(keyPosition))
ipl := slot.InternalPayload(n.page)
return ipl.ValueAsPagePointer(n.page), nil
}

188
tstore/btreetypes.go Normal file
View file

@ -0,0 +1,188 @@
// Copyright 2023 Molecula Corp. All rights reserved.
package tstore
import (
"bytes"
"encoding/binary"
"github.com/pkg/errors"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
)
const (
KEY_SIZE_INT64 = 8
)
var ErrNeedsExclusive = errors.New("ErrNeedsExclusive")
type BTreeTuple struct {
TupleSchema types.Schema
Tuple types.Row
}
func (t *BTreeTuple) keyValue() Sortable {
kv, ok := t.Tuple[0].(int64)
if !ok {
return nil
}
return Int(kv)
}
func NewBTreeTupleFromBytes(b []byte, schema types.Schema) *BTreeTuple {
t := &BTreeTuple{
TupleSchema: schema,
Tuple: make(types.Row, len(schema)),
}
rdr := bytes.NewReader(b)
for i, s := range schema {
switch s.Type.(type) {
case *parser.DataTypeVarchar:
var l int32
binary.Read(rdr, binary.BigEndian, &l)
bvalue := make([]byte, l)
binary.Read(rdr, binary.BigEndian, &bvalue)
t.Tuple[i] = string(bvalue)
default:
panic("unexpected type")
}
}
return t
}
func (b *BTreeTuple) Bytes() ([]byte, error) {
var valueBuf bytes.Buffer
for cidx, c := range b.TupleSchema {
// skip the key column
if cidx == 0 {
continue
}
rd := b.Tuple[cidx]
switch ty := c.Type.(type) {
case *parser.DataTypeVarchar:
if rd == nil {
b := []byte{0, 0, 0, 0}
valueBuf.Write(b)
} else {
data, ok := rd.(string)
if !ok {
return []byte{}, errors.Errorf("unexpected type conversion '%T'", rd)
}
b := make([]byte, 4)
binary.BigEndian.PutUint32(b, uint32(len(data)))
valueBuf.Write(b)
valueBuf.WriteString(data)
}
default:
return []byte{}, errors.Errorf("unexpected type '%T'", ty)
}
}
return valueBuf.Bytes(), nil
}
type Equatable interface {
Equals(b Equatable) bool
}
type Sortable interface {
Equatable
Less(b Sortable) bool
Length() int
Bytes() []byte
}
type String string
func (s String) Equals(other Equatable) bool {
if o, ok := other.(String); ok {
return s == o
} else {
return false
}
}
func (s String) Less(other Sortable) bool {
if o, ok := other.(String); ok {
return s < o
} else {
return false
}
}
func (i String) Length() int {
return len(i)
}
func (i String) Bytes() []byte {
return []byte(i)
}
type ByteSlice []byte
// func (i ByteSlice) Hash() int {
// return int(i)
// }
// func (i ByteSlice) Less(other Sortable) bool {
// if o, ok := other.(Int); ok {
// return i < o
// } else {
// return false
// }
// }
// func (i ByteSlice) Equals(other Equatable) bool {
// if o, ok := other.(Int); ok {
// return i == o
// } else {
// return false
// }
// }
func (i ByteSlice) Length() int {
return len(i)
}
func (i ByteSlice) Bytes() []byte {
return i
}
func (i ByteSlice) AsInt() int {
return int(binary.BigEndian.Uint32(i))
}
type Int int
func (i Int) Hash() int {
return int(i)
}
func (i Int) Less(other Sortable) bool {
if o, ok := other.(Int); ok {
return i < o
} else {
return false
}
}
func (i Int) Equals(other Equatable) bool {
if o, ok := other.(Int); ok {
return i == o
} else {
return false
}
}
func (i Int) Length() int {
return 4
}
func (i Int) Bytes() []byte {
b := make([]byte, 4)
binary.BigEndian.PutUint32(b, uint32(i))
return b
}

57
wireprotocol/wireprimitives.go
View file

@ -36,6 +36,7 @@ const (
TYPE_IDSET int8 = 0x06
TYPE_STRING int8 = 0x07
TYPE_STRINGSET int8 = 0x08
TYPE_VARCHAR int8 = 0x09
)
func ExpectToken(reader io.Reader, token int16) (int16, error) {
@ -50,6 +51,15 @@ func ExpectToken(reader io.Reader, token int16) (int16, error) {
return tk, nil
}
func ReadToken(reader io.Reader) (int16, error) {
var tk int16
err := binary.Read(reader, binary.BigEndian, &tk)
if err != nil {
return 0, err
}
return tk, nil
}
// TOKEN_COLUMN_INFO message
// length (bytes)
// token 2
@ -109,6 +119,10 @@ func WriteSchema(schema types.Schema) ([]byte, error) {
case *parser.DataTypeStringSet:
writeInt8(writer, TYPE_STRINGSET)
case *parser.DataTypeVarchar:
writeInt8(writer, TYPE_VARCHAR)
writeInt32(writer, int32(ty.Length))
default:
return []byte{}, errors.Errorf("unexpected type '%T'", s.Type)
}
@ -179,6 +193,14 @@ func ReadSchema(reader io.Reader) (types.Schema, error) {
case TYPE_STRINGSET:
dataType = parser.NewDataTypeStringSet()
case TYPE_VARCHAR:
var length int32
err = binary.Read(reader, binary.BigEndian, &length)
if err != nil {
return nil, err
}
dataType = parser.NewDataTypeVarchar(int64(length))
}
schema = append(schema, &types.PlannerColumn{
@ -338,6 +360,18 @@ func WriteRow(row types.Row, schema types.Schema) ([]byte, error) {
}
}
case *parser.DataTypeVarchar:
if val == nil {
writeInt32(writer, 0)
} else {
v, ok := row[i].(string)
if !ok {
return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
}
writeInt32(writer, int32(len(v)))
writer.WriteString(v)
}
default:
return []byte{}, errors.Errorf("unexpected type '%T'", s.Type)
}
@ -483,6 +517,23 @@ func ReadRow(reader io.Reader, schema types.Schema) (types.Row, error) {
row[idx] = set
}
case *parser.DataTypeVarchar:
var len int32
err := binary.Read(reader, binary.BigEndian, &len)
if err != nil {
return nil, err
}
if len == 0 {
row[idx] = nil
} else {
bvalue := make([]byte, len)
err = binary.Read(reader, binary.BigEndian, &bvalue)
if err != nil {
return nil, err
}
row[idx] = string(bvalue)
}
default:
return nil, errors.Errorf("unexpected type '%T'", s.Type)
}
@ -558,6 +609,12 @@ func writeInt16(w io.Writer, i int16) {
w.Write(b)
}
func writeInt32(w io.Writer, i int32) {
b := make([]byte, 4)
binary.BigEndian.PutUint32(b, uint32(i))
w.Write(b)
}
func writeInt64(w io.Writer, i int64) {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(i))