From 12e00c79491d48b29efa4dd8fd0fa1d836620e14 Mon Sep 17 00:00:00 2001 From: tgruben Date: Wed, 5 Apr 2023 14:49:38 -0500 Subject: [PATCH] Tstore PQL and rangeIterator (#2368) * added varchar type; create table works; show columns works; wip on b+tree * added page compaction on inserts where page has no free space * fix bug with unpinning pages where depth of internal nodes > 1 * Now handling schema in it's own btree * fixed some bugs * null inserts, better tuple payload * refactor tstore holdings * refactor tstore management * seekable read iterator compiles * tstore rangeiteraor Tstore pql command * tstore rangeiteraor Tstore pql command * encoding/decoding support for TupleResults(Tstore) * simplified range iterator * add column filter for tstore * bug fix --------- Co-authored-by: pokeeffe-molecula --- api.go | 38 +- encoding/proto/proto.go | 25 ++ executor.go | 7 + index.go | 47 +++ metrics.go | 12 +- pb/public.pb.go | 630 +++++++++++++++++++++++----------- pb/public.proto | 4 + pql/ast.go | 6 + sql3/planner/opsystemtable.go | 1 + tstore.go | 265 ++++++++++++++ tstore/debug_utils.go | 248 +++++++++++++ tstore/debug_utils_test.go | 209 +++++++++++ tstore/range.go | 119 +++++++ 13 files changed, 1385 insertions(+), 226 deletions(-) create mode 100644 tstore.go create mode 100644 tstore/debug_utils.go create mode 100644 tstore/debug_utils_test.go create mode 100644 tstore/range.go diff --git a/api.go b/api.go index fb764b676..f357bf757 100644 --- a/api.go +++ b/api.go @@ -15,7 +15,6 @@ import ( "math" "net/url" "os" - "path/filepath" "runtime" "sort" "strconv" @@ -36,7 +35,6 @@ import ( "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" - "github.com/featurebasedb/featurebase/v3/sql3/parser" planner_types "github.com/featurebasedb/featurebase/v3/sql3/planner/types" "github.com/featurebasedb/featurebase/v3/tracing" "github.com/pkg/errors" @@ -1804,7 +1802,6 @@ func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error { } func (api *API) importTuples(ctx context.Context, tx Tx, shard uint64, tableName string, tupleData []byte) error { - // get the table index, err := api.Index(ctx, tableName) if err != nil { @@ -1815,36 +1812,8 @@ func (api *API) importTuples(ctx context.Context, tx Tx, shard uint64, tableName if index.ID == 0 { return errors.Errorf("cannot insert into table '%s' because it does not have a non-zero object id", tableName) } - basePath := index.TStorePath() - // open or create btreefile for this shard - dataFile := filepath.Join(basePath, fmt.Sprintf("ts-shard.%04d", shard)) - api.holder.tstoredisk.CreateOrOpenShard(index.ID, int32(shard), dataFile) - - // get the schema from the FeatureBase table in the form of a planner_types.Schema - // we just want the t-store types - - fieldList := make([]*Field, 0) - for _, f := range index.fields { - if strings.EqualFold(f.options.Type, FieldTypeVarchar) { - fieldList = append(fieldList, f) - } - } - - sort.Slice(fieldList, func(i, j int) bool { - return fieldList[i].CreatedAt() < fieldList[j].CreatedAt() - }) - - indexSchema := make(planner_types.Schema, len(fieldList)) - for i, f := range fieldList { - indexSchema[i] = &planner_types.PlannerColumn{ - ColumnName: f.name, - Type: parser.NewDataTypeVarchar(f.options.Length), - } - } - - // create the b-tree we're going to use - b, err := tstore.NewBTree(tstore.KEY_SIZE_INT64, index.ID, int32(shard), indexSchema, api.holder.tstorepool) + b, err := index.GetTStore(shard) if err != nil { return err } @@ -3461,18 +3430,23 @@ func (n *NopSchemaAPI) DropDatabase(context.Context, dax.DatabaseID) error { re func (n *NopSchemaAPI) DatabaseByName(ctx context.Context, dbname dax.DatabaseName) (*dax.Database, error) { return nil, nil } + func (n *NopSchemaAPI) DatabaseByID(ctx context.Context, dbid dax.DatabaseID) (*dax.Database, error) { return nil, nil } + func (n *NopSchemaAPI) SetDatabaseOption(ctx context.Context, dbid dax.DatabaseID, option string, value string) error { return nil } + func (n *NopSchemaAPI) Databases(context.Context, ...dax.DatabaseID) ([]*dax.Database, error) { return nil, nil } + func (n *NopSchemaAPI) TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error) { return nil, nil } + func (n *NopSchemaAPI) TableByID(ctx context.Context, tid dax.TableID) (*dax.Table, error) { return nil, nil } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 7ab5742d9..01ab7399f 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -561,6 +561,9 @@ func (s Serializer) encodeQueryResponse(m *pilosa.QueryResponse) *pb.QueryRespon case pilosa.ExtractedIDMatrixSorted: resp.Results[i].Type = queryResultTypeExtractedIDMatrixSorted resp.Results[i].ExtractedIDMatrixSorted = s.endcodeExtractedIDMatrixSorted(result) + case *pilosa.TupleResults: + resp.Results[i].Type = queryResultTypeTupleResults + resp.Results[i].TupleResults = s.encodeTupleResults(result) default: panic(fmt.Errorf("unknown type: %T", m.Results[i])) } @@ -1351,6 +1354,7 @@ const ( queryResultTypeDataFrame queryResultTypeArrowTable queryResultTypeExtractedIDMatrixSorted + queryResultTypeTupleResults ) func (s Serializer) decodeQueryResult(pb *pb.QueryResult) interface{} { @@ -1395,6 +1399,8 @@ func (s Serializer) decodeQueryResult(pb *pb.QueryResult) interface{} { return s.decodeArrowTable(pb.ArrowTable) case queryResultTypeExtractedIDMatrixSorted: return s.decodeExtractedIDMatrixSorted(pb.ExtractedIDMatrixSorted) + case queryResultTypeTupleResults: + return s.decodeTupleResults(pb.TupleResults) } panic(fmt.Sprintf("unknown type: %d", pb.Type)) } @@ -1991,3 +1997,22 @@ func (s Serializer) decodeRowKVs(m []*pb.RowKV) []pilosa.RowKV { } return rows } + +func (s Serializer) decodeTupleResults(tr *pb.TupleResults) *pilosa.TupleResults { + r, _ := pilosa.NewTupleResultFromBytes(tr.Data) + return r +} + +func (s Serializer) encodeTupleResults(tr *pilosa.TupleResults) *pb.TupleResults { + if tr == nil { + return &pb.TupleResults{} // Generated proto code doesn't like a nil Row. + } + buff, err := tr.ToBytes() + if err != nil { + panic(err) + } + // ugh hate having to swallow error here + return &pb.TupleResults{ + Data: buff, + } +} diff --git a/executor.go b/executor.go index 2f59d9436..7ecbbbfe3 100644 --- a/executor.go +++ b/executor.go @@ -352,6 +352,9 @@ func safeCopy(resp QueryResponse) (out QueryResponse) { out.Results = append(out.Results, x) case ExtractedIDMatrixSorted: out.Results = append(out.Results, x) + case *TupleResults: + // dumpTable(x) + out.Results = append(out.Results, x) default: panic(fmt.Sprintf("handle %T here", v)) } @@ -836,6 +839,10 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p statFn(CounterQueryArrowTotal) res, err := e.executeArrow(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeArrow") + case "Tstore": + statFn(CounterQueryTstoreTotal) + res, err := e.executeTstore(ctx, qcx, index, c, shards, opt) + return res, errors.Wrap(err, "executeTstore") default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap. res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeBitmapCall") diff --git a/index.go b/index.go index 57732a2a8..99496720f 100644 --- a/index.go +++ b/index.go @@ -18,7 +18,10 @@ import ( "github.com/featurebasedb/featurebase/v3/disco" "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + planner_types "github.com/featurebasedb/featurebase/v3/sql3/planner/types" "github.com/featurebasedb/featurebase/v3/testhook" + "github.com/featurebasedb/featurebase/v3/tstore" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" @@ -64,6 +67,8 @@ type Index struct { // indicate that we're closing and should wrap up and not allow new actions closing chan struct{} + + tstores map[uint64]*tstore.BTree } // NewIndex returns an existing (but possibly empty) instance of @@ -94,6 +99,7 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) { translationSyncer: NopTranslationSyncer, OpenTranslateStore: OpenInMemTranslateStore, + tstores: make(map[uint64]*tstore.BTree), } return idx, nil } @@ -1097,6 +1103,47 @@ func (i *Index) GetDataFramePath(shard uint64) string { return filepath.Join(path, shardpad) } +func (i *Index) GetTStore(shard uint64) (*tstore.BTree, error) { + b, ok := i.tstores[shard] + if ok { + return b, nil + } + basePath := i.TStorePath() + // open or create btreefile for this shard + dataFile := filepath.Join(basePath, fmt.Sprintf("ts-shard.%04d", shard)) + i.holder.tstoredisk.CreateOrOpenShard(i.ID, int32(shard), dataFile) + + // get the schema from the FeatureBase table in the form of a planner_types.Schema + // we just want the t-store types + + fieldList := make([]*Field, 0) + for _, f := range i.fields { + if strings.EqualFold(f.options.Type, FieldTypeVarchar) { + fieldList = append(fieldList, f) + } + } + + sort.Slice(fieldList, func(i, j int) bool { + return fieldList[i].CreatedAt() < fieldList[j].CreatedAt() + }) + + indexSchema := make(planner_types.Schema, len(fieldList)) + for i, f := range fieldList { + indexSchema[i] = &planner_types.PlannerColumn{ + ColumnName: f.name, + Type: parser.NewDataTypeVarchar(f.options.Length), + } + } + + // create the b-tree we're going to use + b, err := tstore.NewBTree(tstore.KEY_SIZE_INT64, i.ID, int32(shard), indexSchema, i.holder.tstorepool) + if err != nil { + return nil, err + } + i.tstores[shard] = b + return b, nil +} + type indexSlice []*Index func (p indexSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/metrics.go b/metrics.go index c529fc283..35984f978 100644 --- a/metrics.go +++ b/metrics.go @@ -807,6 +807,17 @@ var CounterQueryArrowTotal = prometheus.NewCounterVec( }, ) +var CounterQueryTstoreTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_tstore_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + // CounterQueryBitmapTotal represents bitmap calls. var CounterQueryBitmapTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -1048,5 +1059,4 @@ func init() { // index related prometheus.MustRegister(GaugeIndexMaxShard) - } diff --git a/pb/public.pb.go b/pb/public.pb.go index bf6af62af..939433cb3 100644 --- a/pb/public.pb.go +++ b/pb/public.pb.go @@ -575,7 +575,6 @@ func (m *KeyList) GetKeys() []string { type ExtractedTableValue struct { // Types that are valid to be assigned to Value: - // // *ExtractedTableValue_IDs // *ExtractedTableValue_Keys // *ExtractedTableValue_BSIValue @@ -716,7 +715,6 @@ func (*ExtractedTableValue) XXX_OneofWrappers() []interface{} { type ExtractedTableColumn struct { // Types that are valid to be assigned to KeyOrID: - // // *ExtractedTableColumn_Key // *ExtractedTableColumn_ID KeyOrID isExtractedTableColumn_KeyOrID `protobuf_oneof:"KeyOrID"` @@ -1665,6 +1663,7 @@ type QueryResult struct { DataFrame *DataFrame `protobuf:"bytes,18,opt,name=DataFrame,proto3" json:"DataFrame,omitempty"` ArrowTable *ArrowTable `protobuf:"bytes,19,opt,name=ArrowTable,proto3" json:"ArrowTable,omitempty"` ExtractedIDMatrixSorted *ExtractedIDMatrixSorted `protobuf:"bytes,20,opt,name=ExtractedIDMatrixSorted,proto3" json:"ExtractedIDMatrixSorted,omitempty"` + TupleResults *TupleResults `protobuf:"bytes,21,opt,name=TupleResults,proto3" json:"TupleResults,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1843,6 +1842,13 @@ func (m *QueryResult) GetExtractedIDMatrixSorted() *ExtractedIDMatrixSorted { return nil } +func (m *QueryResult) GetTupleResults() *TupleResults { + if m != nil { + return m.TupleResults + } + return nil +} + type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -2876,6 +2882,53 @@ func (m *ArrowTable) GetData() []byte { return nil } +type TupleResults struct { + Data []byte `protobuf:"bytes,1,opt,name=Data,proto3" json:"Data,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TupleResults) Reset() { *m = TupleResults{} } +func (m *TupleResults) String() string { return proto.CompactTextString(m) } +func (*TupleResults) ProtoMessage() {} +func (*TupleResults) Descriptor() ([]byte, []int) { + return fileDescriptor_413a91106d7bcce8, []int{41} +} +func (m *TupleResults) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *TupleResults) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_TupleResults.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *TupleResults) XXX_Merge(src proto.Message) { + xxx_messageInfo_TupleResults.Merge(m, src) +} +func (m *TupleResults) XXX_Size() int { + return m.Size() +} +func (m *TupleResults) XXX_DiscardUnknown() { + xxx_messageInfo_TupleResults.DiscardUnknown(m) +} + +var xxx_messageInfo_TupleResults proto.InternalMessageInfo + +func (m *TupleResults) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + func init() { proto.RegisterType((*Row)(nil), "pb.Row") proto.RegisterType((*RowMatrix)(nil), "pb.RowMatrix") @@ -2918,128 +2971,130 @@ func init() { proto.RegisterType((*GroupCounts)(nil), "pb.GroupCounts") proto.RegisterType((*DataFrame)(nil), "pb.DataFrame") proto.RegisterType((*ArrowTable)(nil), "pb.ArrowTable") + proto.RegisterType((*TupleResults)(nil), "pb.TupleResults") } func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1844 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x18, 0x4d, 0x73, 0x23, 0x47, - 0xd5, 0xa3, 0xd1, 0xe7, 0x93, 0xec, 0xb5, 0x7b, 0x9d, 0xcd, 0x64, 0xe3, 0x38, 0xda, 0x29, 0x2a, - 0x28, 0x18, 0x36, 0x85, 0xa1, 0x52, 0x54, 0xaa, 0x20, 0x65, 0xaf, 0xbc, 0xac, 0xca, 0xbb, 0xce, - 0xd2, 0xda, 0x88, 0x4b, 0x2e, 0x63, 0xa9, 0x51, 0xa6, 0x18, 0x69, 0xc4, 0xcc, 0x28, 0xb2, 0x8f, - 0x1c, 0x28, 0xf8, 0x09, 0x9c, 0xe0, 0xd7, 0x50, 0x70, 0x83, 0x23, 0x47, 0x6a, 0xf9, 0x23, 0xd4, - 0x7b, 0xaf, 0x7b, 0xa6, 0x47, 0x92, 0xb7, 0xc2, 0x56, 0x6e, 0xfd, 0x3e, 0xfa, 0xf5, 0xfb, 0x7e, - 0x6f, 0x06, 0x3a, 0x8b, 0xe5, 0x75, 0x14, 0x8e, 0x1f, 0x2f, 0x92, 0x38, 0x8b, 0x45, 0x65, 0x71, - 0xed, 0xdf, 0x82, 0x2b, 0xe3, 0x95, 0xf0, 0xa0, 0xf1, 0x24, 0x8e, 0x96, 0xb3, 0x79, 0xea, 0x39, - 0x5d, 0xb7, 0x57, 0x95, 0x06, 0x14, 0x02, 0xaa, 0x97, 0xea, 0x36, 0xf5, 0xdc, 0xae, 0xdb, 0x6b, - 0x49, 0x3a, 0x23, 0xb7, 0x8c, 0x83, 0x24, 0x9c, 0x4f, 0xbd, 0x6a, 0xd7, 0xe9, 0x75, 0xa4, 0x01, - 0xc5, 0x21, 0xd4, 0x06, 0xf3, 0x89, 0xba, 0xf1, 0x6a, 0x5d, 0xa7, 0xd7, 0x92, 0x0c, 0x20, 0xf6, - 0x69, 0xa8, 0xa2, 0x89, 0x57, 0x67, 0x2c, 0x01, 0x7e, 0x0f, 0x5a, 0x32, 0x5e, 0xbd, 0x08, 0xb2, - 0x24, 0xbc, 0x11, 0xef, 0x43, 0x55, 0xc6, 0x2b, 0x7e, 0xbd, 0x7d, 0xda, 0x78, 0xbc, 0xb8, 0x7e, - 0x2c, 0xe3, 0x95, 0x24, 0xa4, 0x7f, 0x06, 0xad, 0x61, 0x38, 0x9d, 0xab, 0x09, 0xaa, 0xfa, 0x1e, - 0xb8, 0x2f, 0x63, 0x64, 0x74, 0x6c, 0x46, 0xc4, 0x21, 0xe9, 0x4a, 0x4d, 0xbd, 0xca, 0x1a, 0xe9, - 0x4a, 0x4d, 0xfd, 0x9f, 0xc1, 0x9e, 0x8c, 0x57, 0x83, 0x89, 0x9a, 0x67, 0xe1, 0x6f, 0x42, 0x95, - 0x90, 0x61, 0xf9, 0x8b, 0x55, 0x7e, 0x28, 0x37, 0xb6, 0x52, 0x18, 0xeb, 0x3f, 0x84, 0xfa, 0xa0, - 0xff, 0x3c, 0x4c, 0x33, 0xb1, 0x0f, 0xee, 0xa0, 0x6f, 0x2e, 0xe0, 0xd1, 0x7f, 0x02, 0x07, 0x17, - 0x37, 0x59, 0x12, 0x8c, 0x33, 0x35, 0x19, 0xf4, 0xd9, 0x65, 0x62, 0x0f, 0x2a, 0x83, 0x3e, 0xe9, - 0x57, 0x95, 0x95, 0x41, 0x5f, 0x1c, 0x43, 0x75, 0x14, 0x44, 0x2c, 0xb4, 0x7d, 0x0a, 0xa8, 0x16, - 0x0b, 0x94, 0x84, 0xf7, 0x7f, 0xef, 0xc0, 0xbb, 0x96, 0x14, 0x76, 0xc8, 0x30, 0x4e, 0x32, 0x35, - 0x11, 0xe5, 0x07, 0x98, 0xa4, 0x4d, 0x7f, 0x07, 0x05, 0x6d, 0x10, 0xe5, 0x26, 0xbf, 0x78, 0x04, - 0x75, 0x19, 0xaf, 0x2e, 0x47, 0x46, 0x85, 0x96, 0xf6, 0xcc, 0xe5, 0x48, 0x6a, 0x82, 0xff, 0x1c, - 0x6a, 0x74, 0xc2, 0x50, 0xa1, 0x9f, 0x8c, 0xfe, 0x0c, 0x88, 0x1f, 0x41, 0x6d, 0x14, 0x44, 0x4b, - 0xa5, 0x5d, 0xfb, 0x6e, 0xe9, 0xe9, 0x57, 0xc1, 0x75, 0xa4, 0x88, 0x2c, 0x99, 0xcb, 0xff, 0x6a, - 0x8b, 0xd6, 0xe2, 0x01, 0xd4, 0x29, 0xee, 0xec, 0xc0, 0x96, 0xd4, 0x90, 0xf8, 0xa4, 0x48, 0x3d, - 0x56, 0x6f, 0xdd, 0x30, 0xa6, 0xe6, 0x19, 0xe9, 0x7f, 0x00, 0x8d, 0x4b, 0x75, 0x4b, 0x11, 0x31, - 0xf1, 0x72, 0xac, 0x78, 0xfd, 0xd3, 0x81, 0xfb, 0x5b, 0x74, 0x13, 0xc7, 0x26, 0x7a, 0x4e, 0x39, - 0x0a, 0xcf, 0x76, 0x28, 0x96, 0xe2, 0x51, 0x1e, 0x7b, 0x64, 0x68, 0x23, 0x83, 0x7e, 0xe6, 0xd9, - 0x8e, 0xce, 0xfb, 0x23, 0x68, 0x9e, 0x0f, 0x07, 0xec, 0x09, 0xb7, 0xeb, 0xf4, 0xdc, 0x67, 0x3b, - 0x32, 0xc7, 0x88, 0x87, 0xd0, 0x78, 0xb1, 0xcc, 0xd4, 0xcd, 0xa0, 0x4f, 0x55, 0x51, 0x7d, 0xb6, - 0x23, 0x0d, 0x02, 0x6f, 0xd2, 0xf1, 0x52, 0xdd, 0x72, 0x69, 0xe0, 0x4d, 0x83, 0x11, 0x87, 0x50, - 0x3d, 0x8f, 0xe3, 0x88, 0xca, 0xa3, 0x89, 0xaf, 0x21, 0x74, 0xde, 0xd0, 0x4e, 0xf7, 0x6f, 0xe0, - 0xb0, 0x6c, 0x90, 0x4e, 0x34, 0x01, 0x2e, 0xca, 0x73, 0xb4, 0x3c, 0x04, 0xc4, 0x3e, 0x25, 0x5f, - 0x45, 0xbf, 0x8f, 0xe9, 0xf7, 0x09, 0xd4, 0x49, 0x0c, 0x97, 0xf0, 0x1b, 0x82, 0xa7, 0xd9, 0xce, - 0x5b, 0xe4, 0xdf, 0x2f, 0x92, 0x41, 0xdf, 0xff, 0xf9, 0xba, 0x2b, 0x29, 0x66, 0xe8, 0xf6, 0xab, - 0x60, 0xa6, 0xf8, 0x65, 0x49, 0x67, 0xc4, 0xbd, 0xba, 0x5d, 0x70, 0x86, 0xb4, 0x24, 0x9d, 0xfd, - 0x25, 0xec, 0x95, 0xaf, 0xa3, 0x32, 0x56, 0x12, 0x6c, 0x55, 0x86, 0xe8, 0x79, 0x76, 0x9c, 0xae, - 0x67, 0x87, 0xb7, 0x79, 0x63, 0x3d, 0x41, 0x7e, 0x01, 0xd5, 0x97, 0x41, 0x98, 0x6c, 0x14, 0xe2, - 0x3e, 0xfb, 0xcb, 0x25, 0x0d, 0x5d, 0x76, 0x7c, 0xed, 0x49, 0xbc, 0x9c, 0x67, 0xec, 0x30, 0xc9, - 0x80, 0xff, 0x39, 0xb4, 0xf0, 0x3e, 0xdb, 0x7a, 0xc4, 0xc2, 0x74, 0xde, 0x34, 0xf1, 0x75, 0x84, - 0x25, 0x3f, 0x91, 0x77, 0xb6, 0x8a, 0xdd, 0xd9, 0xce, 0x01, 0x90, 0x9a, 0xb2, 0x84, 0x63, 0xa8, - 0x11, 0xa4, 0x4d, 0x2e, 0x44, 0x30, 0xfa, 0x0e, 0x19, 0x1f, 0x60, 0x27, 0xcd, 0x3e, 0xfd, 0x29, - 0x92, 0x39, 0xe3, 0x50, 0x03, 0xd7, 0x94, 0x58, 0x0c, 0x4d, 0x76, 0x54, 0xbc, 0x2a, 0x04, 0x38, - 0x96, 0x80, 0xa2, 0x92, 0x2b, 0x76, 0x25, 0x3f, 0xe0, 0x5e, 0x90, 0xbb, 0x41, 0x43, 0xe2, 0x43, - 0xf3, 0x4a, 0x95, 0xec, 0xa4, 0x16, 0x41, 0xef, 0x9b, 0x07, 0xff, 0xe0, 0x00, 0xfc, 0x32, 0x89, - 0x97, 0x0b, 0xf2, 0x91, 0xf0, 0xa1, 0x46, 0x90, 0x36, 0xaa, 0x83, 0xfc, 0x46, 0x21, 0xc9, 0xa4, - 0xed, 0xde, 0xc5, 0x28, 0x9c, 0x4d, 0xa7, 0x5c, 0x3f, 0x12, 0x8f, 0xe2, 0x04, 0xa0, 0xaf, 0xc6, - 0xe1, 0x2c, 0x88, 0x90, 0x50, 0x2d, 0xea, 0x4f, 0x63, 0xa5, 0x45, 0xf6, 0xff, 0xea, 0x40, 0x73, - 0x14, 0x44, 0xb9, 0xac, 0x51, 0x10, 0x69, 0xcf, 0xe0, 0xb1, 0xfc, 0xa6, 0x6b, 0xde, 0x7c, 0x08, - 0xcd, 0xa7, 0x51, 0x1c, 0x64, 0xc8, 0x8c, 0x0f, 0x3b, 0x32, 0x87, 0xad, 0xd7, 0x91, 0xfa, 0x86, - 0xd7, 0x91, 0xd9, 0x87, 0xce, 0xab, 0x70, 0xa6, 0xd2, 0x2c, 0x98, 0x2d, 0x90, 0x9d, 0xc7, 0x5c, - 0x09, 0x87, 0x9e, 0x6a, 0xe8, 0x2b, 0xdb, 0x83, 0x87, 0xd8, 0xe1, 0x38, 0x88, 0x94, 0x51, 0x92, - 0x00, 0x71, 0x0c, 0x70, 0xa5, 0x56, 0x23, 0x95, 0xa4, 0x61, 0x3c, 0x27, 0x35, 0x9b, 0xd2, 0xc2, - 0x60, 0xe8, 0x46, 0x41, 0x74, 0x76, 0x9d, 0xea, 0xa1, 0xab, 0x21, 0x8d, 0xc7, 0xc1, 0x57, 0xa3, - 0x3b, 0x1a, 0xf2, 0x3f, 0x87, 0x83, 0x7e, 0x98, 0x66, 0xe1, 0x7c, 0x9c, 0xe5, 0xfa, 0x69, 0x66, - 0xec, 0x06, 0xba, 0x0b, 0x33, 0x94, 0x97, 0x74, 0xa5, 0x28, 0x69, 0xff, 0x6f, 0x0e, 0x74, 0x7e, - 0xb5, 0x54, 0xc9, 0xad, 0x54, 0xbf, 0x5b, 0xaa, 0x34, 0x43, 0xbd, 0x09, 0x36, 0x89, 0x46, 0x00, - 0x8a, 0x1c, 0x7e, 0x1d, 0x24, 0x13, 0xae, 0xd0, 0xaa, 0xd4, 0x10, 0xa5, 0x9a, 0x9a, 0xc5, 0x99, - 0x32, 0x7a, 0x31, 0x24, 0x4e, 0xa0, 0x73, 0x31, 0xbb, 0x56, 0x93, 0x89, 0x9a, 0xf4, 0x83, 0x2c, - 0xf0, 0x9a, 0xe5, 0x91, 0x5f, 0x22, 0x8a, 0xef, 0xc1, 0xee, 0xcb, 0x44, 0xbd, 0x4a, 0x82, 0x79, - 0x1a, 0x05, 0x99, 0x9a, 0x78, 0x2d, 0x92, 0x55, 0x46, 0x8a, 0x23, 0x68, 0xbd, 0x08, 0x6e, 0x5e, - 0xa8, 0x59, 0x9c, 0xdc, 0x7a, 0x40, 0x4e, 0x2d, 0x10, 0xfe, 0x73, 0xd8, 0xd5, 0x66, 0xa4, 0x8b, - 0x78, 0x9e, 0x2a, 0x4c, 0x9b, 0x8b, 0x24, 0xd1, 0x56, 0xe0, 0x51, 0x7c, 0x0c, 0x0d, 0xa9, 0xd2, - 0x65, 0x94, 0x99, 0x36, 0x73, 0x0f, 0xd5, 0x31, 0xb7, 0x96, 0x51, 0x26, 0x0d, 0xdd, 0xff, 0x4b, - 0x03, 0xda, 0x16, 0x21, 0x6f, 0x7c, 0xd8, 0xbc, 0x77, 0xb9, 0xf1, 0xe1, 0x22, 0x22, 0xe3, 0xd5, - 0xc6, 0x8e, 0x82, 0xc5, 0xda, 0x01, 0xe7, 0x4a, 0x17, 0x84, 0x73, 0x55, 0xf4, 0x06, 0x77, 0x7b, - 0x6f, 0xc0, 0xbd, 0xec, 0xeb, 0x60, 0x3e, 0x55, 0x13, 0x0a, 0x7a, 0x53, 0x1a, 0x50, 0xf4, 0x8a, - 0x32, 0x20, 0xff, 0xea, 0x1a, 0x34, 0x38, 0x59, 0x14, 0x09, 0x97, 0x3c, 0xce, 0xbe, 0x06, 0xc7, - 0x87, 0x21, 0xf1, 0x29, 0xec, 0x7d, 0x11, 0x4d, 0x8a, 0x9a, 0x4e, 0x75, 0x24, 0xf6, 0x50, 0x4e, - 0x81, 0x96, 0x6b, 0x5c, 0xe2, 0xb3, 0xf5, 0x55, 0x8a, 0x62, 0xd2, 0x3e, 0x15, 0xda, 0x4e, 0x8b, - 0x22, 0xd7, 0x97, 0xae, 0x13, 0x6b, 0x93, 0xa3, 0x40, 0xb5, 0x4f, 0x77, 0xf1, 0x5a, 0x8e, 0x94, - 0xd6, 0xa6, 0xf7, 0xd8, 0x6e, 0xa3, 0x5e, 0x9b, 0xb8, 0xf7, 0x8c, 0x87, 0x18, 0x2b, 0xed, 0x46, - 0x7b, 0x62, 0xf5, 0x6d, 0xaf, 0x53, 0x08, 0xcf, 0x91, 0xd2, 0xea, 0xeb, 0x5b, 0x37, 0xab, 0xdd, - 0xff, 0x73, 0xb3, 0xfa, 0x6c, 0x7d, 0xc0, 0x79, 0x7b, 0x85, 0x2b, 0xca, 0x14, 0xb9, 0x3e, 0x0a, - 0x4f, 0xac, 0xf5, 0xd7, 0xbb, 0x57, 0x68, 0x9b, 0x23, 0xa5, 0xb5, 0x1e, 0xff, 0x18, 0xda, 0x76, - 0xa0, 0xf6, 0x89, 0xfd, 0x5e, 0x39, 0x50, 0xa9, 0xb4, 0x79, 0xd0, 0xc0, 0x8d, 0xf2, 0xf7, 0x0e, - 0x0a, 0x03, 0x37, 0x88, 0x72, 0x4b, 0xbb, 0x38, 0x81, 0x16, 0x96, 0xe1, 0xd3, 0x04, 0x7b, 0x83, - 0x28, 0x94, 0xcc, 0x91, 0xb2, 0xa0, 0x63, 0xbc, 0xce, 0x92, 0x24, 0x5e, 0xb1, 0x27, 0xee, 0x17, - 0xf1, 0x2a, 0xb0, 0xd2, 0xe2, 0x10, 0x5f, 0xde, 0xb9, 0xf7, 0x7a, 0x87, 0x74, 0xf9, 0xfd, 0xad, - 0x81, 0x60, 0x16, 0x79, 0xd7, 0x5d, 0xff, 0xef, 0x15, 0xd8, 0x1d, 0xcc, 0x16, 0x71, 0x92, 0x59, - 0x7d, 0x8b, 0xbf, 0x4a, 0x9c, 0xad, 0x5f, 0x25, 0x95, 0xb5, 0xb1, 0x49, 0xfd, 0x8b, 0x1a, 0x70, - 0x55, 0x32, 0x60, 0xd5, 0x50, 0xb5, 0x54, 0x43, 0x47, 0xd0, 0xe2, 0xad, 0x03, 0x49, 0x35, 0x22, - 0x15, 0x08, 0xfe, 0x4e, 0x5a, 0xd1, 0x56, 0xd9, 0xa0, 0x6e, 0x6b, 0x40, 0xec, 0xf5, 0xcc, 0x46, - 0xc4, 0x26, 0x11, 0x2d, 0x0c, 0xd2, 0xf3, 0x20, 0xa4, 0x5e, 0xbd, 0xeb, 0xf6, 0x5c, 0x69, 0x61, - 0xc4, 0x47, 0xb0, 0x47, 0x46, 0x3c, 0x49, 0x14, 0x36, 0xc0, 0xb3, 0x8c, 0x6a, 0xd0, 0x95, 0x6b, - 0x58, 0xe4, 0x23, 0xb3, 0x0a, 0x3e, 0xee, 0x8e, 0x6b, 0x58, 0x1a, 0x9b, 0x91, 0x0a, 0x12, 0xaa, - 0xb2, 0xa6, 0x64, 0xc0, 0xff, 0x77, 0x05, 0x04, 0x7b, 0x92, 0x37, 0xc4, 0xef, 0xcc, 0x9d, 0x6f, - 0x76, 0x5b, 0xd9, 0x39, 0x8d, 0x0d, 0xe7, 0x14, 0x33, 0x8c, 0x1d, 0x63, 0x66, 0x58, 0x17, 0xda, - 0x66, 0xaa, 0x23, 0x11, 0xbd, 0xea, 0x48, 0x1b, 0x85, 0xe3, 0x7b, 0x98, 0xe1, 0x87, 0xaa, 0x66, - 0x69, 0x91, 0xec, 0x12, 0x6e, 0x8b, 0x6b, 0xe1, 0x5b, 0xba, 0xb6, 0xfd, 0x66, 0xd7, 0x76, 0x6c, - 0xd7, 0xfe, 0xd1, 0x81, 0xce, 0x59, 0x16, 0xcf, 0xc2, 0xb1, 0x54, 0xe3, 0x38, 0x99, 0xdc, 0xed, - 0x54, 0x76, 0x5f, 0xc5, 0x76, 0x5f, 0x0f, 0xdc, 0xc1, 0x37, 0x89, 0x9e, 0x19, 0x0f, 0x68, 0x55, - 0xdb, 0x88, 0x92, 0x44, 0x16, 0xf1, 0x08, 0x2a, 0x83, 0x84, 0x72, 0xb6, 0x7d, 0x7a, 0x50, 0x30, - 0x1a, 0x9e, 0xca, 0x20, 0xf1, 0x7f, 0x08, 0x87, 0xac, 0x88, 0x21, 0xe9, 0x21, 0x79, 0x08, 0xb5, - 0x8b, 0x24, 0x89, 0xcd, 0x98, 0x64, 0x00, 0xbf, 0x45, 0xf2, 0xb9, 0x8b, 0xc1, 0x78, 0x9b, 0x9c, - 0xd8, 0xf6, 0x4b, 0xa1, 0x0b, 0xed, 0xab, 0x38, 0xfb, 0x75, 0x12, 0x66, 0xd4, 0x3c, 0x78, 0xd8, - 0xd9, 0x28, 0xff, 0x63, 0x78, 0x67, 0xed, 0xe5, 0x62, 0x9a, 0x63, 0x1a, 0xb9, 0xc5, 0x67, 0xf9, - 0x10, 0xee, 0xe7, 0xac, 0x83, 0xfe, 0x5b, 0xe9, 0xb8, 0x29, 0xf4, 0x07, 0x96, 0xe5, 0x24, 0x54, - 0x3f, 0xbf, 0xc5, 0x1a, 0xff, 0x1c, 0x3c, 0xed, 0x4d, 0xfe, 0x2f, 0xa2, 0x35, 0x18, 0x85, 0x6a, - 0x75, 0xd7, 0xc7, 0x13, 0xad, 0x42, 0x15, 0x5a, 0xec, 0xe8, 0xec, 0xff, 0xa9, 0x02, 0x87, 0xdb, - 0x84, 0x14, 0x09, 0xe5, 0x58, 0x09, 0x25, 0x4e, 0xa1, 0xf6, 0x4d, 0xa8, 0x56, 0x66, 0x7f, 0x39, - 0xb2, 0x82, 0xbd, 0xa1, 0x83, 0x64, 0x56, 0x2c, 0xa4, 0xb3, 0x71, 0x66, 0xb6, 0xcd, 0x96, 0xd4, - 0x10, 0xbe, 0x70, 0x1e, 0xc5, 0xe3, 0xdf, 0xf2, 0x77, 0xac, 0x64, 0x60, 0x4b, 0x61, 0xd4, 0xbe, - 0x65, 0x61, 0xd4, 0xb7, 0x16, 0x46, 0x0f, 0xee, 0x7d, 0xb9, 0x98, 0x04, 0x99, 0xba, 0xb8, 0x09, - 0xd3, 0x4c, 0xcd, 0xc7, 0xca, 0x6b, 0x90, 0x45, 0xeb, 0x68, 0xdc, 0xa8, 0x77, 0xb5, 0x15, 0x4c, - 0xba, 0xe3, 0x93, 0x47, 0x40, 0x15, 0xcd, 0x33, 0x4b, 0x2c, 0xb9, 0x3b, 0xf7, 0x96, 0x4b, 0xbe, - 0xd5, 0xde, 0xda, 0x07, 0x77, 0xa8, 0x32, 0xbd, 0x48, 0xe3, 0x11, 0x5b, 0x03, 0x91, 0xb8, 0x1c, - 0x53, 0xbd, 0xb3, 0x96, 0x70, 0xfe, 0x57, 0xf0, 0x5e, 0xc9, 0xa5, 0x54, 0x8d, 0x26, 0x2c, 0xc5, - 0xba, 0xeb, 0x94, 0xd6, 0xdd, 0xef, 0x43, 0x6d, 0x64, 0x05, 0xe6, 0x80, 0x67, 0xbc, 0x65, 0x8c, - 0x64, 0xba, 0x3f, 0x2c, 0xcd, 0x78, 0xec, 0x91, 0x67, 0xd3, 0x69, 0xa2, 0xa6, 0x41, 0x66, 0x92, - 0xa5, 0x40, 0x88, 0x8f, 0xa0, 0x4e, 0xcc, 0x46, 0xec, 0xfa, 0xd2, 0xa6, 0xa9, 0xfe, 0x87, 0xd6, - 0x00, 0xcf, 0xd3, 0xcc, 0xb1, 0xd2, 0xac, 0x6b, 0x0f, 0xed, 0x6d, 0x1c, 0xe7, 0xfb, 0xff, 0x78, - 0x7d, 0xec, 0xfc, 0xeb, 0xf5, 0xb1, 0xf3, 0x9f, 0xd7, 0xc7, 0xce, 0x9f, 0xff, 0x7b, 0xbc, 0x73, - 0x5d, 0xa7, 0xff, 0x87, 0x3f, 0xf9, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x49, 0x89, 0x63, 0x75, - 0x4f, 0x14, 0x00, 0x00, + // 1872 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0xe3, 0xc6, + 0x15, 0x37, 0x49, 0xfd, 0x7d, 0x92, 0xbd, 0xf6, 0xac, 0x77, 0xc3, 0x6c, 0x1c, 0x47, 0x4b, 0x14, + 0xa9, 0x52, 0xb7, 0x1b, 0xd4, 0x0d, 0x82, 0x22, 0x40, 0x1b, 0x58, 0x2b, 0x6f, 0x57, 0xf0, 0xae, + 0xb3, 0x1d, 0x39, 0xea, 0x25, 0x17, 0x5a, 0x9a, 0x2a, 0x44, 0x29, 0x51, 0x25, 0xa9, 0xc8, 0x3e, + 0xf6, 0x50, 0xb4, 0x1f, 0xa1, 0xb7, 0x7e, 0x9a, 0xa2, 0xbd, 0xb5, 0x87, 0x1e, 0x7a, 0x2c, 0xb6, + 0x5f, 0xa4, 0x78, 0x6f, 0x66, 0xc8, 0x21, 0x45, 0x1b, 0x69, 0xd0, 0x1b, 0xdf, 0x1f, 0xbe, 0x79, + 0xef, 0xf7, 0xfe, 0xcc, 0x23, 0xa1, 0xbb, 0x5a, 0x5f, 0x87, 0xc1, 0xf4, 0xd9, 0x2a, 0x8e, 0xd2, + 0x88, 0xd9, 0xab, 0x6b, 0xef, 0x16, 0x1c, 0x1e, 0x6d, 0x98, 0x0b, 0xcd, 0xe7, 0x51, 0xb8, 0x5e, + 0x2c, 0x13, 0xd7, 0xea, 0x39, 0xfd, 0x1a, 0xd7, 0x24, 0x63, 0x50, 0xbb, 0x10, 0xb7, 0x89, 0xeb, + 0xf4, 0x9c, 0x7e, 0x9b, 0xd3, 0x33, 0x6a, 0xf3, 0xc8, 0x8f, 0x83, 0xe5, 0xdc, 0xad, 0xf5, 0xac, + 0x7e, 0x97, 0x6b, 0x92, 0x1d, 0x42, 0x7d, 0xb4, 0x9c, 0x89, 0x1b, 0xb7, 0xde, 0xb3, 0xfa, 0x6d, + 0x2e, 0x09, 0xe4, 0xbe, 0x08, 0x44, 0x38, 0x73, 0x1b, 0x92, 0x4b, 0x84, 0xd7, 0x87, 0x36, 0x8f, + 0x36, 0xaf, 0xfd, 0x34, 0x0e, 0x6e, 0xd8, 0x7b, 0x50, 0xe3, 0xd1, 0x46, 0x9e, 0xde, 0x39, 0x6d, + 0x3e, 0x5b, 0x5d, 0x3f, 0xe3, 0xd1, 0x86, 0x13, 0xd3, 0x3b, 0x83, 0xf6, 0x38, 0x98, 0x2f, 0xc5, + 0x0c, 0x5d, 0x7d, 0x17, 0x9c, 0x37, 0x11, 0x2a, 0x5a, 0xa6, 0x22, 0xf2, 0x50, 0x74, 0x29, 0xe6, + 0xae, 0x5d, 0x12, 0x5d, 0x8a, 0xb9, 0xf7, 0x53, 0xd8, 0xe3, 0xd1, 0x66, 0x34, 0x13, 0xcb, 0x34, + 0xf8, 0x75, 0x20, 0x62, 0x0a, 0x2c, 0x3b, 0xb1, 0x26, 0x0f, 0xca, 0x82, 0xb5, 0xf3, 0x60, 0xbd, + 0x27, 0xd0, 0x18, 0x0d, 0x5f, 0x05, 0x49, 0xca, 0xf6, 0xc1, 0x19, 0x0d, 0xf5, 0x0b, 0xf8, 0xe8, + 0x3d, 0x87, 0x83, 0xf3, 0x9b, 0x34, 0xf6, 0xa7, 0xa9, 0x98, 0x8d, 0x86, 0x12, 0x32, 0xb6, 0x07, + 0xf6, 0x68, 0x48, 0xfe, 0xd5, 0xb8, 0x3d, 0x1a, 0xb2, 0x63, 0xa8, 0x4d, 0xfc, 0x50, 0x1a, 0xed, + 0x9c, 0x02, 0xba, 0x25, 0x0d, 0x72, 0xe2, 0x7b, 0xbf, 0xb3, 0xe0, 0x1d, 0xc3, 0x8a, 0x04, 0x64, + 0x1c, 0xc5, 0xa9, 0x98, 0xb1, 0xe2, 0x01, 0x52, 0xa4, 0x42, 0x7f, 0x84, 0x86, 0xb6, 0x84, 0x7c, + 0x5b, 0x9f, 0x3d, 0x85, 0x06, 0x8f, 0x36, 0x17, 0x13, 0xed, 0x42, 0x5b, 0x21, 0x73, 0x31, 0xe1, + 0x4a, 0xe0, 0xbd, 0x82, 0x3a, 0x3d, 0x61, 0xaa, 0x10, 0x27, 0xed, 0xbf, 0x24, 0xd8, 0x8f, 0xa0, + 0x3e, 0xf1, 0xc3, 0xb5, 0x50, 0xd0, 0xbe, 0x53, 0x38, 0xfa, 0xca, 0xbf, 0x0e, 0x05, 0x89, 0xb9, + 0xd4, 0xf2, 0xbe, 0xaa, 0xf0, 0x9a, 0x3d, 0x86, 0x06, 0xe5, 0x5d, 0x02, 0xd8, 0xe6, 0x8a, 0x62, + 0x1f, 0xe7, 0xa5, 0x27, 0xdd, 0x2b, 0x07, 0x26, 0xa5, 0x59, 0x45, 0x7a, 0xef, 0x43, 0xf3, 0x42, + 0xdc, 0x52, 0x46, 0x74, 0xbe, 0x2c, 0x23, 0x5f, 0x7f, 0xb7, 0xe0, 0x61, 0x85, 0x6f, 0xec, 0x58, + 0x67, 0xcf, 0x2a, 0x66, 0xe1, 0xe5, 0x0e, 0xe5, 0x92, 0x3d, 0xcd, 0x72, 0x8f, 0x0a, 0x1d, 0x54, + 0x50, 0xc7, 0xbc, 0xdc, 0x51, 0x75, 0x7f, 0x04, 0xad, 0xc1, 0x78, 0x24, 0x91, 0x70, 0x7a, 0x56, + 0xdf, 0x79, 0xb9, 0xc3, 0x33, 0x0e, 0x7b, 0x02, 0xcd, 0xd7, 0xeb, 0x54, 0xdc, 0x8c, 0x86, 0xd4, + 0x15, 0xb5, 0x97, 0x3b, 0x5c, 0x33, 0xf0, 0x4d, 0x7a, 0xbc, 0x10, 0xb7, 0xb2, 0x35, 0xf0, 0x4d, + 0xcd, 0x61, 0x87, 0x50, 0x1b, 0x44, 0x51, 0x48, 0xed, 0xd1, 0xc2, 0xd3, 0x90, 0x1a, 0x34, 0x15, + 0xe8, 0xde, 0x0d, 0x1c, 0x16, 0x03, 0x52, 0x85, 0xc6, 0xc0, 0x41, 0x7b, 0x96, 0xb2, 0x87, 0x04, + 0xdb, 0xa7, 0xe2, 0xb3, 0xd5, 0xf9, 0x58, 0x7e, 0x1f, 0x43, 0x83, 0xcc, 0xc8, 0x16, 0xbe, 0x27, + 0x79, 0x4a, 0x6d, 0xd0, 0x26, 0x7c, 0xbf, 0x88, 0x47, 0x43, 0xef, 0x67, 0x65, 0x28, 0x29, 0x67, + 0x08, 0xfb, 0xa5, 0xbf, 0x10, 0xf2, 0x64, 0x4e, 0xcf, 0xc8, 0xbb, 0xba, 0x5d, 0xc9, 0x0a, 0x69, + 0x73, 0x7a, 0xf6, 0xd6, 0xb0, 0x57, 0x7c, 0x1d, 0x9d, 0x31, 0x8a, 0xa0, 0xd2, 0x19, 0x92, 0x67, + 0xd5, 0x71, 0x5a, 0xae, 0x0e, 0x77, 0xfb, 0x8d, 0x72, 0x81, 0xfc, 0x1c, 0x6a, 0x6f, 0xfc, 0x20, + 0xde, 0x6a, 0xc4, 0x7d, 0x89, 0x97, 0x43, 0x1e, 0x3a, 0x12, 0xf8, 0xfa, 0xf3, 0x68, 0xbd, 0x4c, + 0x25, 0x60, 0x5c, 0x12, 0xde, 0xe7, 0xd0, 0xc6, 0xf7, 0x65, 0xac, 0x47, 0xd2, 0x98, 0xaa, 0x9b, + 0x16, 0x9e, 0x8e, 0x34, 0x97, 0x47, 0x64, 0x93, 0xcd, 0x36, 0x27, 0xdb, 0x00, 0x00, 0xa5, 0x89, + 0xb4, 0x70, 0x0c, 0x75, 0xa2, 0x54, 0xc8, 0xb9, 0x09, 0xc9, 0xbe, 0xc3, 0xc6, 0xfb, 0x38, 0x49, + 0xd3, 0x4f, 0x3f, 0x41, 0xb1, 0xac, 0x38, 0xf4, 0xc0, 0xd1, 0x2d, 0x16, 0x41, 0x4b, 0x02, 0x15, + 0x6d, 0x72, 0x03, 0x96, 0x61, 0x20, 0xef, 0x64, 0xdb, 0xec, 0xe4, 0xc7, 0x72, 0x16, 0x64, 0x30, + 0x28, 0x8a, 0x7d, 0xa0, 0x4f, 0xa9, 0x51, 0x9c, 0x34, 0x22, 0xe8, 0x7c, 0x7d, 0xe0, 0xef, 0x2d, + 0x80, 0x5f, 0xc4, 0xd1, 0x7a, 0x45, 0x18, 0x31, 0x0f, 0xea, 0x44, 0xa9, 0xa0, 0xba, 0xa8, 0xaf, + 0x1d, 0xe2, 0x52, 0x54, 0x8d, 0x2e, 0x66, 0xe1, 0x6c, 0x3e, 0x97, 0xfd, 0xc3, 0xf1, 0x91, 0x9d, + 0x00, 0x0c, 0xc5, 0x34, 0x58, 0xf8, 0x21, 0x0a, 0x6a, 0x79, 0xff, 0x29, 0x2e, 0x37, 0xc4, 0xde, + 0x9f, 0x2d, 0x68, 0x4d, 0xfc, 0x30, 0xb3, 0x35, 0xf1, 0x43, 0x85, 0x0c, 0x3e, 0x16, 0xcf, 0x74, + 0xf4, 0x99, 0x4f, 0xa0, 0xf5, 0x22, 0x8c, 0xfc, 0x14, 0x95, 0xf1, 0x60, 0x8b, 0x67, 0xb4, 0x71, + 0x3a, 0x4a, 0xef, 0x39, 0x1d, 0x95, 0x3d, 0xe8, 0x5e, 0x05, 0x0b, 0x91, 0xa4, 0xfe, 0x62, 0x85, + 0xea, 0xf2, 0x9a, 0x2b, 0xf0, 0x10, 0xa9, 0xa6, 0x7a, 0xa5, 0x3a, 0x79, 0xc8, 0x1d, 0x4f, 0xfd, + 0x50, 0x68, 0x27, 0x89, 0x60, 0xc7, 0x00, 0x97, 0x62, 0x33, 0x11, 0x71, 0x12, 0x44, 0x4b, 0x72, + 0xb3, 0xc5, 0x0d, 0x0e, 0xa6, 0x6e, 0xe2, 0x87, 0x67, 0xd7, 0x89, 0xba, 0x74, 0x15, 0xa5, 0xf8, + 0x78, 0xf1, 0xd5, 0xe9, 0x1d, 0x45, 0x79, 0x9f, 0xc3, 0xc1, 0x30, 0x48, 0xd2, 0x60, 0x39, 0x4d, + 0x33, 0xff, 0x94, 0x32, 0x4e, 0x03, 0x35, 0x85, 0x25, 0x95, 0xb5, 0xb4, 0x9d, 0xb7, 0xb4, 0xf7, + 0x17, 0x0b, 0xba, 0xbf, 0x5c, 0x8b, 0xf8, 0x96, 0x8b, 0xdf, 0xae, 0x45, 0x92, 0xa2, 0xdf, 0x44, + 0xeb, 0x42, 0x23, 0x02, 0x4d, 0x8e, 0xbf, 0xf6, 0xe3, 0x99, 0xec, 0xd0, 0x1a, 0x57, 0x14, 0x95, + 0x9a, 0x58, 0x44, 0xa9, 0xd0, 0x7e, 0x49, 0x8a, 0x9d, 0x40, 0xf7, 0x7c, 0x71, 0x2d, 0x66, 0x33, + 0x31, 0x1b, 0xfa, 0xa9, 0xef, 0xb6, 0x8a, 0x57, 0x7e, 0x41, 0xc8, 0xbe, 0x07, 0xbb, 0x6f, 0x62, + 0x71, 0x15, 0xfb, 0xcb, 0x24, 0xf4, 0x53, 0x31, 0x73, 0xdb, 0x64, 0xab, 0xc8, 0x64, 0x47, 0xd0, + 0x7e, 0xed, 0xdf, 0xbc, 0x16, 0x8b, 0x28, 0xbe, 0x75, 0x81, 0x40, 0xcd, 0x19, 0xde, 0x2b, 0xd8, + 0x55, 0x61, 0x24, 0xab, 0x68, 0x99, 0x08, 0x2c, 0x9b, 0xf3, 0x38, 0x56, 0x51, 0xe0, 0x23, 0xfb, + 0x08, 0x9a, 0x5c, 0x24, 0xeb, 0x30, 0xd5, 0x63, 0xe6, 0x01, 0xba, 0xa3, 0xdf, 0x5a, 0x87, 0x29, + 0xd7, 0x72, 0xef, 0x9f, 0x4d, 0xe8, 0x18, 0x82, 0x6c, 0xf0, 0xe1, 0xf0, 0xde, 0x95, 0x83, 0x0f, + 0x17, 0x11, 0x1e, 0x6d, 0xb6, 0x76, 0x14, 0x6c, 0xd6, 0x2e, 0x58, 0x97, 0xaa, 0x21, 0xac, 0xcb, + 0x7c, 0x36, 0x38, 0xd5, 0xb3, 0x01, 0xf7, 0xb2, 0xaf, 0xfd, 0xe5, 0x5c, 0xcc, 0x28, 0xe9, 0x2d, + 0xae, 0x49, 0xd6, 0xcf, 0xdb, 0x80, 0xf0, 0x55, 0x3d, 0xa8, 0x79, 0x3c, 0x6f, 0x12, 0xd9, 0xf2, + 0x78, 0xf7, 0x35, 0x65, 0x7e, 0x24, 0xc5, 0x3e, 0x85, 0xbd, 0x2f, 0xc2, 0x59, 0xde, 0xd3, 0x89, + 0xca, 0xc4, 0x1e, 0xda, 0xc9, 0xd9, 0xbc, 0xa4, 0xc5, 0x3e, 0x2b, 0xaf, 0x52, 0x94, 0x93, 0xce, + 0x29, 0x53, 0x71, 0x1a, 0x12, 0x5e, 0x5e, 0xba, 0x4e, 0x8c, 0x4d, 0x8e, 0x12, 0xd5, 0x39, 0xdd, + 0xc5, 0xd7, 0x32, 0x26, 0x37, 0x36, 0xbd, 0x67, 0xe6, 0x18, 0x75, 0x3b, 0xa4, 0xbd, 0xa7, 0x11, + 0x92, 0x5c, 0x6e, 0x0e, 0xda, 0x13, 0x63, 0x6e, 0xbb, 0xdd, 0xdc, 0x78, 0xc6, 0xe4, 0xc6, 0x5c, + 0xaf, 0xdc, 0xac, 0x76, 0xff, 0xc7, 0xcd, 0xea, 0xb3, 0xf2, 0x05, 0xe7, 0xee, 0xe5, 0x50, 0x14, + 0x25, 0xbc, 0x7c, 0x15, 0x9e, 0x18, 0xeb, 0xaf, 0xfb, 0x20, 0xf7, 0x36, 0x63, 0x72, 0x63, 0x3d, + 0xfe, 0x31, 0x74, 0xcc, 0x44, 0xed, 0x93, 0xfa, 0x83, 0x62, 0xa2, 0x12, 0x6e, 0xea, 0x60, 0x80, + 0x5b, 0xed, 0xef, 0x1e, 0xe4, 0x01, 0x6e, 0x09, 0x79, 0xc5, 0xb8, 0x38, 0x81, 0x36, 0xb6, 0xe1, + 0x8b, 0x18, 0x67, 0x03, 0xcb, 0x9d, 0xcc, 0x98, 0x3c, 0x97, 0x63, 0xbe, 0xce, 0xe2, 0x38, 0xda, + 0x48, 0x24, 0x1e, 0xe6, 0xf9, 0xca, 0xb9, 0xdc, 0xd0, 0x60, 0x5f, 0xde, 0xb9, 0xf7, 0xba, 0x87, + 0xf4, 0xf2, 0x7b, 0x95, 0x89, 0x90, 0x2a, 0xfc, 0xce, 0x9d, 0xf9, 0x13, 0xe8, 0x5e, 0xad, 0x57, + 0xa1, 0xd0, 0x0d, 0xfd, 0x88, 0x6c, 0xed, 0xa3, 0x2d, 0x93, 0xcf, 0x0b, 0x5a, 0xde, 0x5f, 0x6d, + 0xd8, 0x1d, 0x2d, 0x56, 0x51, 0x9c, 0x1a, 0xd3, 0x4e, 0x7e, 0xcb, 0x58, 0x95, 0xdf, 0x32, 0x76, + 0xe9, 0xb2, 0xa5, 0xa9, 0x47, 0x63, 0xbb, 0xc6, 0x25, 0x61, 0x74, 0x5e, 0xad, 0xd0, 0x79, 0x47, + 0xd0, 0x96, 0xbb, 0x0a, 0x8a, 0xea, 0x24, 0xca, 0x19, 0xf2, 0xeb, 0x6a, 0x43, 0xbb, 0x68, 0x93, + 0x66, 0xb4, 0x26, 0xf1, 0x86, 0x90, 0x6a, 0x24, 0x6c, 0x91, 0xd0, 0xe0, 0xa0, 0x3c, 0x4b, 0x5d, + 0xe2, 0x36, 0x7a, 0x4e, 0xdf, 0xe1, 0x06, 0x87, 0x7d, 0x08, 0x7b, 0x14, 0xc4, 0xf3, 0x58, 0xe0, + 0xd8, 0x3c, 0x4b, 0xa9, 0x73, 0x1d, 0x5e, 0xe2, 0xa2, 0x1e, 0x85, 0x95, 0xeb, 0xc9, 0x99, 0x5a, + 0xe2, 0xd2, 0x65, 0x1b, 0x0a, 0x3f, 0xa6, 0xde, 0x6c, 0x71, 0x49, 0x78, 0xff, 0xb2, 0x81, 0x49, + 0x24, 0xe5, 0x5e, 0xf9, 0x7f, 0x83, 0xf3, 0x7e, 0xd8, 0x8a, 0xe0, 0x34, 0xb7, 0xc0, 0xc9, 0x6f, + 0x3e, 0x09, 0x8c, 0xbe, 0xf9, 0x7a, 0xd0, 0xd1, 0xbb, 0x00, 0x0a, 0x11, 0x55, 0x8b, 0x9b, 0x2c, + 0xbc, 0xf4, 0xc7, 0x29, 0x7e, 0xde, 0x2a, 0x95, 0x36, 0xd9, 0x2e, 0xf0, 0x2a, 0xa0, 0x85, 0x6f, + 0x09, 0x6d, 0xe7, 0x7e, 0x68, 0xbb, 0x26, 0xb4, 0x7f, 0xb0, 0xa0, 0x7b, 0x96, 0x46, 0x8b, 0x60, + 0xca, 0xc5, 0x34, 0x8a, 0x67, 0x77, 0x83, 0x2a, 0xe1, 0xb3, 0x4d, 0xf8, 0xfa, 0xe0, 0x8c, 0xbe, + 0x89, 0xd5, 0x4d, 0xf3, 0x98, 0x16, 0xbc, 0xad, 0x2c, 0x71, 0x54, 0x61, 0x4f, 0xc1, 0x1e, 0xc5, + 0x54, 0xb3, 0x9d, 0xd3, 0x83, 0x5c, 0x51, 0xeb, 0xd8, 0xa3, 0xd8, 0xfb, 0x21, 0x1c, 0x4a, 0x47, + 0xb4, 0x48, 0x5d, 0xad, 0x87, 0x50, 0x3f, 0x8f, 0xe3, 0x48, 0x5f, 0xae, 0x92, 0xc0, 0x2f, 0x98, + 0xec, 0xb6, 0xc6, 0x64, 0x7c, 0x97, 0x9a, 0xa8, 0xfa, 0x11, 0xd1, 0x83, 0xce, 0x65, 0x94, 0xfe, + 0x2a, 0x0e, 0x52, 0x1a, 0x39, 0xf2, 0x8a, 0x34, 0x59, 0xde, 0x47, 0xf0, 0xa8, 0x74, 0x72, 0xbe, + 0x03, 0x60, 0x19, 0x39, 0xf9, 0xc7, 0xfc, 0x18, 0x1e, 0x66, 0xaa, 0xa3, 0xe1, 0x77, 0xf2, 0x71, + 0xdb, 0xe8, 0x0f, 0x8c, 0xc8, 0xc9, 0xa8, 0x3a, 0xbe, 0x22, 0x1a, 0x6f, 0x00, 0xae, 0x42, 0x53, + 0xfe, 0x4d, 0x51, 0x1e, 0x4c, 0x02, 0xb1, 0xb9, 0xeb, 0x93, 0x8b, 0x16, 0x28, 0x9b, 0xd6, 0x41, + 0x7a, 0xf6, 0xfe, 0x68, 0xc3, 0x61, 0x95, 0x91, 0xbc, 0xa0, 0x2c, 0xa3, 0xa0, 0xd8, 0x29, 0xd4, + 0xbf, 0x09, 0xc4, 0x46, 0x6f, 0x3d, 0x47, 0x46, 0xb2, 0xb7, 0x7c, 0xe0, 0x52, 0x15, 0x1b, 0xe9, + 0x6c, 0x9a, 0xea, 0x1d, 0xb5, 0xcd, 0x15, 0x85, 0x27, 0x0c, 0xc2, 0x68, 0xfa, 0x1b, 0xf9, 0xf5, + 0xcb, 0x25, 0x51, 0xd1, 0x18, 0xf5, 0x6f, 0xd9, 0x18, 0x8d, 0xca, 0xc6, 0xe8, 0xc3, 0x83, 0x2f, + 0x57, 0x33, 0x3f, 0x15, 0xe7, 0x37, 0x41, 0x92, 0x8a, 0xe5, 0x54, 0xb8, 0x4d, 0x8a, 0xa8, 0xcc, + 0xc6, 0x3d, 0x7c, 0x57, 0x45, 0x21, 0x45, 0x77, 0x7c, 0x28, 0x31, 0xa8, 0x61, 0x78, 0x7a, 0xf5, + 0x25, 0xb8, 0x33, 0xb4, 0x1c, 0xc2, 0x56, 0xa1, 0xb5, 0x0f, 0xce, 0x58, 0xa4, 0x6a, 0xfd, 0xc6, + 0x47, 0x1c, 0x0d, 0x24, 0x92, 0xed, 0x98, 0xa8, 0x4d, 0xb7, 0xc0, 0xf3, 0xbe, 0x82, 0x77, 0x0b, + 0x90, 0x52, 0x37, 0xea, 0xb4, 0xe4, 0x4b, 0xb2, 0x55, 0x58, 0x92, 0xbf, 0x0f, 0xf5, 0x89, 0x91, + 0x98, 0x03, 0xb9, 0x19, 0x18, 0xc1, 0x70, 0x29, 0xf7, 0xc6, 0x85, 0xcd, 0x00, 0x67, 0xe4, 0xd9, + 0x7c, 0x1e, 0x8b, 0xb9, 0x9f, 0xea, 0x62, 0xc9, 0x19, 0xec, 0x43, 0x68, 0x90, 0xb2, 0x36, 0x5b, + 0x5e, 0xf5, 0x94, 0xd4, 0xfb, 0xc0, 0xb8, 0xf6, 0xb3, 0x32, 0xb3, 0x8c, 0x32, 0xeb, 0x99, 0x57, + 0x7d, 0xa5, 0x86, 0x57, 0xbc, 0x85, 0xab, 0x74, 0x06, 0xfb, 0x7f, 0x7b, 0x7b, 0x6c, 0xfd, 0xe3, + 0xed, 0xb1, 0xf5, 0xef, 0xb7, 0xc7, 0xd6, 0x9f, 0xfe, 0x73, 0xbc, 0x73, 0xdd, 0xa0, 0x3f, 0x93, + 0x3f, 0xf9, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0b, 0x50, 0x7f, 0x0d, 0xa9, 0x14, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -4465,6 +4520,20 @@ func (m *QueryResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.TupleResults != nil { + { + size, err := m.TupleResults.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xaa + } if m.ExtractedIDMatrixSorted != nil { { size, err := m.ExtractedIDMatrixSorted.MarshalToSizedBuffer(dAtA[:i]) @@ -4634,20 +4703,20 @@ func (m *QueryResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } if len(m.RowIDs) > 0 { - dAtA32 := make([]byte, len(m.RowIDs)*10) - var j31 int + dAtA33 := make([]byte, len(m.RowIDs)*10) + var j32 int for _, num := range m.RowIDs { for num >= 1<<7 { - dAtA32[j31] = uint8(uint64(num)&0x7f | 0x80) + dAtA33[j32] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j31++ + j32++ } - dAtA32[j31] = uint8(num) - j31++ + dAtA33[j32] = uint8(num) + j32++ } - i -= j31 - copy(dAtA[i:], dAtA32[:j31]) - i = encodeVarintPublic(dAtA, i, uint64(j31)) + i -= j32 + copy(dAtA[i:], dAtA33[:j32]) + i = encodeVarintPublic(dAtA, i, uint64(j32)) i-- dAtA[i] = 0x3a } @@ -4775,57 +4844,57 @@ func (m *ImportRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } if len(m.Timestamps) > 0 { - dAtA36 := make([]byte, len(m.Timestamps)*10) - var j35 int + dAtA37 := make([]byte, len(m.Timestamps)*10) + var j36 int for _, num1 := range m.Timestamps { num := uint64(num1) for num >= 1<<7 { - dAtA36[j35] = uint8(uint64(num)&0x7f | 0x80) + dAtA37[j36] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j35++ + j36++ } - dAtA36[j35] = uint8(num) - j35++ + dAtA37[j36] = uint8(num) + j36++ } - i -= j35 - copy(dAtA[i:], dAtA36[:j35]) - i = encodeVarintPublic(dAtA, i, uint64(j35)) + i -= j36 + copy(dAtA[i:], dAtA37[:j36]) + i = encodeVarintPublic(dAtA, i, uint64(j36)) i-- dAtA[i] = 0x32 } if len(m.ColumnIDs) > 0 { - dAtA38 := make([]byte, len(m.ColumnIDs)*10) - var j37 int + dAtA39 := make([]byte, len(m.ColumnIDs)*10) + var j38 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA38[j37] = uint8(uint64(num)&0x7f | 0x80) + dAtA39[j38] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j37++ + j38++ } - dAtA38[j37] = uint8(num) - j37++ + dAtA39[j38] = uint8(num) + j38++ } - i -= j37 - copy(dAtA[i:], dAtA38[:j37]) - i = encodeVarintPublic(dAtA, i, uint64(j37)) + i -= j38 + copy(dAtA[i:], dAtA39[:j38]) + i = encodeVarintPublic(dAtA, i, uint64(j38)) i-- dAtA[i] = 0x2a } if len(m.RowIDs) > 0 { - dAtA40 := make([]byte, len(m.RowIDs)*10) - var j39 int + dAtA41 := make([]byte, len(m.RowIDs)*10) + var j40 int for _, num := range m.RowIDs { for num >= 1<<7 { - dAtA40[j39] = uint8(uint64(num)&0x7f | 0x80) + dAtA41[j40] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j39++ + j40++ } - dAtA40[j39] = uint8(num) - j39++ + dAtA41[j40] = uint8(num) + j40++ } - i -= j39 - copy(dAtA[i:], dAtA40[:j39]) - i = encodeVarintPublic(dAtA, i, uint64(j39)) + i -= j40 + copy(dAtA[i:], dAtA41[:j40]) + i = encodeVarintPublic(dAtA, i, uint64(j40)) i-- dAtA[i] = 0x22 } @@ -4906,9 +4975,9 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { } if len(m.FloatValues) > 0 { for iNdEx := len(m.FloatValues) - 1; iNdEx >= 0; iNdEx-- { - f41 := math.Float64bits(float64(m.FloatValues[iNdEx])) + f42 := math.Float64bits(float64(m.FloatValues[iNdEx])) i -= 8 - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f41)) + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f42)) } i = encodeVarintPublic(dAtA, i, uint64(len(m.FloatValues)*8)) i-- @@ -4924,39 +4993,39 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } if len(m.Values) > 0 { - dAtA43 := make([]byte, len(m.Values)*10) - var j42 int + dAtA44 := make([]byte, len(m.Values)*10) + var j43 int for _, num1 := range m.Values { num := uint64(num1) for num >= 1<<7 { - dAtA43[j42] = uint8(uint64(num)&0x7f | 0x80) + dAtA44[j43] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j42++ + j43++ } - dAtA43[j42] = uint8(num) - j42++ + dAtA44[j43] = uint8(num) + j43++ } - i -= j42 - copy(dAtA[i:], dAtA43[:j42]) - i = encodeVarintPublic(dAtA, i, uint64(j42)) + i -= j43 + copy(dAtA[i:], dAtA44[:j43]) + i = encodeVarintPublic(dAtA, i, uint64(j43)) i-- dAtA[i] = 0x32 } if len(m.ColumnIDs) > 0 { - dAtA45 := make([]byte, len(m.ColumnIDs)*10) - var j44 int + dAtA46 := make([]byte, len(m.ColumnIDs)*10) + var j45 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA45[j44] = uint8(uint64(num)&0x7f | 0x80) + dAtA46[j45] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j44++ + j45++ } - dAtA45[j44] = uint8(num) - j44++ + dAtA46[j45] = uint8(num) + j45++ } - i -= j44 - copy(dAtA[i:], dAtA45[:j44]) - i = encodeVarintPublic(dAtA, i, uint64(j44)) + i -= j45 + copy(dAtA[i:], dAtA46[:j45]) + i = encodeVarintPublic(dAtA, i, uint64(j45)) i-- dAtA[i] = 0x2a } @@ -5168,20 +5237,20 @@ func (m *TranslateKeysResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { copy(dAtA[i:], m.XXX_unrecognized) } if len(m.IDs) > 0 { - dAtA47 := make([]byte, len(m.IDs)*10) - var j46 int + dAtA48 := make([]byte, len(m.IDs)*10) + var j47 int for _, num := range m.IDs { for num >= 1<<7 { - dAtA47[j46] = uint8(uint64(num)&0x7f | 0x80) + dAtA48[j47] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j46++ + j47++ } - dAtA47[j46] = uint8(num) - j46++ + dAtA48[j47] = uint8(num) + j47++ } - i -= j46 - copy(dAtA[i:], dAtA47[:j46]) - i = encodeVarintPublic(dAtA, i, uint64(j46)) + i -= j47 + copy(dAtA[i:], dAtA48[:j47]) + i = encodeVarintPublic(dAtA, i, uint64(j47)) i-- dAtA[i] = 0x1a } @@ -5213,20 +5282,20 @@ func (m *TranslateIDsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { copy(dAtA[i:], m.XXX_unrecognized) } if len(m.IDs) > 0 { - dAtA49 := make([]byte, len(m.IDs)*10) - var j48 int + dAtA50 := make([]byte, len(m.IDs)*10) + var j49 int for _, num := range m.IDs { for num >= 1<<7 { - dAtA49[j48] = uint8(uint64(num)&0x7f | 0x80) + dAtA50[j49] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j48++ + j49++ } - dAtA49[j48] = uint8(num) - j48++ + dAtA50[j49] = uint8(num) + j49++ } - i -= j48 - copy(dAtA[i:], dAtA49[:j48]) - i = encodeVarintPublic(dAtA, i, uint64(j48)) + i -= j49 + copy(dAtA[i:], dAtA50[:j49]) + i = encodeVarintPublic(dAtA, i, uint64(j49)) i-- dAtA[i] = 0x1a } @@ -5639,6 +5708,40 @@ func (m *ArrowTable) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *TupleResults) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TupleResults) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *TupleResults) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Data) > 0 { + i -= len(m.Data) + copy(dAtA[i:], m.Data) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Data))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { offset -= sovPublic(v) base := offset @@ -6396,6 +6499,10 @@ func (m *QueryResult) Size() (n int) { l = m.ExtractedIDMatrixSorted.Size() n += 2 + l + sovPublic(uint64(l)) } + if m.TupleResults != nil { + l = m.TupleResults.Size() + n += 2 + l + sovPublic(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -6831,6 +6938,22 @@ func (m *ArrowTable) Size() (n int) { return n } +func (m *TupleResults) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Data) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovPublic(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -11010,6 +11133,42 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 21: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TupleResults", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.TupleResults == nil { + m.TupleResults = &TupleResults{} + } + if err := m.TupleResults.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -13725,6 +13884,91 @@ func (m *ArrowTable) Unmarshal(dAtA []byte) error { } return nil } +func (m *TupleResults) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TupleResults: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TupleResults: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPublic(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/pb/public.proto b/pb/public.proto index 8280a948f..2923ec4bd 100644 --- a/pb/public.proto +++ b/pb/public.proto @@ -175,6 +175,7 @@ message QueryResult { DataFrame DataFrame = 18; ArrowTable ArrowTable = 19; ExtractedIDMatrixSorted ExtractedIDMatrixSorted = 20; + TupleResults TupleResults = 21; } message ImportRequest { @@ -278,4 +279,7 @@ message DataFrame{ message ArrowTable{ bytes Data =1; } +message TupleResults{ + bytes Data =1; +} diff --git a/pql/ast.go b/pql/ast.go index 362d63f67..ef884acec 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -643,6 +643,12 @@ var callInfoByFunc = map[string]callInfo{ "header": interfaceOrVariable, }, }, + "Tstore": { + allowUnknown: false, + prototypes: map[string]interface{}{ + "header": interfaceOrVariable, + }, + }, } // We want to allow case-insensitive names, but we want to continue using diff --git a/sql3/planner/opsystemtable.go b/sql3/planner/opsystemtable.go index 2ac027db8..3f29a6c9e 100644 --- a/sql3/planner/opsystemtable.go +++ b/sql3/planner/opsystemtable.go @@ -614,6 +614,7 @@ func (i *fbTableDDLRowIter) Next(ctx context.Context) (types.Row, error) { for idx, tbl := range tbls { // build the ddl for this table + ddl := generateTableDDL(tbl, "") i.result[idx] = &fbTableDDLRow{ diff --git a/tstore.go b/tstore.go new file mode 100644 index 000000000..f175fe5e1 --- /dev/null +++ b/tstore.go @@ -0,0 +1,265 @@ +// Copyright 2021 Molecula Corp. All rights reserved. +package pilosa + +import ( + "bytes" + "context" + "encoding/json" + "math" + + "github.com/featurebasedb/featurebase/v3/pql" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" + "github.com/featurebasedb/featurebase/v3/tracing" + "github.com/featurebasedb/featurebase/v3/tstore" + "github.com/featurebasedb/featurebase/v3/wireprotocol" + "github.com/pkg/errors" +) + +/* +The function Tstore provides filtered access to the point lookup values in tsore. +If Tstore is just provided a bitmap filter, such as ConstRow or any Bitmap Operation, +all the values associated with each column are returned. This set can be limited with +the addition of the header parameter +Example: +Tstore(ConstRow(columns=[2,4,6]),header=["fval"]) +*/ + +func (e *executor) executeTstore(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*TupleResults, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTstore") + defer span.Finish() + if len(c.Children) > 1 { + return nil, errors.New("Tstore() only accepts a single bitmap input filter") + } + var columnFilter []string + if cols, ok := c.Args["header"].([]interface{}); ok { + columnFilter = make([]string, 0, len(cols)) + for _, v := range cols { + columnFilter = append(columnFilter, v.(string)) + } + } + // Execute calls in bulk on each remote node and merge. + mapFn := func(ctx context.Context, shard uint64, mopt *mapOptions) (_ interface{}, err error) { + return e.executeTstoreShard(ctx, qcx, index, c, shard, columnFilter) + } + results := &TupleResults{} + + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { + if v == nil { + return prev + } + r := v.(*TupleResults) + if results.TupleSchema == nil { + // just use the first one i get, they all should be the same + results.TupleSchema = r.TupleSchema + } + results.Append(r) + return nil + } + + _, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) + if err != nil { + return nil, err + } + return results, nil +} + +func (e *executor) executeTstoreShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, columnFilter []string) (*TupleResults, error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeArrowShard") + defer span.Finish() + + var filter *Row + if len(c.Children) == 1 { + row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard) + if err != nil { + return nil, err + } + filter = row + if !filter.Any() { + // no need to actuall run the query for its not operating against any values + return &TupleResults{}, nil + } + } + // + ids := filter.Columns() // needs to be shard columns + // Fetch index. + idx := e.Holder.Index(index) + if idx == nil { + return nil, newNotFoundError(ErrIndexNotFound, index) + } + b, err := idx.GetTStore(shard) + if err != nil { + return nil, err + } + first := tstore.Int(0) + last := tstore.Int(math.MaxInt32) + i := 0 + in := func(a tstore.Sortable) bool { + if i >= len(ids) { + return false + } + for tstore.Int(ids[i]).Less(a) { + i++ + if len(ids) == i { + return false + } + } + return true + } + if len(ids) == 0 { + in = func(a tstore.Sortable) bool { + return true + } + } else { + first = tstore.Int(ids[0]) + last = tstore.Int(ids[len(ids)-1] + 1) + } + + itr, err := b.NewRangeIterator(tstore.Int(first), tstore.Int(last)) + defer itr.Dispose() + + result := &TupleResults{} + unset := true + fixTuple := func(tuple *tstore.BTreeTuple) types.Row { + return tuple.Tuple + } + for itr.Next() { + item, key := itr.Item() + if unset { + result.TupleSchema = item.TupleSchema + if len(columnFilter) > 0 { + newSchema := make(types.Schema, 0) + parts := make([]int, 0) + for _, name := range columnFilter { + for i := range item.TupleSchema { + if item.TupleSchema[i].ColumnName == name { + newSchema = append(newSchema, item.TupleSchema[i]) + parts = append(parts, i) + } + } + } + if len(newSchema) == 0 { + return &TupleResults{}, nil + } + fixTuple = func(tuple *tstore.BTreeTuple) types.Row { + newrow := make(types.Row, len(parts)) + for i, v := range parts { + newrow[i] = tuple.Tuple[v] + } + return newrow + } + result.TupleSchema = newSchema + + } + unset = false + } + if in(key) { + result.Add(fixTuple(item)) + } + } + return result, nil +} + +type TupleResults struct { + TupleSchema types.Schema + rows []types.Row +} + +func (tr *TupleResults) Add(item types.Row) { + tr.rows = append(tr.rows, item) +} + +func (tr *TupleResults) Append(t *TupleResults) { + tr.rows = append(tr.rows, t.rows...) +} + +func (tr *TupleResults) MarshalJSON() ([]byte, error) { + results := make(map[string]interface{}) + columns := make([][]string, 0) + for i := range tr.TupleSchema { + column := tr.TupleSchema[i] + columns = append(columns, []string{column.ColumnName, column.Type.BaseTypeName()}) + } + rows := make([][]interface{}, 0) + for i := range tr.rows { + item := tr.rows[i] + row := make([]interface{}, len(columns)) + for c := range item { + row[c] = item[c] + } + rows = append(rows, row) + } + results["schema"] = columns + results["rows"] = rows + return json.Marshal(results) +} + +func (tr *TupleResults) ToBytes() ([]byte, error) { + buf := new(bytes.Buffer) + + // get the bytes for the schema + b, err := wireprotocol.WriteSchema(tr.TupleSchema) + if err != nil { + return nil, errors.Wrap(err, "serializing tuple schema") + } + _, err = buf.Write(b) + if err != nil { + return nil, errors.Wrap(err, "serializing tuple schema") + } + + // iterate the tupleData - outside loop is rows + for _, trow := range tr.rows { + rb, err := wireprotocol.WriteRow(trow, tr.TupleSchema) + if err != nil { + return nil, errors.Wrap(err, "serializing tuple row") + } + _, err = buf.Write(rb) + if err != nil { + return nil, errors.Wrap(err, "serializing tuple row") + } + } + + // write done to the buffer + b = wireprotocol.WriteDone() + _, err = buf.Write(b) + if err != nil { + return nil, errors.Wrap(err, "serializing tuples") + } + + return buf.Bytes(), nil +} + +func NewTupleResultFromBytes(data []byte) (*TupleResults, error) { + rdr := bytes.NewReader(data) + _, err := wireprotocol.ExpectToken(rdr, wireprotocol.TOKEN_SCHEMA_INFO) + if err != nil { + return nil, err + } + // get the row schema from the import data + schema, err := wireprotocol.ReadSchema(rdr) + if err != nil { + return nil, err + } + + // read rows until we get to the end + tk, err := wireprotocol.ReadToken(rdr) + if err != nil { + return nil, err + } + rows := make([]types.Row, 0) + for tk == wireprotocol.TOKEN_ROW { + row, err := wireprotocol.ReadRow(rdr, schema) + if err != nil { + return nil, err + } + rows = append(rows, row) + + tk, err = wireprotocol.ReadToken(rdr) + if err != nil { + return nil, err + } + } + if tk != wireprotocol.TOKEN_DONE { + return nil, errors.Errorf("unexpected token '%d'", tk) + } + return &TupleResults{TupleSchema: schema, rows: rows}, nil +} diff --git a/tstore/debug_utils.go b/tstore/debug_utils.go new file mode 100644 index 000000000..e3e04c504 --- /dev/null +++ b/tstore/debug_utils.go @@ -0,0 +1,248 @@ +package tstore + +/*internal node +example SLOTROW--> 102030 +node0[ label =< + + + + %v +
PageID:%v
SlotCount:%v
> + fillcolor="lightgrey" margin="0"]; +*/ +import ( + "bytes" + "fmt" + "io" + "strings" + + "github.com/featurebasedb/featurebase/v3/bufferpool" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" + "github.com/featurebasedb/featurebase/v3/vprint" + "github.com/featurebasedb/featurebase/v3/wireprotocol" +) + +func internalNode(node *BTreeNode, out io.Writer) { + si := bufferpool.NewPageSlotIterator(node.page, 0) + slot := si.Next() + numSlots := 0 + var slotRow strings.Builder + // build slotrow + slotRow.WriteString("") + var linker strings.Builder + nodeid := fmt.Sprintf("%v", node.page.ID().Page) + for slot != nil { + numSlots++ + pl := slot.KeyPayload(node.page) + k := int(pl.KeyAsInt(node.page)) + ipl := slot.InternalPayload(node.page) + pageID := bufferpool.PageID(ipl.ValueAsPagePointer(node.page)) + slotRow.WriteString(fmt.Sprintf(`%v\n`, pageID.Page, k)) + linker.WriteString(fmt.Sprintf("\np%v:p%v->p%v;", nodeid, pageID.Page, pageID.Page)) + slot = si.Next() + } + pn := node.page.ReadNextPointer() + if pn.Page != bufferpool.INVALID_PAGE { + slotRow.WriteString(fmt.Sprintf(`∅\n`, pn.Page)) + linker.WriteString(fmt.Sprintf("\np%v:p%v->p%v;", nodeid, pn.Page, pn.Page)) + } + + slotRow.WriteString("") + + fmt.Fprintf(out, ` + p%v[ label =< + + + + %v +
PageID:%v
SlotCount:%v
> + fillcolor="lightgrey" margin="0"]`, node.page.ID().Page, numSlots, node.page.ID().Page, numSlots, numSlots, slotRow.String()) + out.Write([]byte(linker.String())) +} + +func (b *BTree) leafNode(node *BTreeNode, out io.Writer, short bool, schema types.Schema) { + sc := int16(node.slotCount()) + si := bufferpool.NewPageSlotIterator(node.page, 0) + slot := si.Next() + fmt.Fprintf(out, ` + p%v[label =< + +`, + node.page.ID().Page, + node.page.ID().Page, + sc) + last := "" + cnt := 0 + var overflowPages strings.Builder + id := node.page.ID() + for slot != nil { + cnt++ + pl := slot.KeyPayload(node.page) + k := int(pl.KeyAsInt(node.page)) + // Get Value // TODO(twg) 2023/03/28 (pok) better way todo this? + lpl := slot.LeafPayload(node.page) + rdr := lpl.GetPayloadReader(node.page) + payload := make([]byte, rdr.PayloadTotalLength) + copy(payload, rdr.PayloadChunkBytes) + // check for overflow( rdr.Flags==1) + if rdr.Flags == 1 { + bytesReceived := rdr.PayloadChunkLength + nextPtr := rdr.OverflowPtr + fmt.Fprintf(out, "\n", k, nextPtr, "OVERFLOW" /*payload*/) + for nextPtr != bufferpool.INVALID_PAGE { + onode, _ := b.fetchNode(bufferpool.PageID{ObjectID: id.ObjectID, Shard: id.Shard, Page: nextPtr}) + onode.takeReadLatch() + // read the overflow bytes + clen, cbytes := onode.page.ReadLeafPagePayloadBytes(bufferpool.PAGE_SLOTS_START_OFFSET) + overflowPages.WriteString(fmt.Sprintf(` + p%v[label=<
PageID:%v
SlotCount:%v
%v%v
+ + +
PageID:%v
%v
> + fillcolor="cornflowerblue" margin="0"]; + p%v:p%v->p%v; + `, nextPtr, nextPtr, nextPtr, "cbytes", id.Page, nextPtr, nextPtr)) + copy(payload[bytesReceived:], cbytes) + bytesReceived += clen + + nextPtr = onode.page.ReadNextPointer().Page + onode.releaseReadLatch() + b.unpin(onode) + + } + } else { + t := NewBTreeTupleFromBytes(payload, schema) + values := make([]string, len(t.Tuple)) + types := make([]string, len(t.Tuple)) + names := make([]string, len(t.Tuple)) + for i := range t.Tuple { + values[i] = fmt.Sprintf("%v", t.Tuple[i]) + types[i] = t.TupleSchema[i].Type.TypeDescription() + names[i] = t.TupleSchema[i].ColumnName + /// + } + if short { + if cnt < 2 { + fmt.Fprintf(out, "\n%v%v%v%v", k, strings.Join(names, "
"), strings.Join(types, "
"), strings.Join(values, "
")) + } else { + last = fmt.Sprintf("\n%v%v%v%v", k, strings.Join(names, "
"), strings.Join(types, "
"), strings.Join(values, "
")) + } + } else { + fmt.Fprintf(out, "\n%v%v%v%v", k, strings.Join(names, "
"), strings.Join(types, "
"), strings.Join(values, "
")) + } + } + slot = si.Next() + } + if short { + fmt.Fprintf(out, "\n............") + out.Write([]byte(last)) + fmt.Fprintln(out, "") + } + out.Write([]byte(`> fillcolor="turquoise" margin="0"]`)) + op := overflowPages.String() + if len(op) > 0 { + out.Write([]byte(overflowPages.String())) + } +} + +func (b *BTree) Dot(out io.Writer, src string, short bool) { + fmt.Fprintf(out, `digraph g { + labelloc="t" + labeljust="r" + label=< + + + +
BTREE:%v
keysPerLeafPage:%v
keysPerInternalPage:%v
> + rankdir="TB" + node [shape = plaintext,height=.1, style=filled ];`, src, b.keysPerLeafPage, b.keysPerInternalPage) + + node, _ := b.fetchNode(b.rootNode) + defer b.bufferpool.UnpinPage(node.page.ID()) + b.dot(node, out, short) + fmt.Printf("}") +} + +func (b *BTree) dot(node *BTreeNode, out io.Writer, short bool) { + if node.isLeaf() { + b.leafNode(node, out, short, b.schema) + } else { + internalNode(node, out) + si := bufferpool.NewPageSlotIterator(node.page, 0) + slot := si.Next() + for slot != nil { + ipl := slot.InternalPayload(node.page) + pn := bufferpool.PageID(ipl.ValueAsPagePointer(node.page)) + cn, _ := b.fetchNode(pn) + b.dot(cn, out, short) + b.bufferpool.UnpinPage(cn.page.ID()) + slot = si.Next() + } + pn := node.page.ReadNextPointer() + if pn.Page != bufferpool.INVALID_PAGE { + cn, _ := b.fetchNode(pn) + b.dot(cn, out, short) + b.bufferpool.UnpinPage(cn.page.ID()) + } + } +} + +func OpenBtree(path string) (*BTree, error) { + diskManager := bufferpool.NewTupleStoreDiskManager() + objectID, shard := int32(0), int32(0) // these don't matter just to open the file + diskManager.CreateOrOpenShard(objectID, shard, path) + page0, err := diskManager.ReadPage(bufferpool.PageID{ObjectID: objectID, Shard: shard, Page: 0}) + if err != nil { + return nil, err + } + // ok so I have a page,now what todo with it? + // need to find the schema, I think its in slot 1? + slot := page0.ReadPageSlot(1) + ipl := slot.InternalPayload(page0) + schemaPageID := ipl.ValueAsPagePointer(page0) + schemaPage, err := diskManager.ReadPage(schemaPageID) + if err != nil { + return nil, err + } + + schema, err := getSchemaFrom(schemaPage) + if err != nil { + return nil, err + } + bufferPool := bufferpool.NewBufferPool(100, diskManager) + return NewBTree(8, objectID, shard, schema, bufferPool) +} + +func getSchemaFrom(page *bufferpool.Page) (types.Schema, error) { + sc := page.ReadSlotCount() + vprint.VV("slots:%v", sc) + slot := page.ReadPageSlot(int16(1)) + lpl := slot.LeafPayload(page) + rdr := lpl.GetPayloadReader(page) + payload := make([]byte, rdr.PayloadTotalLength) + copy(payload, rdr.PayloadChunkBytes) + // this works as long as the schema doesn't overflow + if rdr.Flags == 1 { + panic("fix this") + } + schema := types.Schema{ + &types.PlannerColumn{ + ColumnName: "schema", + Type: parser.NewDataTypeVarbinary(16384), + }, + } + ss := NewBTreeTupleFromBytes(payload, schema) + b := ss.Tuple[0].([]byte) + rd := bytes.NewReader(b) + _, err := wireprotocol.ExpectToken(rd, wireprotocol.TOKEN_SCHEMA_INFO) + if err != nil { + return nil, err + } + + s, err := wireprotocol.ReadSchema(rd) + if err != nil { + return nil, err + } + return s, nil +} diff --git a/tstore/debug_utils_test.go b/tstore/debug_utils_test.go new file mode 100644 index 000000000..2c48902cc --- /dev/null +++ b/tstore/debug_utils_test.go @@ -0,0 +1,209 @@ +package tstore + +import ( + "fmt" + "math/rand" + "os" + "sort" + "testing" + + "github.com/featurebasedb/featurebase/v3/bufferpool" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" + "github.com/stretchr/testify/assert" +) + +func buildSample(objectID int32, shard int32) (*BTree, error, func()) { + diskManager := bufferpool.NewTupleStoreDiskManager() + + dataFile := fmt.Sprintf("ts-shard.%04d", shard) + + diskManager.CreateOrOpenShard(objectID, shard, dataFile) + + bufferPool := bufferpool.NewBufferPool(100, diskManager) + + tableSchema := types.Schema{ + &types.PlannerColumn{ + ColumnName: "vtest", + Type: parser.NewDataTypeVarchar(50), + }, + } + b, e := NewBTree(8, objectID, shard, tableSchema, bufferPool) + return b, e, func() { + os.Remove(dataFile) + } +} + +func TestBTree_RangeIterator(t *testing.T) { + b, err, c := buildSample(1, 0) + defer c() + assert.Nil(t, err) + + rowSchema := types.Schema{ + &types.PlannerColumn{ + ColumnName: "_id", + Type: parser.NewDataTypeID(), + }, + &types.PlannerColumn{ + ColumnName: "vtest", + Type: parser.NewDataTypeVarchar(50), + }, + } + + inserts := make([]int, 0) + for i := 1; i <= 90; i++ { + inserts = append(inserts, i) + } + rand.Shuffle(len(inserts), func(i, j int) { inserts[i], inserts[j] = inserts[j], inserts[i] }) + + rr := make(types.Row, 2) + want1 := make([]string, 0) + want2 := make([]string, 0) + want3 := make([]string, 0) + for _, i := range inserts { + rr[0] = int64(i) + rr[1] = fmt.Sprintf("This is a test of things %d", i) + + if i >= 1 && i < 10 { // items in first page + want1 = append(want1, rr[1].(string)) + } + + if i >= 75 && i < 80 { // items in last page + want2 = append(want2, rr[1].(string)) + } + if i >= 35 && i < 75 { // items in both pages + want3 = append(want3, rr[1].(string)) + } + + tup := &BTreeTuple{ + TupleSchema: rowSchema, + Tuple: rr, + } + err = b.Insert(tup) + if err != nil { + t.Fatal(err) + } + } + + // b.Dot(os.Stdout, "first", false) + root, err := b.fetchNode(b.rootNode) + assert.Nil(t, err) + t.Run("first page", func(t *testing.T) { + itr := NewRangeIterator(b, root, Int(1), Int(10), b.schema) + defer itr.Dispose() + got := make([]string, 0) + for itr.Next() { + item, _ := itr.Item() + got = append(got, item.Tuple[0].(string)) + } + sort.Strings(got) + sort.Strings(want1) + assert.Equal(t, want1, got) + }) + t.Run("last page", func(t *testing.T) { + itr := NewRangeIterator(b, root, Int(75), Int(80), b.schema) + defer itr.Dispose() + got := make([]string, 0) + for itr.Next() { + item, _ := itr.Item() + got = append(got, item.Tuple[0].(string)) + } + sort.Strings(got) + sort.Strings(want2) + assert.Equal(t, want2, got) + }) + t.Run("cover pages", func(t *testing.T) { + itr := NewRangeIterator(b, root, Int(35), Int(75), b.schema) + defer itr.Dispose() + got := make([]string, 0) + for itr.Next() { + item, _ := itr.Item() + got = append(got, item.Tuple[0].(string)) + } + sort.Strings(got) + sort.Strings(want3) + assert.Equal(t, want3, got) + }) +} + +func TestDotOverflow(t *testing.T) { + t.Skip("need to figure out fail") + objectId := int32(1) + shard := int32(0) + dot, err := OpenBtree(fmt.Sprintf("ts-shard.%04d", shard)) + assert.NotNil(t, err) + dot.Dot(os.Stdout, "next", true) + + diskManager := bufferpool.NewTupleStoreDiskManager() + + dataFile := fmt.Sprintf("ts-shard.%04d", shard) + defer os.Remove(dataFile) + + diskManager.CreateOrOpenShard(objectId, shard, dataFile) + bufferPool := bufferpool.NewBufferPool(100, diskManager) + + tableSchema := make(types.Schema, 0) + + numCols := 100 + numRecs := 10 + + // build schema + for i := 0; i < numCols; i++ { + tableSchema = append(tableSchema, &types.PlannerColumn{ + ColumnName: fmt.Sprintf("vtest%d", i+1), + Type: parser.NewDataTypeVarchar(4), + }) + } + + b, err := NewBTree(8, objectId, shard, tableSchema, bufferPool) + _ = b + _ = numRecs + if err != nil { + t.Fatal(err) + } + rowSchema := make(types.Schema, 0) + + rowSchema = append(rowSchema, &types.PlannerColumn{ + ColumnName: "_id", + Type: parser.NewDataTypeID(), + }) + + for i := 0; i < numCols; i++ { + rowSchema = append(rowSchema, &types.PlannerColumn{ + ColumnName: fmt.Sprintf("vtest%d", i+1), + Type: parser.NewDataTypeVarchar(4), + }) + } + + inserts := make([]int, 0) + + for i := 1; i <= numRecs; i++ { + inserts = append(inserts, i) + } + + rr := make(types.Row, numCols+1) + + for _, i := range inserts { + rr[0] = int64(i) + + for j := 0; j < numCols; j++ { + rr[j+1] = fmt.Sprintf("%04d", j) + } + + tup := &BTreeTuple{ + TupleSchema: rowSchema, + Tuple: rr, + } + + // fmt.Printf("[%d]row key %v\n\n", j, i) + + err = b.Insert(tup) + if err != nil { + t.Fatal(err) + } + + } + + // b.Dump(0) + b.Dot(os.Stdout, "first", true) +} diff --git a/tstore/range.go b/tstore/range.go new file mode 100644 index 000000000..1dcae7d6a --- /dev/null +++ b/tstore/range.go @@ -0,0 +1,119 @@ +package tstore + +import ( + "github.com/featurebasedb/featurebase/v3/bufferpool" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" + "github.com/featurebasedb/featurebase/v3/vprint" +) + +type stackItem struct { + n *BTreeNode + i int +} + +type TstoreIterator interface { + Next() bool + Item() (*BTreeTuple, Sortable) + Dispose() +} + +type RangeIterator struct { + tree *BTree + node *BTreeNode + schema types.Schema + from Sortable + to Sortable + item *BTreeTuple + key Sortable + // + cursor int + done bool + err error +} + +func (b *BTree) NewRangeIterator(first, last Sortable) (*RangeIterator, error) { + root, err := b.fetchNode(b.rootNode) + if err != nil { + return nil, err + } + return NewRangeIterator(b, root, first, last, b.schema), nil +} + +func NewRangeIterator(tree *BTree, initialNode *BTreeNode, from, to Sortable, schema types.Schema) *RangeIterator { + r := &RangeIterator{ + tree: tree, + node: initialNode, + schema: schema, + from: from, + to: to, + } + r.Seek(from) + return r +} + +func (iter *RangeIterator) Dump() { + vprint.VV("id:%v sc:%v ls:%v", iter.node.page.ID(), iter.node.slotCount(), iter.node.latchState()) +} + +func (iter *RangeIterator) Dispose() { + if iter.node.latchState() != bufferpool.None { + iter.node.releaseReadLatch() + iter.tree.unpin(iter.node) + } +} + +func (iter *RangeIterator) Seek(key Sortable) { + iter.node, iter.err = iter.tree.fetchNode(iter.tree.rootNode) + if iter.err != nil { + return + } + iter.node.takeReadLatch() + iter.done, iter.err = iter.seek(iter.node, iter.schema, key) +} + +func (iter *RangeIterator) seek(currentNode *BTreeNode, schema types.Schema, key Sortable) (bool, error) { + if currentNode.isLeaf() { + iter.node = currentNode + c, _ := currentNode.findKey(key) + iter.cursor = c - 1 + return c == currentNode.slotCount(), nil + } + nodePtr := iter.tree.findNextPointer(currentNode, key) + node, err := iter.tree.fetchNode(nodePtr) + if err != nil { + return false, err + } + node.takeReadLatch() + currentNode.releaseReadLatch() + iter.tree.unpin(currentNode) + return iter.seek(node, schema, key) +} + +func (iter *RangeIterator) Item() (*BTreeTuple, Sortable) { + return iter.item, iter.key +} + +func (iter *RangeIterator) Next() bool { + if iter.done { + return false + } + iter.cursor++ + if iter.cursor == iter.node.slotCount() { + page := iter.node.page.ReadNextPointer() + iter.node.releaseReadLatch() + iter.tree.unpin(iter.node) + if page.Page == bufferpool.INVALID_PAGE { // at the end + return false + } + n, err := iter.tree.fetchNode(page) + iter.err = err + if iter.err != nil { + return false + } + iter.node = n + iter.node.takeReadLatch() + iter.cursor = 0 + } + iter.key, iter.item, iter.err = iter.tree.getTuple(iter.node, int(iter.cursor), iter.schema) + return iter.key.Less(iter.to) && iter.err == nil +}