scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Dataframe (#2241)

Browse source 
* Dataframe

(cherry picked from commit 2f1beaf119)
This commit is contained in:
tgruben 2022-11-21 17:38:48 -06:00 • committed by Fletcher Haynes
parent e2500fcee8
commit 2fc7abd72e
45 changed files with 6261 additions and 2912 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -82,4 +82,4 @@ staticcheck.conf
.quick
dax/dax-data
coverage-from-docker
coverage-from-docker

57
api.go
View file

@ -310,6 +310,31 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) {
return index, nil
}
func (api *API) DeleteDataframe(ctx context.Context, indexName string) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.DeleteDataframe")
defer span.Finish()
if err := api.validate(apiDeleteDataframe); err != nil {
return errors.Wrap(err, "validating api method")
}
// Delete index from the holder.
err := api.holder.DeleteDataframe(indexName)
if err != nil {
return errors.Wrap(err, "deleting index")
}
// Send the delete index message to all nodes.
err = api.server.SendSync(
&DeleteDataframeMessage{
Index: indexName,
})
if err != nil {
api.server.logger.Errorf("problem sending DeleteIndex message: %s", err)
return errors.Wrap(err, "sending DeleteIndex message")
}
api.holder.Stats.Count(MetricDeleteDataframe, 1, 1.0)
return nil
}
// DeleteIndex removes the named index. If the index is not found it does
// nothing and returns no error.
func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
@ -542,7 +567,6 @@ func importWorker(importWork chan importJob) {
}
err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear)
if err != nil {
return errors.Wrap(err, "importing standard roaring")
}
@ -1359,7 +1383,6 @@ func OptImportOptionsSuppressLog(b bool) ImportOption {
var ErrAborted = fmt.Errorf("error: update was aborted")
func (api *API) ImportAtomicRecord(ctx context.Context, qcx *Qcx, req *AtomicRecord, opts ...ImportOption) error {
simPowerLoss := false
lossAfter := -1
var opt ImportOptions
@ -2533,6 +2556,7 @@ func (api *API) WriteIDAllocDataTo(w io.Writer) error {
_, err := api.holder.ida.WriteTo(w)
return err
}
func (api *API) RestoreIDAlloc(r io.Reader) error {
return api.holder.ida.Replace(r)
}
@ -2584,7 +2608,7 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
}
idx := api.holder.Index(indexName)
//need to get a dbShard
// need to get a dbShard
dbs, err := api.holder.Txf().dbPerShard.GetDBShard(indexName, shard, idx)
if err != nil {
return err
@ -2592,7 +2616,7 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
db := dbs.W
finalPath := db.Path() + "/data"
tempPath := finalPath + ".tmp"
o, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
o, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
if err != nil {
return err
}
@ -2631,8 +2655,8 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
return err
}
defer tx.Rollback()
//arguments idx,shard do not matter for rbf they
//are ignored
// arguments idx,shard do not matter for rbf they
// are ignored
flvs, err := tx.GetSortedFieldViewList(idx, shard)
if err != nil {
return nil
@ -2811,8 +2835,8 @@ func (api *API) MutexCheck(ctx context.Context, qcx *Qcx, indexName string, fiel
useIndexKeys := index.Keys()
// If we're not doing details, we won't translate field keys even if we could.
useFieldKeys := field.Keys() && details
var indexKeys = map[uint64]string{}
var fieldKeys = map[uint64]string{}
indexKeys := map[uint64]string{}
fieldKeys := map[uint64]string{}
var indexIDs []uint64
var fieldIDs []uint64
// We'll use the string "untranslated" as our default value and overwrite
@ -3235,20 +3259,20 @@ const (
apiTranslateData
apiFieldTranslateData
apiField
//apiHosts // not implemented
// apiHosts // not implemented
apiImport
apiImportValue
apiIndex
//apiLocalID // not implemented
//apiLongQueryTime // not implemented
//apiMaxShards // not implemented
// apiLocalID // not implemented
// apiLongQueryTime // not implemented
// apiMaxShards // not implemented
apiQuery
apiRecalculateCaches
apiSchema
apiShardNodes
apiState
//apiStatsWithTags // not implemented
//apiVersion // not implemented
// apiStatsWithTags // not implemented
// apiVersion // not implemented
apiViews
apiApplySchema
apiStartTransaction
@ -3262,6 +3286,8 @@ const (
apiIDReset
apiPartitionNodes
apiMutexCheck
apiApplyChangeset
apiDeleteDataframe
)
var methodsCommon = map[apiMethod]struct{}{
@ -3322,6 +3348,8 @@ var methodsNormal = map[apiMethod]struct{}{
apiIDReset: {},
apiPartitionNodes: {},
apiMutexCheck: {},
apiApplyChangeset: {},
apiDeleteDataframe: {},
}
func shardInShards(i dax.ShardNum, s dax.VersionedShards) bool {
@ -3413,6 +3441,7 @@ func NewNopComputeAPI() *NopComputeAPI {
func (c *NopComputeAPI) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts ...ImportOption) error {
return nil
}
func (c *NopComputeAPI) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error {
return nil
}

623
apply.go Normal file
View file

@ -0,0 +1,623 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"github.com/apache/arrow/go/v10/arrow"
"github.com/apache/arrow/go/v10/arrow/array"
"github.com/apache/arrow/go/v10/arrow/memory"
"github.com/apache/arrow/go/v10/parquet/file"
"github.com/apache/arrow/go/v10/parquet/pqarrow"
"github.com/gomem/gomem/pkg/dataframe"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/tracing"
"github.com/molecula/featurebase/v3/vprint"
"github.com/pkg/errors"
ivy "robpike.io/ivy/arrow"
config "robpike.io/ivy/config"
"robpike.io/ivy/exec"
"robpike.io/ivy/parse"
"robpike.io/ivy/run"
"robpike.io/ivy/scan"
"robpike.io/ivy/value"
)
type (
ApplyResult *arrow.Column
)
func runIvyString(context value.Context, str string) (ok bool, err error) {
defer func() {
if r := recover(); r != nil {
err = r.(value.Error)
}
}()
scanner := scan.New(context, "<args>", strings.NewReader(str))
parser := parse.NewParser("<args>", scanner, context)
ok = run.Run(parser, context, false)
return
}
// Possibly combine all arrays together then apply some interesting
// computation at the end?
func IvyReduce(reduceCode string, opCode string, opt *ExecOptions) (func(ctx context.Context, prev, v interface{}) interface{}, func() (*dataframe.DataFrame, error)) {
var accumulator value.Value
mu := &sync.Mutex{}
concat := value.BinaryOps[opCode]
conf := getDefaultConfig()
ctxIvy := exec.NewContext(&conf)
// concat returned results at coordinating node.
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
if v == nil {
return prev
}
if accumulator == nil {
switch val := v.(type) {
case *dataframe.DataFrame:
col := val.ColumnAt(0)
resolver := dataframe.NewChunkResolver(col)
accumulator = value.NewArrowVector(col, &conf, &resolver)
case value.Value:
accumulator = v.(value.Value)
default:
return errors.New(fmt.Sprintf("ivy reduction failed first unexpected type %T", v))
}
return nil
}
switch val := v.(type) {
case *dataframe.DataFrame:
col := val.ColumnAt(0)
resolver := dataframe.NewChunkResolver(col)
x := value.NewArrowVector(col, &conf, &resolver)
mu.Lock() // i'm being overyerly cautious..need to confirm this can be concurrent
accumulator = concat.EvalBinary(ctxIvy, accumulator, x)
mu.Unlock()
case value.Value:
mu.Lock()
accumulator = concat.EvalBinary(ctxIvy, accumulator, val)
mu.Unlock()
default:
return errors.New(fmt.Sprintf("ivy reduction failed unexpected type %T", v))
}
return nil
}
tablerFn := func() (*dataframe.DataFrame, error) {
pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
if opt.Remote {
col := value.ToArrowColumn(accumulator, pool)
return dataframe.NewDataFrameFromColumns(pool, []arrow.Column{*col})
}
// only acutally reduce on the initiating node i hate the network
// over head but oh well
ctxIvy.AssignGlobal("_", accumulator)
ok, err := runIvyString(ctxIvy, reduceCode)
if err != nil {
return nil, err
}
if ok {
v := ctxIvy.Global("_")
if v == nil {
return nil, errors.New("ivy reduction no result ")
}
col := value.ToArrowColumn(ctxIvy.Global("_"), pool)
return dataframe.NewDataFrameFromColumns(pool, []arrow.Column{*col})
}
return nil, errors.New("ivy reduction failed ")
}
return reduceFn, tablerFn
}
// executeApply executes a Apply() call.
func (e *executor) executeApply(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*dataframe.DataFrame, error) {
if !e.dataframeEnabled {
return nil, errors.New("Dataframe support not enabled")
}
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMax")
defer span.Finish()
if _, err := c.FirstStringArg("_ivy"); err != nil {
return nil, errors.Wrap(err, " no ivy program supplied")
}
if len(c.Children) > 1 {
return nil, errors.New("Apply() only accepts a single bitmap input filter")
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(ctx context.Context, shard uint64, mopt *mapOptions) (_ interface{}, err error) {
return e.executeApplyShard(ctx, qcx, index, c, shard)
}
ivyReduce, ok, err := c.StringArg("_ivyReduce")
if err != nil {
return nil, err
}
reduceFn, tablerFn := IvyReduce("_", ",", opt)
if ok {
reduceFn, tablerFn = IvyReduce(ivyReduce, ",", opt)
}
_, err = e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return nil, err
}
return tablerFn()
}
func getDefaultConfig() config.Config {
maxbits := uint(1e9) // "maximum size of an integer, in bits; 0 means no limit")
maxdigits := uint(1e4) // "above this many `digits`, integers print as floating point; 0 disables")
maxstack := uint(100000)
origin := 1 // "set index origin to `n` (must be 0 or 1)")
prompt := "" // flag.String("prompt", "", "command `prompt`")
format := ""
// debugFlag := "" // flag.String("debug", "", "comma-separated `names` of debug settings to enable")
conf := config.Config{}
conf.SetFormat(format)
conf.SetMaxBits(maxbits)
conf.SetMaxDigits(maxdigits)
conf.SetMaxStack(maxstack)
conf.SetOrigin(origin)
conf.SetPrompt(prompt)
conf.SetOutput(io.Discard)
conf.SetErrOutput(io.Discard)
conf.SetEmbedded(true) // needed to propagate panic
return conf
}
func filterDataframe(resolver dataframe.Resolver, pool memory.Allocator, filter []int64) (*dataframe.IndexResolver, error) {
if resolver.NumRows() == 0 {
return nil, errors.New("No data")
}
indexResolver := dataframe.NewIndexResolver(len(filter), uint32(ShardWidth-1))
for i, id := range filter {
if int(id) >= resolver.NumRows() {
continue
}
c, o := resolver.Resolve(int(id))
indexResolver.Set(i, c, o)
}
return indexResolver, nil
}
func (e *executor) executeApplyShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (value.Value, error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeApplyShard")
defer span.Finish()
ivyProgram, ok, err := c.StringArg("_ivy")
if err != nil || !ok {
return nil, errors.Wrap(err, "finding ivy program")
}
var filter *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard)
if err != nil {
return nil, err
}
filter = row
if !filter.Any() {
// no need to actuall run the query for its not operating against any values
return value.NewVector([]value.Value{}), nil
}
}
//
pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
ids := filter.ShardColumns() // needs to be shard columns
// Fetch index.
idx := e.Holder.Index(index)
if idx == nil {
return nil, newNotFoundError(ErrIndexNotFound, index)
}
fname := idx.GetDataFramePath(shard)
if _, err := os.Stat(fname + ".parquet"); os.IsNotExist(err) {
return value.NewVector([]value.Value{}), nil
}
table, err := readTableParquet(fname)
if err != nil {
return nil, err
}
defer table.Release()
df, err := dataframe.NewDataFrameFromTable(pool, table)
if err != nil {
return nil, err
}
p := dataframe.NewChunkResolver(df.ColumnAt(0))
var resolver dataframe.Resolver
resolver = &p
if filter != nil {
if len(ids) == 0 {
return value.NewVector([]value.Value{}), nil
}
resolver, err = filterDataframe(resolver, pool, ids)
if err != nil {
return nil, err
}
}
conf := getDefaultConfig()
context, err := ivy.RunArrow(dataframe.NewTableFacade(df), ivyProgram, conf, resolver)
if err != nil {
return nil, fmt.Errorf("ivy map error: %w", err)
}
return context.Global("_"), nil
}
func readTableParquet(filename string) (arrow.Table, error) {
r, err := os.Open(filename + ".parquet")
if err != nil {
return nil, err
}
pf, err := file.NewParquetReader(r)
if err != nil {
return nil, err
}
reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, memory.DefaultAllocator)
if err != nil {
return nil, err
}
return reader.ReadTable(context.Background())
}
func readTableParquetCtx(ctx context.Context, filename string, mem memory.Allocator) (arrow.Table, error) {
r, err := os.Open(filename + ".parquet")
if err != nil {
return nil, err
}
pf, err := file.NewParquetReader(r)
if err != nil {
return nil, err
}
reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, mem)
if err != nil {
return nil, err
}
return reader.ReadTable(ctx)
}
// ///////////////////////////////////////////////////////
// all the ingest supporting functions
// ///////////////////////////////////////////////////////
func NewShardFile(name string) (*ShardFile, error) {
if _, err := os.Stat(name + ".parquet"); os.IsNotExist(err) {
return &ShardFile{dest: name}, nil
}
// else read in existing
table, err := readTableParquet(name)
if err != nil {
return nil, err
}
return &ShardFile{table: table, schema: table.Schema(), dest: name}, nil
}
type NameType struct {
Name string
DataType arrow.DataType
}
type ChangesetRequest struct {
ShardIds []int64 // only shardwidth bits to provide 0 indexing inside shard file
Columns []interface{}
SimpleSchema []NameType
}
// TODO(twg) 2022/09/30 Needs a refactor
func cast(v interface{}) arrow.DataType {
switch v.(type) {
case *arrow.Int64Type:
return arrow.PrimitiveTypes.Int64
case int64:
return arrow.PrimitiveTypes.Int64
case *arrow.Float64Type:
return arrow.PrimitiveTypes.Float64
case float64:
return arrow.PrimitiveTypes.Float64
default:
vprint.VV("%T .... %v", v, v)
}
return arrow.PrimitiveTypes.Int64
}
func (cr *ChangesetRequest) ArrowSchema() *arrow.Schema {
fields := make([]arrow.Field, len(cr.SimpleSchema))
for i := range cr.SimpleSchema {
fields[i] = arrow.Field{Name: cr.SimpleSchema[i].Name, Type: cast(cr.SimpleSchema[i].DataType)}
}
return arrow.NewSchema(fields, nil)
}
type ShardFile struct {
table arrow.Table
schema *arrow.Schema
beforeRows int64
added int64
columns []interface{}
dest string
}
func compareSchema(s1, s2 *arrow.Schema) bool {
if s1 == nil || s2 == nil {
return false
}
if len(s1.Fields()) != len(s2.Fields()) {
return false
}
for i := 0; i < len(s1.Fields()); i++ {
f1 := s1.Field(i)
f2 := s2.Field(i)
if f1.Name != f2.Name {
return false
}
if f1.Type != f2.Type {
return false
}
}
return true
}
func (sf *ShardFile) EnsureSchema(cs *ChangesetRequest) error {
schema := cs.ArrowSchema()
if sf.schema == nil {
sf.schema = schema
} else {
if !compareSchema(sf.schema, schema) {
vprint.VV("incomeing schema", schema)
vprint.VV("existing schema", sf.schema)
return errors.New("dataframe schema's don't match")
}
}
sf.columns = make([]interface{}, len(sf.schema.Fields()))
return nil
}
func (sf *ShardFile) buildAppenders(maxid int64) {
if sf.table != nil {
sf.beforeRows = sf.table.NumRows()
}
if maxid < sf.beforeRows {
// no need to add new rows
return
}
newSize := maxid - sf.beforeRows + 1
for i := 0; i < len(sf.schema.Fields()); i++ {
switch sf.schema.Field(i).Type {
case arrow.PrimitiveTypes.Int64:
sf.columns[i] = make([]int64, newSize)
case arrow.PrimitiveTypes.Float64:
sf.columns[i] = make([]float64, newSize)
}
}
sf.added = newSize
}
// the row offset must be reset to 0 for the slices being appended
func (sf *ShardFile) SetIntValue(col int, row int64, val int64) {
v := sf.columns[col].([]int64)
v[row-sf.beforeRows] = val
}
func (sf *ShardFile) SetFloatValue(col int, row int64, val float64) {
v := sf.columns[col].([]float64)
v[row-sf.beforeRows] = val
}
func (sf *ShardFile) Process(cs *ChangesetRequest) error {
err := sf.process(cs)
if err != nil {
return err
}
rtemp := sf.dest + ".temp"
err = sf.Save(rtemp)
if err != nil {
return err
}
return os.Rename(rtemp+".parquet", sf.dest+".parquet")
}
func (sf *ShardFile) process(cs *ChangesetRequest) error {
offset := 0
if sf.table != nil {
column := sf.table.Column(0)
resolver := dataframe.NewChunkResolver(column)
for i, rowid := range cs.ShardIds {
offset = i
if rowid >= sf.table.NumRows() {
break
}
chunk, l := resolver.Resolve(int(rowid))
for col := 0; col < len(sf.schema.Fields()); col++ {
column := sf.table.Column(col)
switch column.DataType() {
case arrow.PrimitiveTypes.Int64:
v := column.Data().Chunk(chunk).(*array.Int64).Int64Values()
v[l] = cs.Columns[col].([]int64)[i]
case arrow.PrimitiveTypes.Float64:
v := column.Data().Chunk(chunk).(*array.Float64).Float64Values()
v[l] = cs.Columns[col].([]float64)[i]
default:
panic(fmt.Sprintf("Unknown Type %v", column.DataType()))
}
}
}
}
max := cs.ShardIds[len(cs.ShardIds)-1]
sf.buildAppenders(max)
// need to check if only replace and no apend
if sf.added > 0 {
for i, rowid := range cs.ShardIds[offset:] {
i += offset
for col := 0; col < len(sf.schema.Fields()); col++ {
switch sf.schema.Field(col).Type {
case arrow.PrimitiveTypes.Int64:
sf.SetIntValue(col, rowid, cs.Columns[col].([]int64)[i])
case arrow.PrimitiveTypes.Float64:
sf.SetFloatValue(col, rowid, cs.Columns[col].([]float64)[i])
default:
panic(fmt.Sprintf("2 Unknown Type %v", sf.schema.Field(col).Type))
}
}
}
}
return nil
}
func (sf *ShardFile) Save(name string) error {
parts := make([]arrow.Array, 0)
mem := memory.NewGoAllocator()
for col := 0; col < len(sf.schema.Fields()); col++ {
chunks := make([]arrow.Array, 0)
if sf.table != nil {
// we append if there was existing parquet file
column := sf.table.Column(col)
chunks = append(chunks, column.Data().Chunks()...)
}
switch sf.schema.Field(col).Type {
case arrow.PrimitiveTypes.Int64:
// case *arrow.Int64Type:
if sf.added > 0 {
ibuild := array.NewInt64Builder(mem)
ibuild.AppendValues(sf.columns[col].([]int64), nil) // TODO(twg) 2022/09/28 need to handle null
newChunk := ibuild.NewArray()
chunks = append(chunks, newChunk)
}
record, err := array.Concatenate(chunks, mem)
if err != nil {
return err
}
parts = append(parts, record)
case arrow.PrimitiveTypes.Float64:
// case *arrow.Float64Type:
if sf.added > 0 {
fbuild := array.NewFloat64Builder(mem)
fbuild.AppendValues(sf.columns[col].([]float64), nil) // TODO(twg) 2022/09/28 need to handle null
newChunk := fbuild.NewArray()
chunks = append(chunks, newChunk)
}
record, err := array.Concatenate(chunks, mem)
if err != nil {
return err
}
parts = append(parts, record)
default:
vprint.VV("UNKNOWN %T", sf.schema.Field(col).Type)
}
}
rec := array.NewRecord(sf.schema, parts, sf.beforeRows+sf.added)
df, err := dataframe.NewDataFrameFromRecord(mem, rec)
if err != nil {
return err
}
// confirm change
w, err := os.Create(name + ".parquet")
if err != nil {
return err
}
err = df.ToParquet(w, 1024)
if err != nil {
return err
}
w.Close()
return nil
}
// TODO(twg) 2022/10/03 Not a huge fan of the global variable will look at adding to executor structure
// when dataframe is fully integrated
var (
dataframeShardLocks map[uint64]*sync.Mutex
muWriteDataframe sync.Mutex
)
func init() {
dataframeShardLocks = make(map[uint64]*sync.Mutex)
}
func getDataframeWritelock(shard uint64) *sync.Mutex {
muWriteDataframe.Lock()
defer muWriteDataframe.Unlock()
lock, ok := dataframeShardLocks[shard]
if ok {
return lock
}
newLock := sync.Mutex{}
dataframeShardLocks[shard] = &newLock
return &newLock
}
func (api *API) ApplyDataframeChangeset(ctx context.Context, index string, cs *ChangesetRequest, shard uint64) error {
// TODO(twg) 2022/09/29 need to validate api call
idx := api.Holder().Index(index)
// check if dataframe exists
fname := idx.GetDataFramePath(shard)
// only 1 shard writer allowed at at time so wait for it to be available
mu := getDataframeWritelock(shard)
mu.Lock()
defer mu.Unlock()
shardFile, err := NewShardFile(fname)
if err != nil {
return err
}
err = shardFile.EnsureSchema(cs)
if err != nil {
return err
}
return shardFile.Process(cs)
}
type column struct {
Name string
Type string
}
func (api *API) GetDataframeSchema(ctx context.Context, indexName string) (interface{}, error) {
idx, err := api.Index(ctx, indexName)
if err != nil {
return nil, err
}
base := idx.DataframesPath()
dir, _ := os.Open(base)
files, _ := dir.Readdir(0)
parts := make([]column, 0)
for i := range files {
file := files[i]
name := file.Name()
if strings.HasSuffix(name, ".parquet") {
// strip off the parquet extenison
name = strings.TrimSuffix(name, filepath.Ext(name))
// read the parquet file and extract the schema
table, err := readTableParquet(filepath.Join(base, name))
if err != nil {
return nil, err
}
for i := 0; i < int(table.NumCols()); i++ {
col := table.Column(i)
part := column{Name: col.Name(), Type: col.DataType().String()}
parts = append(parts, part)
}
break // only go on first file
}
}
return parts, nil
}

405
arrow.go Normal file
View file

@ -0,0 +1,405 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"context"
"encoding/json"
"fmt"
"os"
"sync"
"github.com/apache/arrow/go/v10/arrow"
"github.com/apache/arrow/go/v10/arrow/array"
"github.com/apache/arrow/go/v10/arrow/memory"
"github.com/gomem/gomem/pkg/dataframe"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/tracing"
"github.com/pkg/errors"
)
/*
The function Arrow provides filtered access to the raw values stored in the dataframe.
If Arrow is just provided a bitmap filter, such as ConstRow or any Bitmap Operation,
all the values associated with each column are returned. This set can be limited with
the addition of the header parameter
Example:
Arrow(ConstRow(columns=[2,4,6]),header=["fval"])
*/
// executeApply executes a Arrow() call.
func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (arrow.Table, error) {
if !e.dataframeEnabled {
return nil, errors.New("Dataframe support not enabled")
}
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeArrow")
defer span.Finish()
if len(c.Children) > 1 {
return nil, errors.New("Apply() only accepts a single bitmap input filter")
}
var columnFilter []string
if cols, ok := c.Args["header"].([]interface{}); ok {
columnFilter = make([]string, 0, len(cols))
for _, v := range cols {
columnFilter = append(columnFilter, v.(string))
}
}
mapcounter := 0
reducecounter := 0
pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
// Execute calls in bulk on each remote node and merge.
mu := &sync.Mutex{}
mapFn := func(ctx context.Context, shard uint64, mopt *mapOptions) (_ interface{}, err error) {
mu.Lock()
mapcounter++
mu.Unlock()
return e.executeArrowShard(ctx, qcx, index, c, shard, pool, columnFilter)
}
tables := make([]*basicTable, 0)
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
mu.Lock()
reducecounter++
mu.Unlock()
if v == nil {
return prev
}
switch t := v.(type) {
case *basicTable:
if t.resolver != nil {
mu.Lock()
tables = append(tables, t)
mu.Unlock()
}
case arrow.Table:
if t.NumRows() > 0 {
bt := BasicTableFromArrow(t, pool)
mu.Lock()
tables = append(tables, bt)
mu.Unlock()
}
}
return nil
}
_, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return nil, err
}
if len(tables) == 0 {
return &basicTable{name: "empty"}, nil
}
tbl := Concat(tables[0].Schema(), tables, pool)
r := dataframe.NewChunkResolver(tbl.Column(0))
return &basicTable{resolver: &r, table: tbl}, nil
}
type basicTable struct {
resolver dataframe.Resolver
table arrow.Table
filtered bool
name string
}
func (st *basicTable) Name() string {
return st.name
}
func (st *basicTable) Schema() *arrow.Schema {
if st.table != nil {
return st.table.Schema()
}
return &arrow.Schema{}
}
func (st *basicTable) IsFiltered() bool {
return st.filtered
}
func (st *basicTable) NumRows() int64 {
if st.resolver == nil {
return 0
}
return int64(st.resolver.NumRows())
}
func (st *basicTable) NumCols() int64 {
if st.table != nil {
return st.table.NumCols()
}
return 0
}
func (st *basicTable) Column(i int) *arrow.Column {
if st.table != nil {
return st.table.Column(i)
}
return nil
}
func (st *basicTable) Retain() {
if st.table != nil {
st.table.Retain()
}
}
func (st *basicTable) Release() {
if st.table != nil {
st.table.Retain()
}
}
func (st *basicTable) Get(column, row int) interface{} {
field := st.Schema().Field(column)
c, i := st.resolver.Resolve(row)
chunk := st.Column(column).Data().Chunk(c)
switch field.Type.(type) {
// case *arrow.BooleanType:
// v := chunk.(*array.Boolean).BooleanValues()
// return v[i]
case *arrow.Int8Type:
v := chunk.(*array.Int8).Int8Values()
return v[i]
case *arrow.Int16Type:
v := chunk.(*array.Int16).Int16Values()
return v[i]
case *arrow.Int32Type:
v := chunk.(*array.Int32).Int32Values()
return v[i]
case *arrow.Int64Type:
v := chunk.(*array.Int64).Int64Values()
return v[i]
case *arrow.Uint8Type:
v := chunk.(*array.Uint8).Uint8Values()
return v[i]
case *arrow.Uint16Type:
v := chunk.(*array.Uint16).Uint16Values()
return v[i]
case *arrow.Uint32Type:
v := chunk.(*array.Uint32).Uint32Values()
return v[i]
case *arrow.Uint64Type:
v := chunk.(*array.Uint64).Uint64Values()
return v[i]
case *arrow.Float32Type:
v := chunk.(*array.Float32).Float32Values()
return v[i]
case *arrow.Float64Type:
v := chunk.(*array.Float64).Float64Values()
return v[i]
}
return 0
}
func builderFrom(mem memory.Allocator, dt arrow.DataType, size int64) array.Builder {
var bldr array.Builder
switch dt := dt.(type) {
case *arrow.BooleanType:
bldr = array.NewBooleanBuilder(mem)
case *arrow.Int8Type:
bldr = array.NewInt8Builder(mem)
case *arrow.Int16Type:
bldr = array.NewInt16Builder(mem)
case *arrow.Int32Type:
bldr = array.NewInt32Builder(mem)
case *arrow.Int64Type:
bldr = array.NewInt64Builder(mem)
case *arrow.Uint8Type:
bldr = array.NewUint8Builder(mem)
case *arrow.Uint16Type:
bldr = array.NewUint16Builder(mem)
case *arrow.Uint32Type:
bldr = array.NewUint32Builder(mem)
case *arrow.Uint64Type:
bldr = array.NewUint64Builder(mem)
case *arrow.Float32Type:
bldr = array.NewFloat32Builder(mem)
case *arrow.Float64Type:
bldr = array.NewFloat64Builder(mem)
default:
panic(fmt.Errorf("npy2root: invalid Arrow type %v", dt))
}
bldr.Reserve(int(size))
return bldr
}
func appendData(bldr array.Builder, v interface{}) {
switch bldr := bldr.(type) {
case *array.BooleanBuilder:
bldr.Append(v.(bool))
case *array.Int8Builder:
bldr.Append(v.(int8))
case *array.Int16Builder:
bldr.Append(v.(int16))
case *array.Int32Builder:
bldr.Append(v.(int32))
case *array.Int64Builder:
bldr.Append(v.(int64))
case *array.Uint8Builder:
bldr.Append(v.(uint8))
case *array.Uint16Builder:
bldr.Append(v.(uint16))
case *array.Uint32Builder:
bldr.Append(v.(uint32))
case *array.Uint64Builder:
bldr.Append(v.(uint64))
case *array.Float32Builder:
bldr.Append(v.(float32))
case *array.Float64Builder:
bldr.Append(v.(float64))
default:
panic(fmt.Errorf("npy2root: invalid Arrow builder type %T", bldr))
}
}
func Concat(schema *arrow.Schema, tables []*basicTable, mem memory.Allocator) arrow.Table {
if len(tables) == 1 {
if !tables[0].IsFiltered() {
return tables[0]
}
}
cols := make([]arrow.Column, len(schema.Fields()))
defer func(cols []arrow.Column) {
for i := range cols {
cols[i].Release()
}
}(cols)
sz := 0
for i := range tables {
sz += int(tables[i].NumRows())
}
for i := range cols {
field := schema.Field(i)
arrs := make([]arrow.Array, 0)
builder := builderFrom(mem, field.Type, int64(sz))
for t := range tables {
table := tables[t]
if table.IsFiltered() {
for row := 0; row < int(table.NumRows()); row++ {
v := table.Get(i, row)
appendData(builder, v)
}
arrs = append(arrs, builder.NewArray())
} else {
parts := table.Column(i).Data()
arrs = append(arrs, parts.Chunks()...)
}
}
chunk := arrow.NewChunked(field.Type, arrs)
cols[i] = *arrow.NewColumn(field, chunk)
chunk.Release()
}
return array.NewTable(schema, cols, -1)
}
func (st *basicTable) MarshalJSON() ([]byte, error) {
results := make(map[string]interface{})
n := 0
if st.table != nil {
n = int(st.table.NumCols())
}
for b := 0; b < n; b++ {
col := st.table.Column(b)
result := make([]interface{}, st.resolver.NumRows())
for n := st.resolver.NumRows() - 1; n >= 0; n-- {
v := st.Get(b, n)
result[n] = v
}
results[col.Name()] = result
}
return json.Marshal(results)
}
func BasicTableFromArrow(table arrow.Table, mem memory.Allocator) *basicTable {
col := table.Column(0)
r := dataframe.NewChunkResolver(col)
return &basicTable{resolver: &r, table: table}
}
func filterColumns(filters []string, table arrow.Table) arrow.Table {
filters = append(filters, "_ID")
schema := table.Schema()
// TODO(twg) 2022/11/09 add glob support
allFields := schema.Fields()
in := func(key string) bool {
for _, v := range filters {
if v == key {
return true
}
}
return false
}
cols := make([]arrow.Column, 0)
fields := make([]arrow.Field, 0)
for i := range allFields {
field := allFields[i]
if in(field.Name) {
cols = append(cols, *table.Column(i))
fields = append(fields, field)
}
}
filterdSchema := arrow.NewSchema(fields, nil) // TODO(twg) 2022/11/09 handle meta:w
return array.NewTable(filterdSchema, cols, table.NumRows())
}
func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*basicTable, error) {
name := fmt.Sprintf("a. %v", shard)
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeArrowShard")
defer span.Finish()
var filter *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard)
if err != nil {
return nil, err
}
filter = row
if !filter.Any() {
// no need to actuall run the query for its not operating against any values
return &basicTable{name: name}, nil
}
}
//
ids := filter.ShardColumns() // needs to be shard columns
// Fetch index.
idx := e.Holder.Index(index)
if idx == nil {
return nil, newNotFoundError(ErrIndexNotFound, index)
}
fname := idx.GetDataFramePath(shard)
if _, err := os.Stat(fname + ".parquet"); os.IsNotExist(err) {
return &basicTable{name: name}, nil
}
table, err := readTableParquetCtx(context.TODO(), fname, pool)
if err != nil {
return nil, errors.Wrap(err, "arrow readTableParquet")
}
defer table.Release()
if len(columnFilter) > 0 {
table = filterColumns(columnFilter, table)
}
df, err := dataframe.NewDataFrameFromTable(pool, table)
if err != nil {
return nil, errors.Wrap(err, "arrow NewDataFromTable")
}
p := dataframe.NewChunkResolver(df.ColumnAt(0))
var resolver dataframe.Resolver
resolver = &p
if filter != nil {
if len(ids) == 0 {
return &basicTable{name: name}, nil
}
resolver, err = filterDataframe(resolver, pool, ids)
if err != nil {
return nil, errors.Wrap(err, "filtering dataframe")
}
}
table.Retain()
return &basicTable{resolver: resolver, table: table, filtered: filter != nil, name: name}, nil
}

5
authn/authenticate.go
View file

@ -148,7 +148,6 @@ func (a *Auth) refreshToken(access, refresh string) (string, string, error) {
"client_secret": {a.oAuthConfig.ClientSecret},
},
)
if err != nil {
return "", "", errors.Wrap(err, "refreshing token")
}
@ -367,7 +366,7 @@ func (a *Auth) SetCookie(w http.ResponseWriter, access, refresh string, expiry t
return nil
}
func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, access, refresh string) error {
func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, access, refresh string) (context.Context, error) {
mCookies := map[string]string{}
if c, ok := md["cookie"]; ok {
for _, cookie := range c {
@ -385,7 +384,7 @@ func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, access, refr
}
md["cookie"] = cookies
return grpc.SetHeader(ctx, md)
return metadata.NewIncomingContext(ctx, md), grpc.SetHeader(ctx, md)
}
func decodeHex(hexstr string) ([]byte, error) {

7
authn/authenticate_internal_test.go
View file

@ -78,7 +78,7 @@ func TestSetGRPCMetadata(t *testing.T) {
if !ok {
t.Fatalf("expected ok, got: %v", ok)
}
err := a.SetGRPCMetadata(ctx, md, "accesstoken!", "refreshtoken!")
ctx, err := a.SetGRPCMetadata(ctx, md, "accesstoken!", "refreshtoken!")
if err != nil {
t.Fatalf("expected no errors, got: %v", err)
}
@ -377,7 +377,6 @@ func TestAuthenticate_CleanCache(t *testing.T) {
t.Errorf("goldy should not have been deleted")
}
})
}
func TestGetGroups(t *testing.T) {
@ -463,7 +462,6 @@ func TestGetGroups(t *testing.T) {
}
})
}
}
func TestDecodeHex(t *testing.T) {
@ -615,7 +613,6 @@ func TestHandlers(t *testing.T) {
t.Fatalf("expected BadRequest, got %v", resp.StatusCode)
}
})
}
// This type is used for mocking ServerTransportStreams in tests
@ -667,7 +664,6 @@ func assertEqual(exp, got interface{}, t *testing.T) {
}
func TestCheckAllowedNetworks(t *testing.T) {
tests := []struct {
requestIP string
configuredIPs []string
@ -721,7 +717,6 @@ func TestCheckAllowedNetworks(t *testing.T) {
}
func TestConvertIP(t *testing.T) {
tests := []struct {
configuredIPs []string
convertedIPs []net.IPNet

5
broadcast.go
View file

@ -72,6 +72,7 @@ const (
messageTypeUNUSED2 // used to be ResizeNodeMessage
messageTypeUNUSED3 // used to be ResizeAbortMessage
messageTypeUpdateField
messageTypeDeleteDataframe
)
// MarshalInternalMessage serializes the pilosa message and adds pilosa internal
@ -117,6 +118,8 @@ func getMessage(typ byte) Message {
return &TransactionMessage{}
case messageTypeUpdateField:
return &UpdateFieldMessage{}
case messageTypeDeleteDataframe:
return &DeleteDataframeMessage{}
default:
panic(fmt.Sprintf("unknown message type %d", typ))
}
@ -154,6 +157,8 @@ func getMessageType(m Message) byte {
return messageTypeTransaction
case *UpdateFieldMessage:
return messageTypeUpdateField
case *DeleteDataframeMessage:
return messageTypeDeleteDataframe
default:
panic(fmt.Sprintf("don't have type for message %#v", m))
}

57
client/client.go
View file

@ -7,6 +7,7 @@ package client
import (
"bytes"
"crypto/tls"
"encoding/gob"
"encoding/json"
"fmt"
"io"
@ -38,6 +39,7 @@ import (
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/vprint"
"github.com/opentracing/opentracing-go"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
@ -228,7 +230,6 @@ func newClientWithOptions(options *ClientOptions) *Client {
c.maxBackoff = 2 * time.Minute
go c.runChangeDetection()
return c
}
// NewClient creates a client with the given address, URI, or cluster and options.
@ -1801,3 +1802,57 @@ func (r *exportReader) Read(p []byte) (n int, err error) {
func (c *Client) SetAuthToken(token string) {
c.AuthToken = token
}
// ApplyDataframeChangeset will create or append the existing dataframe
// if creating, the provided schema will be used
// if appending, the provided schema will be used to validate against
// Only one shard can be written to at a time so slight care must be taken
// Server side locking will prevent writting conflict
func (c *Client) ApplyDataframeChangeset(indexName string, cr *pilosa.ChangesetRequest, shard uint64) (map[string]interface{}, error) {
path := fmt.Sprintf("/index/%s/dataframe/%d", indexName, shard)
headers := c.augmentHeaders(map[string]string{
"Content-Type": "application/octet-stream",
"Accept": "application/json",
})
var buffer bytes.Buffer
enc := gob.NewEncoder(&buffer)
err := enc.Encode(cr)
if err != nil {
return nil, errors.Wrap(err, "encoding changesetrequest")
}
uris, err := c.getURIsForShard(indexName, shard)
if err != nil {
return nil, errors.Wrap(err, "getting URIs for import")
}
eg := errgroup.Group{}
data := buffer.Bytes()
for _, uri := range uris {
uri := uri
eg.Go(func() error {
vprint.VV("SENDING: %v", uri)
status, body, err := c.doRequest(uri, "POST", path, headers, data)
if err != nil {
return errors.Wrap(err, "executing request")
}
if status != http.StatusOK {
return errors.Errorf("find dataframe request failed (status: %d): %q", status, string(body))
}
return nil
})
}
err = eg.Wait()
// status, body, err := c.HTTPRequest(http.MethodPost, path, buffer.Bytes(), headers)
/*
var result map[string]interface{}
err = json.Unmarshal(body, &result)
if err != nil {
return nil, errors.Wrap(err, "unmarshalling response")
}
*/
return nil, err
}

5
cluster.go
View file

@ -1001,3 +1001,8 @@ func intInPartitions(i int, s dax.VersionedPartitions) bool {
}
return false
}
// DeleteDataframeMessage is an internal message indicating dataframe deletion.
type DeleteDataframeMessage struct {
Index string
}

46
ctl/backup.go
View file

@ -10,6 +10,7 @@ import (
"fmt"
"io"
"io/fs"
"log"
"os"
"path/filepath"
"time"
@ -316,6 +317,14 @@ func (cmd *BackupCommand) backupShard(ctx context.Context, indexName string, sha
for _, node := range nodes {
if e := cmd.backupShardNode(ctx, indexName, shard, node); e == nil {
break
} else if err == nil {
err = e // save first error, try next node
}
}
for _, node := range nodes {
if e := cmd.backupShardDataframe(ctx, indexName, shard, node); e == nil {
return nil // backup ok, exit
} else if err == nil {
err = e // save first error, try next node
@ -522,3 +531,40 @@ func (cmd *BackupCommand) syncDir(path string) error {
return f.Close()
}
func (cmd *BackupCommand) backupShardDataframe(ctx context.Context, indexName string, shard uint64, node *disco.Node) error {
logger := cmd.Logger()
logger.Printf("backing up dataframe shard: index=%q shard=%d", indexName, shard)
client := pilosa.NewInternalClientFromURI(&node.URI,
pilosa.GetHTTPClient(cmd.tlsConfig, pilosa.ClientResponseHeaderTimeoutOption(cmd.HeaderTimeout)),
)
resp, err := client.GetDataframeShard(ctx, indexName, shard)
// no error if doesn't exist
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode == 404 {
// no error if not present server maynot have it turned on
return nil
}
filename := filepath.Join(cmd.OutputDir, "indexes", indexName, "dataframe", fmt.Sprintf("%04d", shard))
if err := os.MkdirAll(filepath.Dir(filename), 0o750); err != nil {
return err
}
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()
if _, err := io.Copy(f, resp.Body); err != nil {
return err
} else if err := cmd.syncFile(f); err != nil {
return err
}
return f.Close()
}

75
ctl/backup_tar.go
View file

@ -9,15 +9,18 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"os"
"path"
"path/filepath"
"time"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/authn"
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/featurebasedb/featurebase/v3/encoding/proto"
"github.com/featurebasedb/featurebase/v3/server"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/authn"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/vprint"
"github.com/pkg/errors"
)
@ -184,7 +187,7 @@ func (cmd *BackupTarCommand) backupTarSchema(ctx context.Context, tw *tar.Writer
// Build header & copy data to archive.
if err = tw.WriteHeader(&tar.Header{
Name: "schema",
Mode: 0666,
Mode: 0o666,
Size: int64(len(buf)),
ModTime: time.Now(),
}); err != nil {
@ -215,7 +218,7 @@ func (cmd *BackupTarCommand) backupTarIDAllocData(ctx context.Context, tw *tar.W
// Build header & copy data to archive.
if err = tw.WriteHeader(&tar.Header{
Name: "idalloc",
Mode: 0666,
Mode: 0o666,
Size: int64(buf.Len()),
ModTime: time.Now(),
}); err != nil {
@ -249,7 +252,6 @@ func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer,
if err := cmd.backupTarIndexTranslateData(ctx, tw, ii.Name); err != nil {
return err
}
}
// Back up field translation data.
@ -276,7 +278,15 @@ func (cmd *BackupTarCommand) backupTarShard(ctx context.Context, tw *tar.Writer,
for _, node := range nodes {
if e := cmd.backupTarShardNode(ctx, tw, indexName, shard, node); e == nil {
return nil // backup ok, exit
break
} else if err == nil {
err = e // save first error, try next node
}
}
for _, node := range nodes {
if e := cmd.backupTarShardDataframe(ctx, tw, indexName, shard, node); e == nil {
break
} else if err == nil {
err = e // save first error, try next node
}
@ -296,7 +306,6 @@ func (cmd *BackupTarCommand) backupTarShardNode(ctx context.Context, tw *tar.Wri
pilosa.WithClientRetryPeriod(cmd.RetryPeriod),
pilosa.WithSerializer(proto.Serializer{}))
rc, err := client.ShardReader(ctx, indexName, shard)
if err != nil {
return fmt.Errorf("fetching shard reader: %w", err)
}
@ -312,7 +321,7 @@ func (cmd *BackupTarCommand) backupTarShardNode(ctx context.Context, tw *tar.Wri
// Build header & copy data to archive.
if err = tw.WriteHeader(&tar.Header{
Name: filename,
Mode: 0666,
Mode: 0o666,
Size: int64(buf.Len()),
ModTime: time.Now(),
}); err != nil {
@ -324,6 +333,46 @@ func (cmd *BackupTarCommand) backupTarShardNode(ctx context.Context, tw *tar.Wri
return nil
}
func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, node *disco.Node) error {
logger := cmd.Logger()
logger.Printf("backing up dataframe shard: index=%q shard=%d", indexName, shard)
client := pilosa.NewInternalClientFromURI(&node.URI,
pilosa.GetHTTPClient(cmd.tlsConfig, pilosa.ClientResponseHeaderTimeoutOption(cmd.HeaderTimeout)),
)
resp, err := client.GetDataframeShard(ctx, indexName, shard)
// no error if doesn't exist
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode == 404 {
// no error if not present server maynot have it turned on
return nil
}
filename := filepath.Join("indexes", indexName, "dataframe", fmt.Sprintf("%04d", shard))
vprint.VV("wrting %v", filename)
var buf bytes.Buffer
if _, err := buf.ReadFrom(resp.Body); err != nil {
return fmt.Errorf("copying shard data to memory: %w", err)
}
// Build header & copy data to archive.
if err = tw.WriteHeader(&tar.Header{
Name: filename,
Mode: 0o666,
Size: int64(buf.Len()),
ModTime: time.Now(),
}); err != nil {
return err
} else if _, err := io.Copy(tw, &buf); err != nil {
return fmt.Errorf("copying shard data to archive: %w", err)
}
return nil
}
func (cmd *BackupTarCommand) backupTarIndexTranslateData(ctx context.Context, tw *tar.Writer, name string) error {
// TODO: Fetch holder partition count.
partitionN := disco.DefaultPartitionN
@ -356,7 +405,7 @@ func (cmd *BackupTarCommand) backupTarIndexPartitionTranslateData(ctx context.Co
// Build header & copy data to archive.
if err = tw.WriteHeader(&tar.Header{
Name: path.Join("indexes", name, "translate", fmt.Sprintf("%04d", partitionID)),
Mode: 0666,
Mode: 0o666,
Size: int64(buf.Len()),
ModTime: time.Now(),
}); err != nil {
@ -389,7 +438,7 @@ func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw
// Build header & copy data to archive.
if err = tw.WriteHeader(&tar.Header{
Name: path.Join("indexes", indexName, "fields", fieldName, "translate"),
Mode: 0666,
Mode: 0o666,
Size: int64(buf.Len()),
ModTime: time.Now(),
}); err != nil {

95
ctl/restore.go
View file

@ -110,7 +110,6 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) {
primary = node
break
}
}
if primary == nil {
return errors.New("no primary")
@ -123,6 +122,8 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) {
}
if err := cmd.restoreShards(ctx); err != nil {
return fmt.Errorf("cannot restore shards: %w", err)
} else if err := cmd.restoreDataframes(ctx); err != nil {
return fmt.Errorf("cannot restore dataframes: %w", err)
} else if err := cmd.restoreIndexTranslation(ctx); err != nil {
return fmt.Errorf("cannot restore index translation: %w", err)
} else if err := cmd.restoreFieldTranslation(ctx, nodes); err != nil {
@ -184,7 +185,7 @@ func (cmd *RestoreCommand) restoreSchema(ctx context.Context, primary *disco.Nod
return false
}
logger := cmd.Logger()
//NOTE SHOULD ONLY BE ONE
// NOTE SHOULD ONLY BE ONE
for _, index := range schema.Indexes {
if exists(index.Name) {
return fmt.Errorf("index Exists %v", index.Name)
@ -255,6 +256,38 @@ func (cmd *RestoreCommand) restoreIDAlloc(ctx context.Context, primary *disco.No
return err
}
func (cmd *RestoreCommand) restoreDataframes(ctx context.Context) error {
filenames, err := filepath.Glob(filepath.Join(cmd.Path, "indexes", "*", "dataframe", "*"))
if err != nil {
return err
}
ch := make(chan string, len(filenames))
for _, filename := range filenames {
ch <- filename
}
close(ch)
g, ctx := errgroup.WithContext(ctx)
for i := 0; i < cmd.Concurrency; i++ {
g.Go(func() error {
for {
select {
case <-ctx.Done():
return ctx.Err()
case filename, ok := <-ch:
if !ok {
return nil
} else if err := cmd.restoredDataframeShard(ctx, filename); err != nil {
return err
}
}
}
})
}
return g.Wait()
}
func (cmd *RestoreCommand) restoreShards(ctx context.Context) error {
filenames, err := filepath.Glob(filepath.Join(cmd.Path, "indexes", "*", "shards", "*"))
if err != nil {
@ -474,3 +507,61 @@ func (cmd *RestoreCommand) restoreFieldTranslationFile(ctx context.Context, node
func (cmd *RestoreCommand) TLSHost() string { return cmd.Host }
func (cmd *RestoreCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS }
func (cmd *RestoreCommand) restoredDataframeShard(ctx context.Context, filename string) error {
logger := cmd.Logger()
rel, err := filepath.Rel(cmd.Path, filename)
if err != nil {
return err
}
// Parse filename.
record := strings.Split(rel, string(os.PathSeparator))
indexName := record[1]
shard, err := strconv.ParseUint(record[3], 10, 64)
if err != nil {
return nil // not a shard file
}
nodes, err := cmd.client.FragmentNodes(ctx, indexName, shard)
if err != nil {
return fmt.Errorf("cannot determine fragment nodes: %w", err)
} else if len(nodes) == 0 {
return fmt.Errorf("no nodes available")
}
for _, node := range nodes {
logger.Printf("dataframe shard %v %v", shard, indexName)
f, err := os.Open(filename)
if err != nil {
return err
}
defer f.Close()
url := node.URI.Path(fmt.Sprintf("/internal/dataframe/restore/%v/%v", indexName, shard))
req, err := retryablehttp.NewRequest("POST", url, f)
if err != nil {
return err
}
req = req.WithContext(ctx)
req.Header.Set("Content-Type", "application/octet-stream")
token, ok := authn.GetAccessToken(ctx)
if ok && token != "" {
req.Header.Set("Authorization", token)
}
client := cmd.newClient()
resp, err := client.Do(req)
if err != nil {
return err
} else if err := resp.Body.Close(); err != nil {
return err
} else if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
}
return nil
}

38
ctl/restore_tar.go
View file

@ -123,7 +123,6 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
primary = node
break
}
}
if primary == nil {
return errors.New("no primary")
@ -190,6 +189,37 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
if err := g.Wait(); err != nil {
return err
}
case "dataframe":
shard, err := strconv.ParseUint(record[3], 10, 64)
if err != nil {
return err
}
fragmentNodes, err := cmd.client.FragmentNodes(ctx, indexName, shard)
if err != nil {
return fmt.Errorf("cannot determine fragmentNodes: %w", err)
} else if len(fragmentNodes) == 0 {
return fmt.Errorf("no fragmentNodes available")
}
shardBytes, err := io.ReadAll(tarReader) // this feels wrong but works for now
if err != nil {
return err
}
g, _ := errgroup.WithContext(ctx)
for _, node := range fragmentNodes {
node := node
g.Go(func() error {
client := &gohttp.Client{}
rd := bytes.NewReader(shardBytes)
logger.Printf("dataframe shard %v %v", shard, indexName)
url := node.URI.Path(fmt.Sprintf("/internal/dataframe/restore/%v/%v", indexName, shard))
_, err = client.Post(url, "application/octet-stream", rd)
return err
})
}
if err := g.Wait(); err != nil {
return err
}
case "translate":
partitionID, err := strconv.Atoi(record[3])
logger.Printf("column keys %v (%v)", indexName, partitionID)
@ -221,13 +251,13 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
}
case "attributes":
//skip
// skip
case "fields":
fieldName := record[3]
switch action := record[4]; action {
case "translate":
logger.Printf("field keys %v %v", indexName, fieldName)
//needs to go to all nodes
// needs to go to all nodes
shardBytes, err := io.ReadAll(tarReader) // this feels wrong but works for now
if err != nil {
return err
@ -248,7 +278,7 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
return err
}
case "attributes":
//skip
// skip
default:
return fmt.Errorf("unknown restore action: %v", action)
}

2
ctl/server.go
View file

@ -143,6 +143,8 @@ func serverFlagSet(srv *server.Config, prefix string) *pflag.FlagSet {
flags.BoolVar(&srv.DataDog.GoroutineProfile, pre("datadog.goroutine-profile"), false, "golang pprof goroutine profile")
flags.BoolVar(&srv.DataDog.BlockProfile, pre("datadog.block-profile"), false, "golang pprof goroutine ")
flags.BoolVar(&srv.Dataframe.Enable, pre("dataframe.enable"), false, "EXPERIMENTAL enable support for Apply and Arrow")
return flags
}

187
dataframe_test.go Normal file
View file

@ -0,0 +1,187 @@
package pilosa_test
import (
"bytes"
"context"
"encoding/json"
"math"
"strings"
"testing"
"github.com/apache/arrow/go/v10/arrow"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/test"
)
func TestExecutor_Apply(t *testing.T) {
c := test.MustRunCluster(t, 1, []server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerIsDataframeEnabled(true),
),
})
defer c.Close()
indexName := "ti"
fieldName := "f"
ctx := context.Background()
api := c.GetNode(0).API
idx, err := api.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: false, TrackExistence: true})
if err != nil {
t.Fatal(err)
}
opts := []pilosa.FieldOption{pilosa.OptFieldTypeInt(0, math.MaxInt64)}
_, err = idx.CreateField(fieldName, "", opts...)
if err != nil {
t.Fatal(err)
}
req := &pilosa.ImportValueRequest{}
req.Index = indexName
req.Field = fieldName
req.ColumnIDs = []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
req.Values = []int64{10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
qcx := api.Txf().NewQcx()
defer qcx.Abort()
if err := api.ImportValue(ctx, qcx, req); err != nil {
t.Fatal(err)
}
if err := qcx.Finish(); err != nil {
t.Fatal(err)
}
t.Run("dataframe ingest", func(t *testing.T) {
// func (c *Client) ApplyDataframeChangeset(indexName string, cr *pilosa.ChangesetRequest, shard uint64) (map[string]interface{}, error) {
cr := &pilosa.ChangesetRequest{}
// for each row a list of columns
cr.Columns = []interface{}{
[]int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
[]int64{2, 4, 6, 8, 10, 12, 14, 16, 18, 20},
[]float64{1, 1.414, 1.732, 2, 2.236, 2.449, 2.646, 2.828, 3, 3.162},
}
cr.ShardIds = []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
cr.SimpleSchema = []pilosa.NameType{
{Name: "_ID", DataType: arrow.PrimitiveTypes.Int64},
{Name: "ival", DataType: arrow.PrimitiveTypes.Int64},
{Name: "fval", DataType: arrow.PrimitiveTypes.Float64},
}
shard := uint64(0)
err := api.ApplyDataframeChangeset(ctx, indexName, cr, shard)
if err != nil {
t.Fatal(err)
}
})
t.Run("dataframe schema", func(t *testing.T) {
expectedJSON := `[{"Name":"_ID","Type":"int64"},{"Name":"ival","Type":"int64"},{"Name":"fval","Type":"float64"}]`
parts, err := api.GetDataframeSchema(ctx, indexName)
if err != nil {
t.Fatal(err)
}
w := new(bytes.Buffer)
if err := json.NewEncoder(w).Encode(parts); err != nil {
t.Fatal(err)
}
got := strings.Trim(w.String(), "\t \n")
if strings.Compare(got, expectedJSON) != 0 {
t.Fatalf("expected: %v got: %v", expectedJSON, got)
}
})
t.Run("dataframe apply all", func(t *testing.T) {
pql := `Apply("_ID","_")`
if res, err := api.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
t.Fatal(err)
} else {
// qr := res.Results[0].(*dataframe.DataFrame)
expectedJSON := `{"Results":[{"_ID":{"int":[10,9,8,7,6,5,4,3,2,1,0]}}],"Err":null,"Profile":null}`
w := new(bytes.Buffer)
if err := json.NewEncoder(w).Encode(res); err != nil {
t.Fatal(err)
}
got := strings.Trim(w.String(), "\t \n")
if strings.Compare(got, expectedJSON) != 0 {
t.Fatalf("expected: %v got: %v", expectedJSON, got)
}
}
})
t.Run("dataframe apply filter", func(t *testing.T) {
// TODO(twg) 2022/11/03 refactor this when changing marshalling
pql := `Apply(ConstRow(columns=[2,4,6]),"_ID+0","_")`
if res, err := api.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
t.Fatal(err)
} else {
expectedJSON := `{"Results":[{"I":{"int":[6,4,2]}}],"Err":null,"Profile":null}`
w := new(bytes.Buffer)
if err := json.NewEncoder(w).Encode(res); err != nil {
t.Fatal(err)
}
got := strings.Trim(w.String(), "\t \n")
if strings.Compare(got, expectedJSON) != 0 {
t.Fatalf("expected: %v got: %v", expectedJSON, got)
}
}
})
t.Run("dataframe apply ivy err map", func(t *testing.T) {
pql := `Apply("barf","_")`
if _, err := api.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err == nil {
t.Fatal("expected error")
}
})
t.Run("dataframe apply ivy err reduce", func(t *testing.T) {
pql := `Apply("_ID","barfo")`
if _, err := api.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err == nil {
t.Fatal("expected error")
}
})
t.Run("dataframe arrow filter", func(t *testing.T) {
pql := `Arrow(ConstRow(columns=[2,4,6]))`
if res, err := api.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
t.Fatal(err)
} else {
expectedJSON := `{"Results":[{"_ID":[2,4,6],"fval":[1.414,2,2.449],"ival":[4,8,12]}],"Err":null,"Profile":null}`
w := new(bytes.Buffer)
if err := json.NewEncoder(w).Encode(res); err != nil {
t.Fatal(err)
}
got := strings.Trim(w.String(), "\t \n")
if strings.Compare(got, expectedJSON) != 0 {
t.Fatalf("expected: %v got: %v", expectedJSON, got)
}
}
})
t.Run("dataframe arrow filter with header", func(t *testing.T) {
pql := `Arrow(ConstRow(columns=[2,4,6]),header=["fval"])`
if res, err := api.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
t.Fatal(err)
} else {
expectedJSON := `{"Results":[{"_ID":[2,4,6],"fval":[1.414,2,2.449]}],"Err":null,"Profile":null}`
w := new(bytes.Buffer)
if err := json.NewEncoder(w).Encode(res); err != nil {
t.Fatal(err)
}
got := strings.Trim(w.String(), "\t \n")
if strings.Compare(got, expectedJSON) != 0 {
t.Fatalf("expected: %v got: %v", expectedJSON, got)
}
}
})
t.Run("dataframe delete", func(t *testing.T) {
err := api.DeleteDataframe(ctx, indexName)
if err != nil {
t.Fatal(err)
}
expectedJSON := `[]`
parts, err := api.GetDataframeSchema(ctx, indexName)
if err != nil {
t.Fatal(err)
}
w := new(bytes.Buffer)
if err := json.NewEncoder(w).Encode(parts); err != nil {
t.Fatal(err)
}
got := strings.Trim(w.String(), "\t \n")
if strings.Compare(got, expectedJSON) != 0 {
t.Fatalf("expected: %v got: %v", expectedJSON, got)
}
})
}

106
encoding/proto/proto.go
View file

@ -3,17 +3,27 @@
package proto
import (
"bufio"
"bytes"
"context"
"fmt"
"math/big"
"time"
"github.com/apache/arrow/go/v10/arrow"
"github.com/apache/arrow/go/v10/arrow/memory"
"github.com/apache/arrow/go/v10/parquet"
"github.com/apache/arrow/go/v10/parquet/file"
"github.com/apache/arrow/go/v10/parquet/pqarrow"
"github.com/gogo/protobuf/proto"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/disco"
pnet "github.com/featurebasedb/featurebase/v3/net"
"github.com/featurebasedb/featurebase/v3/pb"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/roaring"
"github.com/gomem/gomem/pkg/dataframe"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/disco"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/pb"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/vprint"
"github.com/pkg/errors"
)
@ -22,8 +32,10 @@ type Serializer struct {
RoaringRows bool
}
var DefaultSerializer = Serializer{}
var RoaringSerializer = Serializer{RoaringRows: true}
var (
DefaultSerializer = Serializer{}
RoaringSerializer = Serializer{RoaringRows: true}
)
// Marshal turns pilosa messages into protobuf serialized bytes.
func (s Serializer) Marshal(m pilosa.Message) ([]byte, error) {
@ -363,6 +375,8 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message {
return s.encodeTransactionMessage(mt)
case *pilosa.AtomicRecord:
return s.encodeAtomicRecord(mt)
case *pilosa.DeleteDataframeMessage:
return s.encodeDeleteDataframeMessage(mt)
}
return nil
}
@ -376,6 +390,7 @@ func (s Serializer) encodeBlockDataRequest(m *pilosa.BlockDataRequest) *pb.Block
Block: m.Block,
}
}
func (s Serializer) encodeBlockDataResponse(m *pilosa.BlockDataResponse) *pb.BlockDataResponse {
return &pb.BlockDataResponse{
RowIDs: m.RowIDs,
@ -537,6 +552,12 @@ func (s Serializer) encodeQueryResponse(m *pilosa.QueryResponse) *pb.QueryRespon
resp.Results[i].DistinctTimestamp = s.encodeDistinctTimestamp(result)
case nil:
resp.Results[i].Type = queryResultTypeNil
case *dataframe.DataFrame:
resp.Results[i].Type = queryResultTypeDataFrame
resp.Results[i].DataFrame = s.encodeDataFrame(result)
case arrow.Table:
resp.Results[i].Type = queryResultTypeArrowTable
resp.Results[i].ArrowTable = s.encodeArrowTable(result)
default:
panic(fmt.Errorf("unknown type: %T", m.Results[i]))
}
@ -885,6 +906,12 @@ func (s Serializer) encodeTransactionStats(stats pilosa.TransactionStats) *pb.Tr
return &pb.TransactionStats{}
}
func (s Serializer) encodeDeleteDataframeMessage(m *pilosa.DeleteDataframeMessage) *pb.DeleteDataframeMessage {
return &pb.DeleteDataframeMessage{
Index: m.Index,
}
}
func (s Serializer) decodeSchema(sc *pb.Schema, m *pilosa.Schema) {
m.Indexes = make([]*pilosa.IndexInfo, len(sc.Indexes))
s.decodeIndexes(sc.Indexes, m.Indexes)
@ -1310,6 +1337,8 @@ const (
queryResultTypeExtractedIDMatrix
queryResultTypeExtractedTable
queryResultTypeDistinctTimestamp
queryResultTypeDataFrame
queryResultTypeArrowTable
)
func (s Serializer) decodeQueryResult(pb *pb.QueryResult) interface{} {
@ -1348,6 +1377,10 @@ func (s Serializer) decodeQueryResult(pb *pb.QueryResult) interface{} {
return s.decodeRowMatrix(pb.RowMatrix)
case queryResultTypeDistinctTimestamp:
return s.decodeDistinctTimestamp(pb.DistinctTimestamp)
case queryResultTypeDataFrame:
return s.decodeDataFrame(pb.DataFrame)
case queryResultTypeArrowTable:
return s.decodeArrowTable(pb.ArrowTable)
}
panic(fmt.Sprintf("unknown type: %d", pb.Type))
}
@ -1565,6 +1598,63 @@ func (s Serializer) decodeDecimalStruct(pb *pb.Decimal) *pql.Decimal {
return d
}
func (s Serializer) decodeDataFrame(pdf *pb.DataFrame) *dataframe.DataFrame {
pool := memory.NewGoAllocator() // TODO(twg) this should probably be asingletoon somwhere
df, _ := dataframe.NewFrameFromArrowBytes(pdf.Data, pool)
return df
}
func (s Serializer) decodeDeleteDataframeMessage(pb *pb.DeleteDataframeMessage, m *pilosa.DeleteDataframeMessage) {
m.Index = pb.Index
}
func (s Serializer) encodeDataFrame(df *dataframe.DataFrame) *pb.DataFrame {
if df == nil {
return &pb.DataFrame{} // Generated proto code doesn't like a nil Row.
}
buff, err := df.ToBytes()
if err != nil {
panic(err)
}
// ugh hate having to swallow error here
return &pb.DataFrame{
Data: buff,
}
}
func (s Serializer) encodeArrowTable(table arrow.Table) *pb.ArrowTable {
props := parquet.NewWriterProperties(parquet.WithDictionaryDefault(false))
arrProps := pqarrow.DefaultWriterProps()
var b bytes.Buffer
w := bufio.NewWriter(&b)
chunkSize := int64(65536) // TODO(twg) 2022/11/09 default 64k?
pqarrow.WriteTable(table, w, chunkSize, props, arrProps)
w.Flush()
return &pb.ArrowTable{
Data: b.Bytes(),
}
}
func (s Serializer) decodeArrowTable(table *pb.ArrowTable) arrow.Table {
mem := memory.NewGoAllocator() // TODO(twg) this should probably be asingletoon somwhere
r := bytes.NewReader(table.Data) // TODO(twg) 2022/11/09 I hate snarfing errors
pf, err := file.NewParquetReader(r)
if err != nil {
vprint.VV("e1 %v", err)
}
reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, mem)
if err != nil {
vprint.VV("e2 %v", err)
}
tbl, err := reader.ReadTable(context.Background())
if err != nil {
vprint.VV("e3 %v", err)
}
vprint.VV("decodeArrowTable %d", tbl.NumRows())
return tbl
}
func (s Serializer) encodeSignedRow(r pilosa.SignedRow) *pb.SignedRow {
ir := &pb.SignedRow{
Pos: s.encodeRow(r.Pos),

81
encoding/proto/proto_test.go
View file

@ -6,8 +6,12 @@ import (
"reflect"
"testing"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/pb"
"github.com/apache/arrow/go/v10/arrow"
"github.com/apache/arrow/go/v10/arrow/array"
"github.com/apache/arrow/go/v10/arrow/memory"
"github.com/gomem/gomem/pkg/dataframe"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/pb"
)
func testOneRoundTrip(t *testing.T, s pilosa.Serializer, obj pilosa.Message, expectedMarshalErr error, expectedUnmarshalErr error, expectedMismatchErr error) {
@ -82,3 +86,76 @@ func TestDecodeQueryResult(t *testing.T) {
}
})
}
func TestDataFrameQueryResult(t *testing.T) {
pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
t.Run("Dataframe", func(t *testing.T) {
df, err := CreateDataframe(pool)
if err != nil {
t.Fatalf("need a good starting point %v", err)
}
s := Serializer{}
pdf := s.encodeDataFrame(df)
q := &pb.QueryResult{Type: queryResultTypeDataFrame, DataFrame: pdf}
decoded := s.decodeQueryResult(q).(*dataframe.DataFrame)
if !df.Equals(decoded) {
t.Errorf("failed to decode Dataframe. expected\n %#v\n got\n %#v", df, decoded)
}
})
}
func TestArrowResult(t *testing.T) {
pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
t.Run("Arrow", func(t *testing.T) {
table := CreateArrowTable(pool)
defer table.Release()
s := Serializer{}
art := s.encodeArrowTable(table)
q := &pb.QueryResult{Type: queryResultTypeArrowTable, ArrowTable: art}
decoded := s.decodeQueryResult(q).(arrow.Table)
if table.NumRows() != decoded.NumRows() {
t.Errorf("failed to decode Dataframe. expected\n %#v\n got\n %#v", table.NumRows(), decoded.NumRows())
}
})
}
func CreateDataframe(pool memory.Allocator) (*dataframe.DataFrame, error) {
table := CreateArrowTable(pool)
return dataframe.NewDataFrameFromTable(pool, table)
}
func CreateArrowTable(pool memory.Allocator) arrow.Table {
schema := arrow.NewSchema(
[]arrow.Field{
{Name: "INT", Type: arrow.PrimitiveTypes.Int64},
{Name: "FLOAT", Type: arrow.PrimitiveTypes.Float64},
},
nil,
)
b := array.NewRecordBuilder(pool, schema)
defer b.Release()
// create 2 chunks
b.Field(0).(*array.Int64Builder).AppendValues([]int64{1, 2, 3, 4, 5, 6}, nil)
b.Field(0).(*array.Int64Builder).AppendValues([]int64{7, 8, 9, 10}, []bool{true, true, false, true})
b.Field(1).(*array.Float64Builder).AppendValues([]float64{1, 2, 3, 4, 5, 6}, nil)
b.Field(1).(*array.Float64Builder).AppendValues([]float64{7, 8, 9, 10}, []bool{true, true, false, true})
rec1 := b.NewRecord()
b.Field(0).(*array.Int64Builder).AppendValues([]int64{11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, nil)
b.Field(1).(*array.Float64Builder).AppendValues([]float64{11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, nil)
rec2 := b.NewRecord()
b.Field(0).(*array.Int64Builder).AppendValues([]int64{31, 32, 33, 34, 35, 36, 37, 38, 39, 40}, nil)
b.Field(1).(*array.Float64Builder).AppendValues([]float64{31, 32, 33, 34, 35, 36, 37, 38, 39, 40}, nil)
rec3 := b.NewRecord()
records := []arrow.Record{rec1, rec2, rec3}
return array.NewTableFromRecords(schema, records)
}

14
et_test.go Normal file
View file

@ -0,0 +1,14 @@
package pilosa_test
import (
"testing"
"github.com/molecula/featurebase/v3/test"
"github.com/molecula/featurebase/v3/vprint"
)
func TestEtcd_StartStop(t *testing.T) {
c := test.MustRunUnsharedCluster(t, 1)
err := c.Close()
vprint.VV("DONE %v", err)
}

1
etcd/embed.go
View file

@ -342,6 +342,7 @@ func (e *Etcd) parseOptions() (*embed.Config, error) {
cfg.Name = e.options.Name
cfg.Dir = e.options.Dir
cfg.InitialClusterToken = e.options.ClusterName
cfg.EnableGRPCGateway = false // TODO(twg) 2022/11/03 fixes tests but unsure ramifications
var err error
cfg.LCUrls, err = types.NewURLs([]string{e.options.LClientURL})
if err != nil {

3
etcd/embed_test.go
View file

@ -14,6 +14,7 @@ import (
func TestRestartEtcd(t *testing.T) {
cfg := embed.NewConfig()
cfg.Dir = "default.etcd"
cfg.EnableGRPCGateway = false
curl, _ := url.Parse(unixSocket(t))
// append-in-place to replace any existing URLs with our new URL
cfg.LPUrls = append(cfg.LPUrls[:0], *curl)
@ -51,7 +52,7 @@ func TestRestartEtcd(t *testing.T) {
}
func TestParseOptions(t *testing.T) {
var e = &Etcd{options: Options{ClusterURL: "http://foo"}, logger: logger.NewLogfLogger(t)}
e := &Etcd{options: Options{ClusterURL: "http://foo"}, logger: logger.NewLogfLogger(t)}
curl, _ := url.Parse(unixSocket(t))
e.options.LClientURL = curl.String()

7
etcd/leasedkv_test.go
View file

@ -18,8 +18,10 @@ import (
"go.etcd.io/etcd/pkg/types"
)
const initVal = "test"
const newVal = "newValue"
const (
initVal = "test"
newVal = "newValue"
)
func TestClusterKv(t *testing.T) {
if !AllowCluster() {
@ -63,6 +65,7 @@ func TestLeasedKv(t *testing.T) {
cfg.LCUrls = types.MustNewURLs([]string{clientURL})
cfg.ACUrls = types.MustNewURLs([]string{clientURL})
cfg.InitialCluster = cfg.Name + "=" + peerURL
cfg.EnableGRPCGateway = false
dir, err := testhook.TempDir(t, "leasedkv-*")
if err != nil {

37
executor.go
View file

@ -17,6 +17,7 @@ import (
"time"
"unsafe"
"github.com/gomem/gomem/pkg/dataframe"
"github.com/lib/pq"
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/featurebasedb/featurebase/v3/pql"
@ -73,6 +74,9 @@ type executor struct {
// Maximum per-request memory usage (Extract() only)
maxMemory int64
// Temporary flag to be removed when stablized
dataframeEnabled bool
}
// executorOption is a functional option type for pilosa.executor
@ -233,7 +237,6 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
qcx = idx.holder.txf.NewWritableQcx()
} else {
qcx = idx.holder.txf.NewQcx()
}
defer qcx.Abort()
@ -336,6 +339,12 @@ func safeCopy(resp QueryResponse) (out QueryResponse) {
out.Results = append(out.Results, x)
case *SortedRow:
out.Results = append(out.Results, x)
case *dataframe.DataFrame:
// dumpTable(x)
out.Results = append(out.Results, x)
case *basicTable:
// dumpTable(x)
out.Results = append(out.Results, x)
default:
panic(fmt.Sprintf("handle %T here", v))
}
@ -697,7 +706,6 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p
shards = []uint64{0}
}
}
// Preprocess the query.
c, err := e.preprocessQuery(ctx, qcx, index, c, shards, opt)
if err != nil {
@ -806,6 +814,12 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p
case "Sort":
res, err := e.executeSort(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeSort")
case "Apply":
res, err := e.executeApply(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeApply")
case "Arrow":
res, err := e.executeArrow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeArrow")
default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap.
statFn()
res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt)
@ -846,7 +860,6 @@ func (e *executor) validateTimeCallArgs(c *pql.Call, indexName string) error {
tq := field.TimeQuantum()
if (from || to) && tq == "" {
return fmt.Errorf("field %s is not a time-field, 'from' and 'to' are not valid options for this field type", f)
}
return nil
@ -3085,7 +3098,6 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c
// TODO as an optimization, we could apply some "having"
// conditions here long as they aren't on the Count(Distinct)
// aggregate
// Calculate Count(Distinct) aggregate if requested.
aggregate, _, err := c.CallArg("aggregate")
if err == nil && aggregate != nil && aggregate.Name == "Count" && len(aggregate.Children) > 0 && aggregate.Children[0].Name == "Distinct" && !opt.Remote {
@ -4473,7 +4485,6 @@ func (e *executor) executeExtract(ctx context.Context, qcx *Qcx, index string, c
default:
return ExtractedIDMatrix{}, errors.New("Extract, unexpected result type found")
}
}
func mergeBits(bits *Row, mask uint64, out map[uint64]uint64) {
@ -6342,7 +6353,9 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu
span, ctx := tracing.StartSpanFromContext(ctx, "executor.mapperLocal")
defer span.Finish()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
defer func() {
cancel()
}()
done := ctx.Done()
// the close process needs to know whether any mappers are currently active.
// note that we bump this BEFORE checking e.shutdown. if we check shutdown
@ -6798,6 +6811,7 @@ func fieldValidateValue(f *Field, val interface{}) error {
case FieldTypeTimestamp:
switch v := val.(type) {
case time.Time:
case int64: // TODO(twg) 2022/09/14 revisit this hack
default:
return errors.Errorf("invalid value %v for timestamp field %q", v, f.Name())
}
@ -7741,7 +7755,6 @@ func TimestampToVal(unit string, ts time.Time) int64 {
return ts.UnixNano()
}
return 0
}
// detectRangeCall returns true if the call or one of its children contains a Range call
@ -8867,10 +8880,10 @@ func DeleteRowsWithOutKeysFlow(ctx context.Context, columns *roaring.Bitmap, idx
}
func DeleteRowsWithFlow(ctx context.Context, src *Row, idx *Index, shard uint64, normalFlow bool) (change bool, err error) {
if len(src.Segments) == 0 { //nothing to remove
if len(src.Segments) == 0 { // nothing to remove
return false, nil
}
columns := src.Segments[0].data //should only be one segment
columns := src.Segments[0].data // should only be one segment
if columns.Count() == 0 {
return false, nil
}
@ -8933,7 +8946,7 @@ func (e *executor) executeSort(ctx context.Context, qcx *Qcx, index string, c *p
if err := ctx.Err(); err != nil {
return err
}
//var result *SortedRow
// var result *SortedRow
switch other := prev.(type) {
case *SortedRow:
if err := other.Merge(v.(*SortedRow), sort_desc); err != nil {
@ -9111,10 +9124,9 @@ func (e *executor) executeSortShard(ctx context.Context, qcx *Qcx, index string,
i++
}
}
}
}
//this is to make sure compare function worked.
// this is to make sure compare function worked.
ok := true
sort.SliceStable(rowKVs, func(i, j int) bool {
if c, k := rowKVs[i].Compare(rowKVs[j], sort_desc); k {
@ -9253,5 +9265,4 @@ func MergeExtractedIDMatrixSorted(a, b ExtractedIDMatrixSorted, sort_desc bool)
},
RowKVs: rowKVs,
}, nil
}

3
executor_internal_test.go
View file

@ -235,7 +235,6 @@ func TestExecutor_GroupCountCondition(t *testing.T) {
t.Run(fmt.Sprintf("test (#%d):", i), func(t *testing.T) {
for j, check := range test.checks {
t.Run(fmt.Sprintf("check (#%d):", j), func(t *testing.T) {
query, err := pql.ParseString(fmt.Sprintf("GroupBy(Rows(a), having=Condition(%s))", check.cond))
if err != nil {
t.Fatalf("parsing query: %v", err)
@ -488,7 +487,6 @@ func TestGetSorter(t *testing.T) {
if !reflect.DeepEqual(gcs, tst.expGCS) {
t.Errorf("exp:\n%+v\ngot:\n%v\n", tst.expGCS, gcs)
}
})
}
}
@ -516,7 +514,6 @@ func TestGetScaledInt(t *testing.T) {
if !reflect.DeepEqual(res, v) {
t.Errorf("expected %v, got %v", v, res)
}
}
func TestDistinctTimestampUnion(t *testing.T) {

78
go.mod
View file

@ -2,7 +2,9 @@ module github.com/featurebasedb/featurebase/v3
replace github.com/go-avro/avro => github.com/pilosa/avro v0.0.0-20200626214113-bc1bf9fd41c1
replace google.golang.org/grpc => google.golang.org/grpc v1.38.0
replace github.com/gomem/gomem => github.com/tgruben/gomem v0.0.0-20221021111114-79fdc77dcf61
replace robpike.io/ivy => github.com/tgruben/ivy v0.0.0-20221107170120-634b546dcdac
require (
github.com/CAFxX/gcnotifier v0.0.0-20220409005548-0153238b886a
@ -14,7 +16,7 @@ require (
github.com/beevik/ntp v0.3.0
github.com/benbjohnson/immutable v0.3.0
github.com/cespare/xxhash v1.1.0
github.com/chzyer/readline v1.5.0
github.com/chzyer/readline v1.5.1
github.com/confluentinc/confluent-kafka-go v1.9.1
github.com/davecgh/go-spew v1.1.1
github.com/denisenkom/go-mssqldb v0.11.0
@ -65,20 +67,19 @@ require (
github.com/zeebo/blake3 v0.2.3
go.etcd.io/bbolt v1.3.6
go.etcd.io/etcd v3.3.27+incompatible
go.etcd.io/etcd/api/v3 v3.5.4
go.etcd.io/etcd/client/pkg/v3 v3.5.4
go.etcd.io/etcd/client/v3 v3.5.4
go.etcd.io/etcd/server/v3 v3.5.4
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
golang.org/x/mod v0.6.0
golang.org/x/net v0.1.0 // indirect
go.etcd.io/etcd/api/v3 v3.5.5
go.etcd.io/etcd/client/pkg/v3 v3.5.5
go.etcd.io/etcd/client/v3 v3.5.5
go.etcd.io/etcd/server/v3 v3.5.5
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91
golang.org/x/mod v0.7.0
golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
golang.org/x/sync v0.1.0
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11
gopkg.in/DataDog/dd-trace-go.v1 v1.38.1
gopkg.in/yaml.v2 v2.4.0
modernc.org/mathutil v1.0.0
modernc.org/strutil v1.0.0
modernc.org/mathutil v1.5.0
modernc.org/strutil v1.1.3
sigs.k8s.io/yaml v1.2.0
vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible
)
@ -86,16 +87,22 @@ require (
require (
github.com/PaesslerAG/gval v1.0.0
github.com/PaesslerAG/jsonpath v0.1.1
github.com/apache/arrow/go/v10 v10.0.0-20221021053532-2f627c213fc3
github.com/gomem/gomem v0.1.0
github.com/google/uuid v1.3.0
github.com/jaffee/commandeer v0.6.0
github.com/linkedin/goavro/v2 v2.11.1
google.golang.org/grpc v1.46.0
google.golang.org/protobuf v1.28.0
google.golang.org/grpc v1.49.0
google.golang.org/protobuf v1.28.1
robpike.io/ivy v0.2.9
)
require (
github.com/DataDog/datadog-go/v5 v5.1.0 // indirect
github.com/DataDog/gostackparse v0.5.0 // indirect
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/apache/thrift v0.16.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
@ -112,10 +119,12 @@ require (
github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.0.4 // indirect
github.com/goccy/go-json v0.9.11 // indirect
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/flatbuffers v2.0.8+incompatible // indirect
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
@ -127,12 +136,16 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.14.2 // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.15.9 // indirect
github.com/klauspost/cpuid/v2 v2.0.12 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magiconair/properties v1.8.5 // indirect
github.com/mattetti/filebuffer v1.0.1 // indirect
github.com/mattn/go-runewidth v0.0.2 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@ -140,7 +153,7 @@ require (
github.com/morikuni/aec v1.0.0 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/pierrec/lz4/v4 v4.1.14 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/common v0.33.0 // indirect
@ -157,32 +170,33 @@ require (
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.etcd.io/etcd/client/v2 v2.305.4 // indirect
go.etcd.io/etcd/pkg/v3 v3.5.4 // indirect
go.etcd.io/etcd/raft/v3 v3.5.4 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.etcd.io/etcd/client/v2 v2.305.5 // indirect
go.etcd.io/etcd/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/raft/v3 v3.5.5 // indirect
go.mongodb.org/mongo-driver v1.7.5 // indirect
go.opentelemetry.io/contrib v0.20.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect
go.opentelemetry.io/otel v0.20.0 // indirect
go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect
go.opentelemetry.io/otel/metric v0.20.0 // indirect
go.opentelemetry.io/otel/sdk v0.20.0 // indirect
go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect
go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect
go.opentelemetry.io/otel/trace v0.20.0 // indirect
go.opentelemetry.io/proto/otlp v0.7.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0 // indirect
go.opentelemetry.io/otel v1.0.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1 // indirect
go.opentelemetry.io/otel/sdk v1.0.1 // indirect
go.opentelemetry.io/otel/trace v1.0.1 // indirect
go.opentelemetry.io/proto/otlp v0.9.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.17.0 // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/sys v0.1.0 // indirect
golang.org/x/net v0.2.0 // indirect
golang.org/x/sys v0.2.0 // indirect
golang.org/x/text v0.4.0 // indirect
golang.org/x/tools v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gotest.tools/v3 v3.3.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gotest.tools/v3 v3.0.2 // indirect
nhooyr.io/websocket v1.8.6 // indirect
)

231
go.sum
View file

@ -1,3 +1,4 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
@ -68,6 +69,8 @@ github.com/DataDog/sketches-go v1.0.0/go.mod h1:O+XkJHWk9w4hDwY2ZUDU31ZC9sNYlYo8
github.com/DataDog/zstd v1.3.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM=
github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU=
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
@ -101,9 +104,15 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2 h1:090cWAt7zsbdvRegKCBVwcCTghjxhUh1PK2KNSq82vw=
github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2/go.mod h1:c5eyz5amZqTKvY3ipqerFO/74a/8CYmXOahSr40c+Ww=
github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/apache/arrow/go/v10 v10.0.0-20221021053532-2f627c213fc3 h1:PpXyIaFzGGj6B7llkhQ82AFJ4oGAPiPxuZzjy3vHGVM=
github.com/apache/arrow/go/v10 v10.0.0-20221021053532-2f627c213fc3/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY=
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@ -130,8 +139,6 @@ github.com/aws/smithy-go v1.0.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB
github.com/aws/smithy-go v1.11.0/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM=
github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw=
github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg=
github.com/benbjohnson/clock v1.0.3 h1:vkLuvpK4fmtSCuo60+yC63p7y0BmQ8gm5ZXGuBCJyXg=
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
github.com/benbjohnson/immutable v0.3.0 h1:TVRhuZx2wG9SZ0LRdqlbs9S5BZ6Y24hJEHTCgWHZEIw=
github.com/benbjohnson/immutable v0.3.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@ -159,18 +166,27 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/logex v1.2.0 h1:+eqR0HfOetur4tgnC8ftU5imRnhi4te+BadWS95c5AM=
github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM=
github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/readline v1.5.0 h1:lSwwFrbNviGePhkewF1az4oLmcwqCZijQ2/Wi3BGHAI=
github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic=
github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI=
github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/chzyer/test v0.0.0-20210722231415-061457976a23 h1:dZ0/VyGgQdVGAss6Ju0dt5P0QltE0SFY5Woh6hbIfiQ=
github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04=
github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E=
@ -237,7 +253,16 @@ github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox
github.com/elastic/go-elasticsearch/v7 v7.17.1/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
@ -362,11 +387,14 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/gobwas/ws v1.0.4 h1:5eXU1CZhpQdq5kXbKb+sECH5Ia5KiO6CYzIzdlVx6Bs=
github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/gocql/gocql v0.0.0-20220224095938-0eacd3183625/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofiber/fiber/v2 v2.11.0/go.mod h1:oZTLWqYnqpMMuF922SjGbsYZsdpE1MCfh416HNdweIM=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@ -387,6 +415,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
@ -424,6 +453,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM=
github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@ -678,14 +709,17 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw=
github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE=
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@ -696,6 +730,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
@ -724,6 +759,7 @@ github.com/linkedin/goavro/v2 v2.11.1 h1:4cuAtbDfqkKnBXp9E+tRkIJGa6W6iAjwonwt8O1
github.com/linkedin/goavro/v2 v2.11.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
@ -734,6 +770,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
github.com/mattetti/filebuffer v1.0.1 h1:gG7pyfnSIZCxdoKq+cPa8T0hhYtD9NxCdI4D7PTjRLM=
github.com/mattetti/filebuffer v1.0.1/go.mod h1:YdMURNDOttIiruleeVr6f56OrMc+MydEnTcXwtkxNVs=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@ -751,8 +789,8 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
@ -761,6 +799,10 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@ -807,7 +849,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE
github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nrwiersma/avro-benchmarks v0.0.0-20210913175520-21aec48c8f76/go.mod h1:iKyFMidsk/sVYONJRE372sJuX/QTRPacU7imPqqsu7g=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@ -864,8 +905,9 @@ github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0je
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pilosa/avro v0.0.0-20200626214113-bc1bf9fd41c1 h1:VEiqiLmebtgmPmpMhcByL5oXeLmDZyCLx+JmCJxDlG4=
github.com/pilosa/avro v0.0.0-20200626214113-bc1bf9fd41c1/go.mod h1:evNEPnmTS6kdXAZXtqlZgYvM+FwoJ9YdGc7YbLb2RVU=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
@ -942,6 +984,7 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U=
github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
@ -1012,7 +1055,6 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3
github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@ -1024,13 +1066,16 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/tgruben/gomem v0.0.0-20221021111114-79fdc77dcf61 h1:3RJ3IN/m4w4B7PeYF8PPOdGLHqu045LIxtymF4lQO7g=
github.com/tgruben/gomem v0.0.0-20221021111114-79fdc77dcf61/go.mod h1:avNE+ynGJYvQNY+/5Gk6aHxtlOtt2aO6V9ShcONBvvI=
github.com/tgruben/ivy v0.0.0-20221107170120-634b546dcdac h1:xVEKycwTG+9q18T1zA+ESrkIMIMQMmlsGOFirh5VxT8=
github.com/tgruben/ivy v0.0.0-20221107170120-634b546dcdac/go.mod h1:/COPfnSdd23BhmpDXFKsYPk1kK2sJpUr21TsZ1nsgcg=
github.com/tidwall/btree v0.3.0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/btree v1.1.0/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4=
github.com/tidwall/buntdb v1.2.0/go.mod h1:XLza/dhlwzO6dc5o/KWor4kfZSt3BP8QV+77ZMKfI58=
@ -1103,12 +1148,14 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
github.com/zenazn/goji v1.0.1/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@ -1119,22 +1166,22 @@ go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mI
go.etcd.io/etcd v3.3.27+incompatible h1:5hMrpf6REqTHV2LW2OclNpRtxI0k9ZplMemJsMSWju0=
go.etcd.io/etcd v3.3.27+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc=
go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0=
go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8=
go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg=
go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8=
go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ=
go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
go.etcd.io/etcd/client/v2 v2.305.4 h1:Dcx3/MYyfKcPNLpR4VVQUP5KgYrBeJtktBwEKkw08Ao=
go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU=
go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4=
go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
go.etcd.io/etcd/pkg/v3 v3.5.4 h1:V5Dvl7S39ZDwjkKqJG2BfXgxZ3QREqqKifWQgIw5IM0=
go.etcd.io/etcd/pkg/v3 v3.5.4/go.mod h1:OI+TtO+Aa3nhQSppMbwE4ld3uF1/fqqwbpfndbbrEe0=
go.etcd.io/etcd/raft/v3 v3.5.4 h1:YGrnAgRfgXloBNuqa+oBI/aRZMcK/1GS6trJePJ/Gqc=
go.etcd.io/etcd/raft/v3 v3.5.4/go.mod h1:SCuunjYvZFC0fBX0vxMSPjuZmpcSk+XaAcMrD6Do03w=
go.etcd.io/etcd/server/v3 v3.5.4 h1:CMAZd0g8Bn5NRhynW6pKhc4FRg41/0QYy3d7aNm9874=
go.etcd.io/etcd/server/v3 v3.5.4/go.mod h1:S5/YTU15KxymM5l3T6b09sNOHPXqGYIZStpuuGbb65c=
go.etcd.io/etcd/client/v2 v2.305.5 h1:DktRP60//JJpnPC0VBymAN/7V71GHMdjDCBt4ZPXDjI=
go.etcd.io/etcd/client/v2 v2.305.5/go.mod h1:zQjKllfqfBVyVStbt4FaosoX2iYd8fV/GRy/PbowgP4=
go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI=
go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c=
go.etcd.io/etcd/pkg/v3 v3.5.5 h1:Ablg7T7OkR+AeeeU32kdVhw/AGDsitkKPl7aW73ssjU=
go.etcd.io/etcd/pkg/v3 v3.5.5/go.mod h1:6ksYFxttiUGzC2uxyqiyOEvhAiD0tuIqSZkX3TyPdaE=
go.etcd.io/etcd/raft/v3 v3.5.5 h1:Ibz6XyZ60OYyRopu73lLM/P+qco3YtlZMOhnXNS051I=
go.etcd.io/etcd/raft/v3 v3.5.5/go.mod h1:76TA48q03g1y1VpTue92jZLr9lIHKUNcYdZOOGyx8rI=
go.etcd.io/etcd/server/v3 v3.5.5 h1:jNjYm/9s+f9A9r6+SC4RvNaz6AqixpOvhrFdT0PvIj0=
go.etcd.io/etcd/server/v3 v3.5.5/go.mod h1:rZ95vDw/jrvsbj9XpTqPrTAB9/kzchVdhRirySPkUBc=
go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw=
go.mongodb.org/mongo-driver v1.7.5 h1:ny3p0reEpgsR2cfA5cjgwFZg3Cv/ofFh/8jbhGtz9VI=
go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
@ -1147,37 +1194,28 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0=
go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0 h1:Wx7nFnvCaissIUZxPkBqDz2963Z+Cl+PkYbDKzTxDqQ=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E=
go.opentelemetry.io/otel v0.11.0/go.mod h1:G8UCk+KooF2HLkgo8RHX9epABH/aRGYET7gQOqBVdB0=
go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g=
go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg=
go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8=
go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw=
go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
go.opentelemetry.io/otel/sdk v0.20.0 h1:JsxtGXd06J8jrnya7fdI/U/MR6yXA5DtbZy+qoHQlr8=
go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
go.opentelemetry.io/otel/sdk/export/metric v0.20.0 h1:c5VRjxCXdQlx1HjzwGdQHzZaVI82b5EbBgOu2ljD92g=
go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
go.opentelemetry.io/otel/sdk/metric v0.20.0 h1:7ao1wpzHRVKf0OQ7GIxiQJA6X7DLX9o14gmVon7mMK8=
go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52lqtnbw=
go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8=
go.opentelemetry.io/otel v1.0.1 h1:4XKyXmfqJLOQ7feyV5DB6gsBFZ0ltB8vLtp6pj4JIcc=
go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1 h1:ofMbch7i29qIUf7VtF+r0HRF6ac0SBaPSziSsKp7wkk=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1 h1:CFMFNoz+CGprjFAFy+RJFrfEe4GBia3RRm2a4fREvCA=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk=
go.opentelemetry.io/otel/sdk v1.0.1 h1:wXxFEWGo7XfXupPwVJvTBOaPBC9FEg0wB8hMNrKk+cA=
go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI=
go.opentelemetry.io/otel/trace v1.0.1 h1:StTeIH6Q3G4r0Fiw34LTokUFESZgIDUr0qIJ7mKmAfw=
go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v0.9.0 h1:C0g6TWmQYvjKRnljRULLWUVJGy8Uvu0NEL/5frY2/t4=
go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
@ -1216,7 +1254,7 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -1235,11 +1273,12 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMk
golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
golang.org/x/exp v0.0.0-20200901203048-c4f52b2c50aa/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20200908183739-ae8ad444f925/go.mod h1:1phAWC201xIgDyaFpmDeZkgf70Q4Pd/CNqfRtVPtxNw=
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE=
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 h1:tnebWN09GYg9OLPss1KXj8txwZc6X6uMr6VFdcGNbHw=
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@ -1250,7 +1289,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
@ -1264,8 +1302,8 @@ golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hM
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA=
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -1327,8 +1365,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -1357,10 +1395,11 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -1434,6 +1473,7 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@ -1449,8 +1489,8 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@ -1477,9 +1517,11 @@ golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13W
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@ -1505,7 +1547,6 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@ -1546,16 +1587,19 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE=
golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM=
golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E=
gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
@ -1582,6 +1626,7 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR
google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -1638,8 +1683,40 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29 h1:DJUvgAPiJWeMBiT+RzBVcJGQN7bAEWS5UEoMshES9xs=
google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw=
google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@ -1653,8 +1730,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/DataDog/dd-trace-go.v1 v1.38.1 h1:nAKgcpJLXRHF56cKCP3bN8gTTQmmNAZFEblbyGKhKTo=
gopkg.in/DataDog/dd-trace-go.v1 v1.38.1/go.mod h1:GBhK4yaMJ1h329ivtKAqRNe1EZ944UnZwtz5lh7CnJc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
@ -1663,8 +1741,8 @@ gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v1 v1.0.0/go.mod h1:CxwszS/Xz1C49Ucd2i6Zil5UToP1EmyrFhKaMVbg1mk=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
@ -1700,7 +1778,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@ -1710,9 +1787,9 @@ gorm.io/driver/sqlserver v1.0.4/go.mod h1:ciEo5btfITTBCj9BkoUVDvgQbUdLWQNqdFY5OG
gorm.io/gorm v1.9.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
gorm.io/gorm v1.20.0/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
gorm.io/gorm v1.20.6/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo=
gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@ -1730,10 +1807,10 @@ k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ=
modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I=
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE=
modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k=
nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

42
holder.go
View file

@ -41,6 +41,9 @@ const (
// FieldsDir is the default fields directory used by each index.
FieldsDir = "fields"
// DataframesDir is the directory where we store the dataframe files (currently Apache Arrow)
DataframesDir = "dataframes"
)
func init() {
@ -427,7 +430,7 @@ func (h *Holder) Open() error {
h.closing = make(chan struct{})
h.Logger.Printf("open holder path: %s", h.path)
if err := os.MkdirAll(h.IndexesPath(), 0750); err != nil {
if err := os.MkdirAll(h.IndexesPath(), 0o750); err != nil {
return errors.Wrap(err, "creating directory")
}
@ -531,7 +534,6 @@ func (h *Holder) Open() error {
h.Logger.Printf("open holder: complete")
return nil
}
func (h *Holder) sendOrSpool(msg Message) error {
@ -1306,11 +1308,11 @@ func (h *Holder) logStartup() error {
time := time.Now().Format(RFC3339NanoFixedWidth)
logLine := fmt.Sprintf("%s\t%s\n", time, Version)
if err := os.MkdirAll(h.path, 0750); err != nil {
if err := os.MkdirAll(h.path, 0o750); err != nil {
return errors.Wrap(err, "creating data directory")
}
f, err := os.OpenFile(h.path+"/startup.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
f, err := os.OpenFile(h.path+"/startup.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o600)
if err != nil {
return errors.Wrap(err, "opening startup log")
}
@ -1381,7 +1383,6 @@ func (s *holderSyncer) resetTranslationSync() error {
return errors.Wrap(err, "initializing translation replication")
}
return nil
}
////////////////////////////////////////////////////////////
@ -1784,3 +1785,34 @@ func decodeCreateFieldMessage(ser Serializer, b []byte) (*CreateFieldMessage, er
}
return &cfm, nil
}
func (h *Holder) DeleteDataframe(name string) error {
h.mu.Lock()
defer h.mu.Unlock()
// Confirm index exists.
idx := h.Index(name)
if idx == nil {
return newNotFoundError(ErrIndexNotFound, name)
}
path := idx.DataframesPath()
// Delete dataframe directory.
if err := os.RemoveAll(path); err != nil {
// There is a rare edge case here: If a cache flush was happening, RemoveAll
// can fail because a file gets created, say in a fragment directory, after
// RemoveAll has deleted everything it found in the directory, but before
// the actual directory is unlinked. In theory, though, this can't happen
// twice; by the time we get here, everything was closed, so at most one
// more file should get created.
err = os.RemoveAll(path)
if err != nil {
return errors.Wrap(err, "removing directory")
}
}
if err := os.Mkdir(path, 0o750); err != nil {
return errors.Wrap(err, "creating dataframe")
}
return nil
}

215
http_handler.go
View file

@ -6,6 +6,7 @@ import (
"bytes"
"context"
"crypto/tls"
"encoding/gob"
"encoding/hex"
"encoding/json"
"expvar"
@ -28,16 +29,7 @@ import (
"sync"
"time"
"github.com/featurebasedb/featurebase/v3/authn"
"github.com/featurebasedb/featurebase/v3/authz"
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/featurebasedb/featurebase/v3/monitor"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/rbf"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
"github.com/featurebasedb/featurebase/v3/storage"
"github.com/featurebasedb/featurebase/v3/tracing"
"github.com/apache/arrow/go/v10/arrow"
"github.com/felixge/fgprof"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
@ -241,8 +233,10 @@ func OptHandlerSQLEnabled(v bool) handlerOption {
}
}
var makeImportOk sync.Once
var importOk []byte
var (
makeImportOk sync.Once
importOk []byte
)
// NewHandler returns a new instance of Handler with a default logger.
func NewHandler(opts ...handlerOption) (*Handler, error) {
@ -341,7 +335,7 @@ func (h *Handler) populateValidators() {
h.validators["GetTransaction"] = queryValidationSpecRequired()
h.validators["PostTransaction"] = queryValidationSpecRequired()
h.validators["PostFinishTransaction"] = queryValidationSpecRequired()
h.validators["DeleteDataframe"] = queryValidationSpecRequired()
}
type contextKeyQuery int
@ -523,7 +517,10 @@ func newRouter(handler *Handler) http.Handler {
router.HandleFunc("/index/{index}", handler.chkAuthZ(handler.handleGetIndex, authz.Read)).Methods("GET").Name("GetIndex")
router.HandleFunc("/index/{index}", handler.chkAuthZ(handler.handlePostIndex, authz.Admin)).Methods("POST").Name("PostIndex")
router.HandleFunc("/index/{index}", handler.chkAuthZ(handler.handleDeleteIndex, authz.Admin)).Methods("DELETE").Name("DeleteIndex")
//router.HandleFunc("/index/{index}/field", handler.chkAuthZ(handler.handleGetFields, authz.Read)).Methods("GET") // Not implemented.
router.HandleFunc("/index/{index}/dataframe/{shard}", handler.chkAuthZ(handler.handlePostDataframe, authz.Write)).Methods("POST").Name("PostDataframe")
router.HandleFunc("/index/{index}/dataframe/{shard}", handler.chkAuthZ(handler.handleGetDataframe, authz.Read)).Methods("GET").Name("GetDataframe")
router.HandleFunc("/index/{index}/dataframe", handler.chkAuthZ(handler.handleGetDataframeSchema, authz.Read)).Methods("GET").Name("GetDataframeSchema")
router.HandleFunc("/index/{index}/dataframe", handler.chkAuthZ(handler.handleDeleteDataframe, authz.Write)).Methods("DELETE").Name("DeleteDataframe")
router.HandleFunc("/index/{index}/field", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/{field}/view", handler.chkAuthZ(handler.handleGetView, authz.Admin)).Methods("GET")
@ -604,6 +601,7 @@ func newRouter(handler *Handler) http.Handler {
router.HandleFunc("/internal/idalloc/data", handler.chkAuthN(handler.handleIDAllocData)).Methods("GET").Name("IDAllocData")
router.HandleFunc("/internal/restore/{index}/{shardID}", handler.chkAuthZ(handler.handlePostRestore, authz.Admin)).Methods("POST").Name("Restore")
router.HandleFunc("/internal/dataframe/restore/{index}/{shardID}", handler.chkAuthZ(handler.handlePostDataframeRestore, authz.Admin)).Methods("POST").Name("Restore")
router.HandleFunc("/internal/debug/rbf", handler.chkAuthZ(handler.handleGetInternalDebugRBFJSON, authz.Admin)).Methods("GET").Name("GetInternalDebugRBFJSON")
@ -711,7 +709,7 @@ func (h *Handler) chkAuthN(handler http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
//if the request is unauthenticated and we have the appropriate header get the userid from the header
// if the request is unauthenticated and we have the appropriate header get the userid from the header
requestUserID := r.Header.Get(HeaderRequestUserID)
ctx = WithUserID(ctx, requestUserID)
@ -750,7 +748,7 @@ func (h *Handler) chkAuthZ(handler http.HandlerFunc, perm authz.Permission) http
return func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
//if the request is unauthenticated and we have the appropriate header get the userid from the header
// if the request is unauthenticated and we have the appropriate header get the userid from the header
requestUserID := r.Header.Get(HeaderRequestUserID)
ctx = WithUserID(ctx, requestUserID)
@ -890,7 +888,6 @@ func GetIP(r *http.Request) string {
forwardedList := strings.Split(forwarded, ",")
if forwardedList[0] != "" {
return forwardedList[0]
}
return r.RemoteAddr
@ -1275,7 +1272,6 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
}
func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
@ -1393,7 +1389,6 @@ func (h *Handler) writeBadRequest(w http.ResponseWriter, r *http.Request, err er
if e != nil {
h.logger.Errorf("write query response error: %v (while trying to write another error: %v)", e, err)
}
}
// handlePostSQL handles /sql requests.
@ -1411,7 +1406,6 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
}
b, err := io.ReadAll(r.Body)
if err != nil {
h.writeBadRequest(w, r, err)
return
@ -1564,7 +1558,6 @@ type SQLSchema struct {
}
func (h *Handler) handleCPUProfileStart(w http.ResponseWriter, r *http.Request) {
if h.pprofCPUProfileBuffer == nil {
h.pprofCPUProfileBuffer = bytes.NewBuffer(nil)
} else {
@ -1581,7 +1574,6 @@ func (h *Handler) handleCPUProfileStart(w http.ResponseWriter, r *http.Request)
}
func (h *Handler) handleCPUProfileStop(w http.ResponseWriter, r *http.Request) {
if h.pprofCPUProfileBuffer == nil {
http.Error(w, "no cpu profile in progress", http.StatusBadRequest)
return
@ -1596,7 +1588,7 @@ func (h *Handler) handleCPUProfileStop(w http.ResponseWriter, r *http.Request) {
// Date: Tue, 03 Nov 2020 18:31:36 GMT
// Content-Length: 939
//Send the headers
// Send the headers
by := h.pprofCPUProfileBuffer.Bytes()
w.Header().Set("Content-Disposition", "attachment; filename=\"profile\"")
w.Header().Set("Content-Type", "application/octet-stream")
@ -1750,7 +1742,6 @@ type _postIndexRequest postIndexRequest
// Custom Unmarshal JSON to validate request body when creating a new index.
func (p *postIndexRequest) UnmarshalJSON(b []byte) error {
// m is an overflow map used to capture additional, unexpected keys.
m := make(map[string]interface{})
if err := json.Unmarshal(b, &m); err != nil {
@ -1937,7 +1928,6 @@ func (h *Handler) handleGetPastQueries(w http.ResponseWriter, r *http.Request) {
if err := json.NewEncoder(w).Encode(queries); err != nil {
h.logger.Errorf("encoding GetActiveQueries response: %s", err)
}
}
func fieldOptionsToFunctionalOpts(opt fieldOptions) []FieldOption {
@ -2362,7 +2352,6 @@ func (h *Handler) doTransactionResponse(w http.ResponseWriter, err error, trns *
if err != nil {
h.logger.Errorf("encoding transaction response: %v", err)
}
}
func (h *Handler) handleGetTransaction(w http.ResponseWriter, r *http.Request) {
@ -3021,7 +3010,6 @@ func GetHTTPClient(t *tls.Config, opts ...ClientOption) *http.Client {
// handlePostImportAtomicRecord handles /import-atomic-record requests
func (h *Handler) handlePostImportAtomicRecord(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if error, code := validateProtobufHeader(r); error != "" {
http.Error(w, error, code)
@ -3785,6 +3773,7 @@ func (h *Handler) handleRestoreIDAlloc(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte("OK")) //nolint:errcheck
}
func (h *Handler) handlePostRestore(w http.ResponseWriter, r *http.Request) {
indexName, ok := mux.Vars(r)["index"]
if !ok {
@ -3802,7 +3791,7 @@ func (h *Handler) handlePostRestore(w http.ResponseWriter, r *http.Request) {
return
}
ctx := context.Background()
//validate shard for this node
// validate shard for this node
err = h.api.RestoreShard(ctx, indexName, shard, r.Body)
if err != nil {
http.Error(w, fmt.Sprintf("failed to restore shard %v %v err:%v", indexName, shard, err), http.StatusBadRequest)
@ -3814,6 +3803,62 @@ func (h *Handler) handlePostRestore(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("OK")) //nolint:errcheck
}
// handleDeleteDataframe handles DELETE /index/dataframe request.
func (h *Handler) handleDeleteDataframe(w http.ResponseWriter, r *http.Request) {
if !h.api.server.dataframeEnabled {
http.Error(w, "Dataframe is disabled", http.StatusBadRequest)
return
}
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
indexName := mux.Vars(r)["index"]
resp := successResponse{h: h}
err := h.api.DeleteDataframe(r.Context(), indexName)
resp.write(w, err)
}
func (h *Handler) handlePostDataframeRestore(w http.ResponseWriter, r *http.Request) {
indexName, ok := mux.Vars(r)["index"]
if !ok {
http.Error(w, "index name is required", http.StatusBadRequest)
return
}
shardID, ok := mux.Vars(r)["shardID"]
if !ok {
http.Error(w, "shardID is required", http.StatusBadRequest)
return
}
shard, err := strconv.ParseUint(shardID, 10, 64)
if err != nil {
http.Error(w, fmt.Sprintf("failed to parse shard %v %v err:%v", indexName, shardID, err), http.StatusBadRequest)
return
}
idx, err := h.api.Index(r.Context(), indexName)
if err != nil {
http.Error(w, fmt.Sprintf("Index %s Not Found", indexName), http.StatusNotFound)
return
}
filename := idx.GetDataFramePath(shard) + ".parquet"
dest, err := os.Create(filename)
if err != nil {
http.Error(w, fmt.Sprintf("failed to create restore dataframe shard %v %v err:%v", indexName, shard, err), http.StatusBadRequest)
return
}
_, err = io.Copy(dest, r.Body)
if err != nil {
http.Error(w, fmt.Sprintf("failed to copy restore dataframe shard %v %v err:%v", indexName, shard, err), http.StatusBadRequest)
return
}
w.Header().Add("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
w.Write([]byte("OK")) //nolint:errcheck
}
func (h *Handler) handleLogin(w http.ResponseWriter, r *http.Request) {
if h.auth == nil {
http.Error(w, "", http.StatusNoContent)
@ -4042,3 +4087,117 @@ func (h *Handler) handlePostSnapshotFieldKeys(w http.ResponseWriter, r *http.Req
func (h *Handler) handleGetHealth(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
func init() {
gob.Register(arrow.PrimitiveTypes.Int64)
gob.Register(arrow.PrimitiveTypes.Float64)
}
// EXPERIMENTAL API MAY CHANGE
func (h *Handler) handleGetDataframeSchema(w http.ResponseWriter, r *http.Request) {
if !h.api.server.dataframeEnabled {
http.Error(w, "Dataframe is disabled", http.StatusBadRequest)
return
}
indexName, ok := mux.Vars(r)["index"]
if !ok {
http.Error(w, "index name is required", http.StatusBadRequest)
return
}
parts, err := h.api.GetDataframeSchema(r.Context(), indexName)
if err != nil {
http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
return
}
if err := json.NewEncoder(w).Encode(parts); err != nil {
h.logger.Errorf("dataframe schema err: %s", err)
}
}
func (h *Handler) handleGetDataframe(w http.ResponseWriter, r *http.Request) {
if !h.api.server.dataframeEnabled {
http.Error(w, "Dataframe is disabled", http.StatusBadRequest)
return
}
indexName, ok := mux.Vars(r)["index"]
if !ok {
http.Error(w, "index name is required", http.StatusBadRequest)
return
}
shardString, ok := mux.Vars(r)["shard"]
if !ok {
http.Error(w, "shard is required", http.StatusBadRequest)
return
}
shard, err := strconv.ParseUint(shardString, 10, 64)
if err != nil {
http.Error(w, "shard should be an unsigned integer", http.StatusBadRequest)
return
}
idx, err := h.api.Index(r.Context(), indexName)
if err != nil {
http.Error(w, fmt.Sprintf("Index %s Not Found", indexName), http.StatusNotFound)
return
}
filename := idx.GetDataFramePath(shard) + ".parquet"
http.ServeFile(w, r, filename)
}
func (h *Handler) handlePostDataframe(w http.ResponseWriter, r *http.Request) {
if !h.api.server.dataframeEnabled {
http.Error(w, "Dataframe is disabled", http.StatusBadRequest)
return
}
// TODO(twg) 2022/09/29 validate the request
indexName, ok := mux.Vars(r)["index"]
if !ok {
http.Error(w, "index name is required", http.StatusBadRequest)
return
}
shardString, ok := mux.Vars(r)["shard"]
if !ok {
http.Error(w, "shard is required", http.StatusBadRequest)
return
}
shard, err := strconv.ParseUint(shardString, 10, 64)
if err != nil {
http.Error(w, "shard should be an unsigned integer", http.StatusBadRequest)
return
}
body, err := readBody(r)
if err != nil {
http.Error(w, "reading body", http.StatusBadRequest)
return
}
blob := bytes.NewReader(body)
dec := gob.NewDecoder(blob)
var changesetRequest ChangesetRequest
err = dec.Decode(&changesetRequest)
if err != nil {
http.Error(w, "decoding request", http.StatusBadRequest)
return
}
err = h.api.ApplyDataframeChangeset(r.Context(), indexName, &changesetRequest, shard)
if err != nil {
http.Error(w, fmt.Sprintf("err:%v", err), http.StatusBadRequest)
return
}
msg := ""
resp := struct {
Index string
Err string
}{
Index: indexName,
Err: msg,
}
w.Header().Add("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(resp)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
}

5
idk/kafka/source.go
View file

@ -210,7 +210,6 @@ func (r *Record) Commit(ctx context.Context) error {
return errors.New("cannot commit a record that has already been committed")
}
section, remaining := r.src.spool[:idx-base], r.src.spool[idx-base:]
sort.Slice(section, func(i, j int) bool {
if *section[i].Topic != *section[j].Topic {
return *section[i].Topic < *section[j].Topic
@ -222,14 +221,16 @@ func (r *Record) Commit(ctx context.Context) error {
p := int32(-1)
s := ""
r.src.highmarks = r.src.highmarks[:0]
// sort by increasing partition, decreasing offset
for _, x := range section {
if s != *x.Topic || p != x.Partition {
r.src.highmarks = append(r.src.highmarks, x)
}
s = *x.Topic
p = x.Partition
s = *x.Topic
}
committedOffsets, err := r.src.CommitMessages(r.src.highmarks)

26
index.go
View file

@ -68,7 +68,6 @@ type Index struct {
// NewIndex returns an existing (but possibly empty) instance of
// Index at path. It will not erase any prior content.
func NewIndex(holder *Holder, path, name string) (*Index, error) {
// Emulate what the spf13/cobra does, letting env vars override
// the defaults, because we may be under a simple "go test" run where
// not all that command line machinery has been spun up.
@ -110,6 +109,11 @@ func (i *Index) CreatedAt() int64 {
return i.createdAt
}
// DataframePath returns the path of the dataframes specific to an index
func (i *Index) DataframesPath() string {
return filepath.Join(i.path, DataframesDir)
}
// Name returns name of the index.
func (i *Index) Name() string { return i.name }
@ -194,9 +198,16 @@ func (i *Index) open(idx *disco.Index) (err error) {
defer i.mu.Unlock()
// Ensure the path exists.
i.holder.Logger.Debugf("ensure index path exists: %s", i.FieldsPath())
if err := os.MkdirAll(i.FieldsPath(), 0750); err != nil {
if err := os.MkdirAll(i.FieldsPath(), 0o750); err != nil {
return errors.Wrap(err, "creating directory")
}
// Ensure the dataframes path exists
i.holder.Logger.Debugf("ensure dataframes path exists: %s", i.DataframesPath())
if err := os.MkdirAll(i.DataframesPath(), 0o750); err != nil {
return errors.Wrap(err, "creating dataframes directory")
}
i.closing = make(chan struct{})
// fmt.Printf("new channel %p for index %p\n", i.closing, i)
@ -496,7 +507,7 @@ func (i *Index) AvailableShards(localOnly bool) *roaring.Bitmap {
b := roaring.NewBitmap()
for _, f := range i.fields {
//b.Union(f.AvailableShards(localOnly))
// b.Union(f.AvailableShards(localOnly))
b.UnionInPlace(f.AvailableShards(localOnly))
}
@ -840,7 +851,6 @@ func (i *Index) UpdateFieldLocal(cfm *CreateFieldMessage, update FieldUpdate) er
}
return nil
}
// createFieldIfNotExists creates the field if it does not already exist in the
@ -974,6 +984,14 @@ func (i *Index) SetTranslatePartitions(tp dax.VersionedPartitions) {
i.translatePartitions = tp
}
// TODO (twg) refine parquet strategy a bit
func (i *Index) GetDataFramePath(shard uint64) string {
path := i.DataframesPath()
os.MkdirAll(i.path, 0o750)
shardpad := fmt.Sprintf("%04d", shard)
return filepath.Join(path, shardpad)
}
type indexSlice []*Index
func (p indexSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

23
internal_client.go
View file

@ -1715,6 +1715,27 @@ func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*Transa
return tr.Transaction, err
}
func (c *InternalClient) GetDataframeShard(ctx context.Context, index string, shard uint64) (*http.Response, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.GetDataframeShard")
defer span.Finish()
// Execute request against the host.
path := fmt.Sprintf("/index/%v/dataframe/%04d", index, shard)
// Build request.
url := uriPathToURL(c.defaultURI, path)
req, err := http.NewRequest("GET", url.String(), nil)
if err != nil {
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+Version)
req.Header.Set("Accept", "application/octet-stream")
AddAuthToken(ctx, &req.Header)
// Execute request.
return c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
}
type executeOpts struct {
// giveRawResponse instructs executeRequest not to process the
// respStatusCode and try to extract errors or whatever.
@ -1730,6 +1751,7 @@ func giveRawResponse(b bool) executeRequestOption {
eo.giveRawResponse = b
}
}
func forwardAuthHeader(b bool) executeRequestOption {
return func(eo *executeOpts) {
eo.forwardAuthHeader = b
@ -2084,6 +2106,7 @@ func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context,
return resp.Body, nil
}
func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, readerFunc func() (io.Reader, error)) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportIndexKeys")
defer span.Finish()

1
metrics.go
View file

@ -51,4 +51,5 @@ const (
MetricExclusiveTransactionBlocked = "transaction_exclusive_blocked"
MetricPqlQueries = "pql_queries_total"
MetricSqlQueries = "sql_queries_total"
MetricDeleteDataframe = "delete_dataframe"
)

400
pb/private.pb.go
View file

@ -2849,6 +2849,53 @@ func (m *ShardedIngestRequest) GetOps() map[uint64]*ShardIngestOperations {
return nil
}
type DeleteDataframeMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *DeleteDataframeMessage) Reset() { *m = DeleteDataframeMessage{} }
func (m *DeleteDataframeMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteDataframeMessage) ProtoMessage() {}
func (*DeleteDataframeMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_d2a91b51c7bdc125, []int{45}
}
func (m *DeleteDataframeMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *DeleteDataframeMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_DeleteDataframeMessage.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *DeleteDataframeMessage) XXX_Merge(src proto.Message) {
xxx_messageInfo_DeleteDataframeMessage.Merge(m, src)
}
func (m *DeleteDataframeMessage) XXX_Size() int {
return m.Size()
}
func (m *DeleteDataframeMessage) XXX_DiscardUnknown() {
xxx_messageInfo_DeleteDataframeMessage.DiscardUnknown(m)
}
var xxx_messageInfo_DeleteDataframeMessage proto.InternalMessageInfo
func (m *DeleteDataframeMessage) GetIndex() string {
if m != nil {
return m.Index
}
return ""
}
func init() {
proto.RegisterType((*IndexMeta)(nil), "pb.IndexMeta")
proto.RegisterType((*FieldOptions)(nil), "pb.FieldOptions")
@ -2898,120 +2945,122 @@ func init() {
proto.RegisterType((*ShardIngestOperations)(nil), "pb.ShardIngestOperations")
proto.RegisterType((*ShardedIngestRequest)(nil), "pb.ShardedIngestRequest")
proto.RegisterMapType((map[uint64]*ShardIngestOperations)(nil), "pb.ShardedIngestRequest.OpsEntry")
proto.RegisterType((*DeleteDataframeMessage)(nil), "pb.DeleteDataframeMessage")
}
func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) }
var fileDescriptor_d2a91b51c7bdc125 = []byte{
// 1728 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0x23, 0x49,
0x15, 0xa7, 0xff, 0xc4, 0x7f, 0x9e, 0xe3, 0x8c, 0x53, 0x1b, 0x86, 0x9e, 0xec, 0x10, 0x79, 0x0a,
0x34, 0x63, 0x46, 0x22, 0x88, 0xec, 0x61, 0x11, 0x7b, 0xd9, 0x89, 0x9d, 0x59, 0xcc, 0xee, 0xcc,
0x64, 0x2b, 0x7f, 0x8e, 0xa0, 0x4a, 0xbb, 0x94, 0xb4, 0xd2, 0xee, 0x36, 0xdd, 0xed, 0xc4, 0xde,
0x03, 0x12, 0x48, 0x08, 0x2e, 0xdc, 0x11, 0x07, 0xbe, 0x05, 0x5f, 0x80, 0x13, 0x17, 0x24, 0x3e,
0x02, 0x1a, 0xbe, 0x08, 0xaa, 0x57, 0x55, 0xdd, 0x65, 0xc7, 0x49, 0x86, 0x68, 0x6f, 0xfd, 0x7e,
0xaf, 0xfa, 0xd5, 0xef, 0xfd, 0xa9, 0x57, 0xaf, 0x1b, 0xda, 0x93, 0x2c, 0xba, 0xe2, 0x85, 0xd8,
0x9d, 0x64, 0x69, 0x91, 0x12, 0x77, 0x72, 0xb6, 0xbd, 0x3e, 0x99, 0x9e, 0xc5, 0x51, 0xa8, 0x10,
0x1a, 0x41, 0x73, 0x98, 0x8c, 0xc4, 0xec, 0x8d, 0x28, 0x38, 0x21, 0xe0, 0x7f, 0x29, 0xe6, 0x79,
0xe0, 0x75, 0x9d, 0x5e, 0x83, 0xe1, 0x33, 0x79, 0x0e, 0x1b, 0xc7, 0x19, 0x0f, 0x2f, 0x0f, 0x66,
0x51, 0x5e, 0x88, 0x24, 0x14, 0x81, 0x8f, 0xda, 0x25, 0x94, 0x74, 0xa1, 0x35, 0x10, 0x79, 0x98,
0x45, 0x93, 0x22, 0x4a, 0x93, 0x60, 0xad, 0xeb, 0xf4, 0x9a, 0xcc, 0x86, 0xe8, 0x3f, 0x3c, 0x58,
0x7f, 0x1d, 0x89, 0x78, 0xf4, 0x0e, 0xe5, 0x5c, 0x6e, 0x77, 0x3c, 0x9f, 0x88, 0xa0, 0x81, 0x6b,
0xf1, 0x99, 0x3c, 0x85, 0x66, 0x9f, 0x87, 0x17, 0x02, 0x15, 0x1e, 0x2a, 0x2a, 0xa0, 0xd4, 0x1e,
0x45, 0xdf, 0x28, 0x1e, 0x6d, 0x56, 0x01, 0x92, 0xc2, 0x71, 0x34, 0x16, 0x5f, 0x4f, 0x79, 0x52,
0x4c, 0xc7, 0x86, 0x82, 0x05, 0x91, 0xc7, 0x50, 0x7b, 0x17, 0x8f, 0xde, 0x44, 0x49, 0xd0, 0xec,
0x3a, 0x3d, 0x8f, 0x69, 0xc9, 0xe0, 0x7c, 0x16, 0x40, 0x85, 0xf3, 0x59, 0x19, 0x90, 0xd6, 0x62,
0x40, 0xde, 0xa6, 0x47, 0x05, 0x4f, 0x46, 0x3c, 0x1b, 0x9d, 0x46, 0xe2, 0x3a, 0x58, 0x57, 0x01,
0x59, 0x44, 0xe5, 0xbb, 0xfb, 0x3c, 0x17, 0x41, 0x1b, 0x2d, 0xe2, 0x33, 0xd9, 0x86, 0xc6, 0x7e,
0x54, 0x0c, 0xc4, 0xa4, 0xb8, 0x08, 0x36, 0xba, 0x4e, 0xcf, 0x67, 0xa5, 0x4c, 0xb6, 0x60, 0xed,
0x28, 0xe4, 0xb1, 0x08, 0x1e, 0xe1, 0x0b, 0x4a, 0x20, 0x14, 0xd6, 0x5f, 0xa7, 0x99, 0x88, 0xce,
0x13, 0x4c, 0x53, 0xd0, 0x41, 0xa7, 0x16, 0x30, 0xf2, 0x7d, 0xf0, 0xa4, 0x4b, 0x9b, 0x5d, 0xa7,
0xd7, 0xda, 0x6b, 0xed, 0x4e, 0xce, 0x76, 0x07, 0x22, 0x8c, 0xc6, 0x3c, 0x66, 0x12, 0x47, 0x35,
0x9f, 0x05, 0x64, 0x95, 0x9a, 0xcf, 0x24, 0x27, 0x19, 0xa2, 0x93, 0x24, 0x2a, 0x82, 0x8f, 0xd0,
0x7a, 0x29, 0x93, 0x0e, 0x78, 0xc7, 0xc7, 0x5f, 0x05, 0x5b, 0x08, 0xcb, 0x47, 0x4a, 0x61, 0x63,
0x38, 0x9e, 0xa4, 0x59, 0xc1, 0x44, 0x3e, 0x49, 0x93, 0x5c, 0xc8, 0x35, 0x07, 0x59, 0x16, 0x38,
0x6a, 0xcd, 0x41, 0x96, 0xd1, 0xdf, 0x42, 0x67, 0x3f, 0x4e, 0xc3, 0xcb, 0x01, 0x2f, 0x38, 0x13,
0xbf, 0x99, 0x8a, 0xbc, 0x90, 0xde, 0x29, 0x07, 0xd4, 0x3a, 0x25, 0x48, 0x14, 0x2b, 0x22, 0x70,
0x15, 0x8a, 0x82, 0x8c, 0x1c, 0xc6, 0x55, 0x25, 0x10, 0x9f, 0x31, 0x3a, 0x17, 0x3c, 0x1b, 0x61,
0xd6, 0x7d, 0xa6, 0x04, 0x89, 0xe2, 0x4e, 0x58, 0x29, 0x3e, 0x53, 0x02, 0x1d, 0xc2, 0xa6, 0xb5,
0xbf, 0xa6, 0xf9, 0x18, 0x6a, 0x2c, 0xbd, 0x1e, 0x0e, 0xf2, 0xc0, 0xe9, 0x7a, 0x3d, 0x9f, 0x69,
0x09, 0x4b, 0x2a, 0x8d, 0xa7, 0xe3, 0x44, 0xaa, 0x5c, 0x54, 0x55, 0x00, 0x7d, 0x02, 0x6b, 0x58,
0x5f, 0xd2, 0xcb, 0xea, 0x5d, 0xf9, 0x48, 0x7f, 0xe7, 0x40, 0xf3, 0x0d, 0x9f, 0x21, 0x91, 0x9c,
0x7c, 0x0a, 0x0d, 0x93, 0x7d, 0x5c, 0xd4, 0xda, 0xfb, 0x58, 0x46, 0xba, 0x5c, 0xb0, 0x6b, 0xb4,
0x07, 0x49, 0x91, 0xcd, 0x59, 0xb9, 0x78, 0xfb, 0x33, 0x68, 0x2f, 0xa8, 0xe4, 0x4e, 0x97, 0x62,
0x6e, 0xe2, 0x79, 0x29, 0xe6, 0xd2, 0xcb, 0x2b, 0x1e, 0x4f, 0x05, 0x46, 0xc9, 0x67, 0x4a, 0xf8,
0xb9, 0xfb, 0x33, 0x87, 0x9e, 0x02, 0xe9, 0x67, 0x82, 0x17, 0x02, 0x37, 0x79, 0x23, 0xf2, 0x9c,
0x9f, 0x8b, 0xfb, 0x62, 0xed, 0xd9, 0xb1, 0x2e, 0xe3, 0xea, 0x5a, 0x71, 0xa5, 0x2f, 0x81, 0x0c,
0x44, 0x2c, 0x0a, 0xa1, 0x7b, 0xc3, 0x1d, 0x76, 0x65, 0x1c, 0x34, 0x89, 0xfb, 0x17, 0x93, 0x67,
0xe0, 0xcb, 0x4e, 0x83, 0xbb, 0xb5, 0xf6, 0xda, 0x32, 0x44, 0x65, 0xfb, 0x61, 0xa8, 0xc2, 0x84,
0xa0, 0xb9, 0xd1, 0xab, 0x02, 0xb9, 0x7a, 0xac, 0x02, 0xa4, 0xd9, 0x77, 0xd7, 0x89, 0xc8, 0x74,
0x71, 0x28, 0x81, 0xfe, 0xb5, 0xe4, 0x80, 0x5e, 0x7d, 0x60, 0x20, 0x16, 0x8a, 0xee, 0x87, 0x9a,
0x99, 0x87, 0xcc, 0x3a, 0x92, 0x99, 0xdd, 0xac, 0x56, 0x91, 0xf3, 0x3f, 0x8c, 0xdc, 0x1f, 0x1c,
0x20, 0x27, 0x93, 0xd1, 0x32, 0xb9, 0xd7, 0xab, 0x28, 0x23, 0xd3, 0xd6, 0xde, 0x63, 0xb9, 0xfd,
0x4d, 0x2d, 0x5b, 0xe5, 0xe4, 0x0b, 0xa8, 0x29, 0xeb, 0x3a, 0xa8, 0x8f, 0x4a, 0xea, 0x0a, 0x66,
0x5a, 0x4d, 0x3f, 0x83, 0x96, 0x05, 0x63, 0xcf, 0x53, 0xbd, 0x5a, 0x45, 0x47, 0x4b, 0xd2, 0x89,
0xd3, 0xb2, 0xda, 0x9a, 0x4c, 0x09, 0xf4, 0x73, 0x53, 0x11, 0x0f, 0x0d, 0x30, 0x0d, 0xe1, 0x63,
0x65, 0xe1, 0xd5, 0x15, 0x8f, 0x62, 0x7e, 0x16, 0xff, 0x5f, 0x45, 0xbb, 0x90, 0xab, 0x00, 0xea,
0xf8, 0xee, 0x70, 0xa0, 0x0f, 0xbe, 0x11, 0xe9, 0x14, 0xaa, 0x1e, 0xf2, 0x96, 0x8f, 0x85, 0xb6,
0x86, 0xcf, 0x65, 0x8a, 0xdd, 0x3b, 0x53, 0x2c, 0xfd, 0x8f, 0xc4, 0xb5, 0xbc, 0x05, 0x3d, 0xf4,
0x5f, 0x0a, 0x77, 0x27, 0x9e, 0xfe, 0x18, 0x6a, 0x47, 0xe1, 0x85, 0x18, 0x73, 0xf2, 0x03, 0xa8,
0x23, 0x73, 0x91, 0xeb, 0x36, 0xd0, 0x2c, 0x6b, 0x9c, 0x19, 0x8d, 0xac, 0x08, 0xed, 0xdf, 0x2a,
0x9a, 0x0b, 0x5b, 0xb9, 0xcb, 0x35, 0xf6, 0x02, 0xea, 0x9a, 0x2f, 0x56, 0xd9, 0x8d, 0x43, 0x64,
0xb4, 0xe4, 0x19, 0xd4, 0xd0, 0xbb, 0x3c, 0xf0, 0x2b, 0x22, 0x88, 0x30, 0xad, 0xa0, 0x07, 0xe0,
0x9d, 0xb0, 0xa1, 0xac, 0x04, 0x64, 0x6f, 0x68, 0x68, 0x49, 0x92, 0xfb, 0x45, 0x9a, 0x17, 0x3a,
0xf6, 0xf8, 0x2c, 0xb1, 0xc3, 0x34, 0x53, 0x07, 0xb3, 0xcd, 0xf0, 0x99, 0xfe, 0xc9, 0x01, 0xff,
0x6d, 0x3a, 0x12, 0x64, 0x03, 0xdc, 0xe1, 0x40, 0x1b, 0x71, 0x87, 0x03, 0xf2, 0x04, 0xed, 0xeb,
0x78, 0xd7, 0xe5, 0xfe, 0x27, 0x6c, 0xc8, 0x70, 0xcf, 0xa7, 0xd0, 0x1c, 0xe6, 0x87, 0x59, 0x34,
0xe6, 0xd9, 0x5c, 0xcf, 0x1b, 0x15, 0x80, 0x5d, 0xa9, 0x90, 0x25, 0xed, 0xab, 0xb4, 0xa3, 0x40,
0x9e, 0x41, 0xfd, 0x0b, 0x76, 0xd8, 0x97, 0x26, 0xd7, 0x16, 0x4d, 0x1a, 0x9c, 0x7e, 0x0e, 0x1d,
0xc9, 0x04, 0xd7, 0x9b, 0xca, 0x7a, 0x0c, 0x35, 0x89, 0x95, 0xcc, 0xb4, 0x54, 0x6d, 0xe2, 0x5a,
0x9b, 0xd0, 0xd7, 0xca, 0xc2, 0xc1, 0x95, 0x48, 0x0a, 0xab, 0x36, 0x51, 0x46, 0x03, 0x6d, 0xa6,
0x04, 0xf2, 0x54, 0x79, 0xad, 0xdd, 0x6b, 0x48, 0x2e, 0x52, 0x66, 0x88, 0xd2, 0x39, 0x80, 0x61,
0x32, 0xcd, 0xcb, 0xb5, 0xce, 0xaa, 0xb5, 0x84, 0x9a, 0xf2, 0xd1, 0xdd, 0x07, 0xa4, 0x5e, 0x21,
0xcc, 0x14, 0xd6, 0x8f, 0xaa, 0xc2, 0x52, 0xf9, 0x7c, 0x54, 0xe6, 0x5d, 0xed, 0x51, 0x95, 0xd7,
0x05, 0xb4, 0x2c, 0x7c, 0x65, 0x8d, 0xbd, 0x28, 0x8b, 0xc3, 0xad, 0x8c, 0x21, 0xa2, 0x8d, 0x69,
0xf5, 0xdd, 0xdd, 0x98, 0x46, 0xba, 0xa5, 0xdc, 0xb1, 0x53, 0x0f, 0x1e, 0x2d, 0x1e, 0x78, 0x73,
0xcb, 0x2e, 0xc3, 0xf7, 0x6c, 0xf5, 0x47, 0x07, 0xda, 0xfd, 0x78, 0x9a, 0x17, 0x22, 0x2b, 0x63,
0xda, 0xd4, 0x40, 0x99, 0xda, 0x0a, 0x58, 0x9d, 0x5d, 0xb2, 0x03, 0x6b, 0x32, 0xe2, 0xea, 0x70,
0xdb, 0x89, 0x50, 0xb0, 0x95, 0x09, 0xff, 0xb6, 0x4c, 0xd0, 0x53, 0x68, 0xec, 0x1f, 0x0d, 0xbf,
0xc8, 0xd2, 0xe9, 0x64, 0xa5, 0xc7, 0x66, 0xac, 0x75, 0xad, 0xb1, 0xb6, 0xa3, 0x46, 0x34, 0xe5,
0x15, 0x4e, 0x65, 0x1d, 0x35, 0x95, 0xf9, 0x1a, 0xe1, 0x33, 0x7a, 0x04, 0x9b, 0xca, 0x5d, 0xd9,
0x71, 0x1e, 0xd2, 0x16, 0xcd, 0xdc, 0xe4, 0x55, 0x73, 0x93, 0x34, 0xaa, 0xba, 0xee, 0xb7, 0x69,
0xf4, 0x5f, 0x2e, 0x6c, 0x32, 0x91, 0x47, 0xdf, 0x88, 0x61, 0x92, 0x17, 0xd9, 0x34, 0x34, 0x17,
0xc7, 0x2f, 0xd3, 0x33, 0x9d, 0x0b, 0x8f, 0x29, 0xe1, 0xee, 0x53, 0x42, 0x28, 0xd4, 0xed, 0x26,
0x60, 0x2f, 0x30, 0x0a, 0xf2, 0x12, 0xea, 0x47, 0xe9, 0x34, 0x0b, 0xcb, 0xca, 0xc7, 0xce, 0xad,
0xf6, 0x57, 0x0a, 0x66, 0x16, 0x90, 0x2f, 0x81, 0x1c, 0x67, 0x3c, 0xc9, 0x63, 0x2e, 0x29, 0x99,
0xd7, 0x1a, 0xd5, 0x40, 0x66, 0x69, 0x17, 0x2c, 0xac, 0x78, 0x8d, 0xec, 0xda, 0x47, 0x38, 0xa8,
0x23, 0xbf, 0x0d, 0xc3, 0x4f, 0x9f, 0x13, 0xfb, 0x90, 0x7f, 0xba, 0x54, 0xa1, 0x41, 0x0d, 0x5f,
0xd9, 0xc4, 0xcb, 0xdc, 0x56, 0xb0, 0xc5, 0x75, 0xf4, 0xf7, 0x0e, 0xac, 0xdb, 0x6c, 0xee, 0x69,
0x17, 0x65, 0xfa, 0xdc, 0xfb, 0xe7, 0x3b, 0x93, 0x3e, 0x7f, 0xd5, 0x2c, 0xbd, 0x66, 0xcf, 0x7c,
0x29, 0x7c, 0xef, 0x96, 0xe0, 0x3c, 0x88, 0x4e, 0x17, 0x5a, 0x87, 0x3c, 0x2b, 0x22, 0x69, 0x4c,
0xdf, 0xd3, 0x6b, 0xcc, 0x86, 0xa8, 0x80, 0x27, 0x37, 0x8a, 0xa8, 0x9f, 0x8e, 0x27, 0xb2, 0x5a,
0x1f, 0x54, 0x4c, 0xb2, 0x4d, 0x67, 0x59, 0x9a, 0x99, 0x08, 0xa0, 0x40, 0xf7, 0xa1, 0x71, 0x9c,
0x4e, 0xd2, 0x38, 0x3d, 0x9f, 0xdf, 0xd3, 0x32, 0x02, 0xa8, 0xab, 0xab, 0x41, 0xb5, 0xa8, 0x26,
0x33, 0x22, 0xfd, 0x48, 0xd6, 0x7b, 0xc8, 0xe3, 0x70, 0x1a, 0xf3, 0x42, 0xe0, 0x17, 0x01, 0x82,
0x5f, 0xa5, 0x7c, 0xa4, 0xba, 0x82, 0x3e, 0x5a, 0xf4, 0xd7, 0xba, 0x00, 0x39, 0xba, 0x63, 0x5d,
0x41, 0xaf, 0x42, 0x7b, 0xd6, 0x52, 0x12, 0xf9, 0x29, 0xb4, 0xac, 0xd5, 0xf6, 0x00, 0x67, 0xc1,
0xcc, 0x5e, 0x43, 0xff, 0xee, 0x2c, 0xbc, 0x73, 0xe3, 0xce, 0xd5, 0x5b, 0x5d, 0xa9, 0x20, 0x35,
0x98, 0x96, 0xa4, 0xeb, 0x07, 0xb3, 0x30, 0x9e, 0xe6, 0x52, 0xa5, 0x2f, 0xdc, 0x12, 0x90, 0xae,
0xcb, 0x8f, 0xbe, 0x74, 0x6a, 0x86, 0x1b, 0x23, 0xca, 0xcf, 0xc3, 0x81, 0xe0, 0xa3, 0x38, 0x4a,
0x04, 0xd6, 0x8b, 0xc7, 0x4a, 0x99, 0xbc, 0x54, 0x3d, 0xd6, 0x14, 0xfa, 0xd6, 0x12, 0x71, 0xd4,
0xa9, 0xce, 0x9b, 0x53, 0x02, 0x9d, 0x65, 0x15, 0xdd, 0x02, 0xa2, 0x2a, 0xe0, 0xd5, 0x59, 0x9a,
0x99, 0xdb, 0x96, 0xf6, 0x4d, 0x73, 0x91, 0xd1, 0xbf, 0xef, 0x12, 0xaf, 0x22, 0xeb, 0xda, 0x91,
0xa5, 0xbf, 0x82, 0x0d, 0x3d, 0xdb, 0x89, 0x0c, 0x0b, 0x5a, 0x06, 0x80, 0x89, 0x30, 0x95, 0x63,
0xa2, 0xf9, 0x8e, 0xab, 0x00, 0x69, 0x07, 0x07, 0x5d, 0x73, 0x3b, 0x69, 0x09, 0x67, 0xa3, 0xe8,
0x3c, 0x11, 0x23, 0xbc, 0x31, 0x3c, 0xa6, 0x25, 0xfa, 0x67, 0x17, 0xb6, 0xd4, 0xd0, 0x99, 0x9c,
0x8b, 0xbc, 0xa8, 0xb6, 0xc1, 0xb1, 0x1a, 0xfb, 0x7f, 0x39, 0x56, 0xe3, 0x0d, 0xf0, 0x1c, 0x36,
0xfa, 0xb1, 0xe0, 0x59, 0xc5, 0x41, 0x6d, 0xb4, 0x84, 0xca, 0x73, 0x83, 0x88, 0xbe, 0x9e, 0xd5,
0x10, 0x6a, 0x43, 0x64, 0x1f, 0x1a, 0xda, 0x35, 0xd3, 0x10, 0x9f, 0xe3, 0x2d, 0xb5, 0x82, 0x8d,
0x99, 0x6f, 0x73, 0xfd, 0xd5, 0x69, 0xc4, 0xed, 0x77, 0xd0, 0x5e, 0x50, 0xad, 0xf8, 0xea, 0xec,
0xd9, 0x5f, 0x9d, 0xad, 0x3d, 0x62, 0x8d, 0xcb, 0xda, 0xba, 0xfd, 0x25, 0xda, 0x87, 0xef, 0xae,
0x22, 0x90, 0x93, 0x97, 0xe0, 0x49, 0xa2, 0x6a, 0x18, 0x0e, 0x6e, 0x23, 0xca, 0xe4, 0x22, 0xfa,
0x37, 0x47, 0x07, 0x55, 0x68, 0xbd, 0xf9, 0x7b, 0xf0, 0x89, 0x6d, 0xe4, 0x59, 0x69, 0x64, 0x69,
0xd9, 0x6e, 0xe9, 0xa8, 0x5c, 0xbd, 0xfd, 0x35, 0x34, 0x56, 0xb9, 0xe7, 0x2b, 0xf7, 0x7e, 0xb2,
0xe8, 0xde, 0x93, 0xdb, 0x98, 0xe5, 0x96, 0x97, 0xfb, 0x9d, 0x7f, 0xbe, 0xdf, 0x71, 0xfe, 0xfd,
0x7e, 0xc7, 0xf9, 0xcf, 0xfb, 0x1d, 0xe7, 0x2f, 0xff, 0xdd, 0xf9, 0xce, 0x59, 0x0d, 0x7f, 0xa3,
0x7d, 0xf2, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa1, 0xcf, 0xef, 0xf0, 0x69, 0x13, 0x00, 0x00,
// 1742 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x6f, 0x23, 0x49,
0x15, 0xa6, 0x2f, 0xf1, 0xe5, 0x38, 0xce, 0x38, 0xb5, 0x21, 0xf4, 0x64, 0x87, 0xc8, 0x53, 0xa0,
0x19, 0x33, 0x12, 0x41, 0x64, 0x1f, 0x16, 0xb1, 0x2f, 0x3b, 0xb1, 0x33, 0x8b, 0xd9, 0x9d, 0xcb,
0x56, 0x2e, 0x8f, 0xa0, 0x4a, 0xbb, 0x48, 0x5a, 0x69, 0x77, 0x9b, 0xee, 0x76, 0x62, 0xef, 0x03,
0x12, 0x48, 0x08, 0x5e, 0x78, 0x47, 0x3c, 0xf0, 0x2f, 0xf8, 0x03, 0x3c, 0xf1, 0x82, 0xc4, 0x4f,
0x40, 0xc3, 0x1f, 0x41, 0x75, 0xaa, 0xaa, 0xbb, 0xec, 0x38, 0xc9, 0x10, 0xf1, 0xd6, 0xe7, 0x3b,
0xd5, 0xa7, 0xbe, 0x73, 0xa9, 0x53, 0xa7, 0x1b, 0xda, 0x93, 0x2c, 0xba, 0xe2, 0x85, 0xd8, 0x9b,
0x64, 0x69, 0x91, 0x12, 0x77, 0x72, 0xb6, 0xb3, 0x3e, 0x99, 0x9e, 0xc5, 0x51, 0xa8, 0x10, 0x1a,
0x41, 0x73, 0x98, 0x8c, 0xc4, 0xec, 0xb5, 0x28, 0x38, 0x21, 0xe0, 0x7f, 0x29, 0xe6, 0x79, 0xe0,
0x75, 0x9d, 0x5e, 0x83, 0xe1, 0x33, 0x79, 0x06, 0x1b, 0xc7, 0x19, 0x0f, 0x2f, 0x0f, 0x67, 0x51,
0x5e, 0x88, 0x24, 0x14, 0x81, 0x8f, 0xda, 0x25, 0x94, 0x74, 0xa1, 0x35, 0x10, 0x79, 0x98, 0x45,
0x93, 0x22, 0x4a, 0x93, 0x60, 0xad, 0xeb, 0xf4, 0x9a, 0xcc, 0x86, 0xe8, 0xdf, 0x3d, 0x58, 0x7f,
0x15, 0x89, 0x78, 0xf4, 0x16, 0xe5, 0x5c, 0x6e, 0x77, 0x3c, 0x9f, 0x88, 0xa0, 0x81, 0x6b, 0xf1,
0x99, 0x3c, 0x81, 0x66, 0x9f, 0x87, 0x17, 0x02, 0x15, 0x1e, 0x2a, 0x2a, 0xa0, 0xd4, 0x1e, 0x45,
0xdf, 0x28, 0x1e, 0x6d, 0x56, 0x01, 0x92, 0xc2, 0x71, 0x34, 0x16, 0x5f, 0x4f, 0x79, 0x52, 0x4c,
0xc7, 0x86, 0x82, 0x05, 0x91, 0x6d, 0xa8, 0xbd, 0x8d, 0x47, 0xaf, 0xa3, 0x24, 0x68, 0x76, 0x9d,
0x9e, 0xc7, 0xb4, 0x64, 0x70, 0x3e, 0x0b, 0xa0, 0xc2, 0xf9, 0xac, 0x0c, 0x48, 0x6b, 0x31, 0x20,
0x6f, 0xd2, 0xa3, 0x82, 0x27, 0x23, 0x9e, 0x8d, 0x4e, 0x23, 0x71, 0x1d, 0xac, 0xab, 0x80, 0x2c,
0xa2, 0xf2, 0xdd, 0x03, 0x9e, 0x8b, 0xa0, 0x8d, 0x16, 0xf1, 0x99, 0xec, 0x40, 0xe3, 0x20, 0x2a,
0x06, 0x62, 0x52, 0x5c, 0x04, 0x1b, 0x5d, 0xa7, 0xe7, 0xb3, 0x52, 0x26, 0x5b, 0xb0, 0x76, 0x14,
0xf2, 0x58, 0x04, 0x8f, 0xf0, 0x05, 0x25, 0x10, 0x0a, 0xeb, 0xaf, 0xd2, 0x4c, 0x44, 0xe7, 0x09,
0xa6, 0x29, 0xe8, 0xa0, 0x53, 0x0b, 0x18, 0xf9, 0x2e, 0x78, 0xd2, 0xa5, 0xcd, 0xae, 0xd3, 0x6b,
0xed, 0xb7, 0xf6, 0x26, 0x67, 0x7b, 0x03, 0x11, 0x46, 0x63, 0x1e, 0x33, 0x89, 0xa3, 0x9a, 0xcf,
0x02, 0xb2, 0x4a, 0xcd, 0x67, 0x92, 0x93, 0x0c, 0xd1, 0x49, 0x12, 0x15, 0xc1, 0x47, 0x68, 0xbd,
0x94, 0x49, 0x07, 0xbc, 0xe3, 0xe3, 0xaf, 0x82, 0x2d, 0x84, 0xe5, 0x23, 0xa5, 0xb0, 0x31, 0x1c,
0x4f, 0xd2, 0xac, 0x60, 0x22, 0x9f, 0xa4, 0x49, 0x2e, 0xe4, 0x9a, 0xc3, 0x2c, 0x0b, 0x1c, 0xb5,
0xe6, 0x30, 0xcb, 0xe8, 0x6f, 0xa0, 0x73, 0x10, 0xa7, 0xe1, 0xe5, 0x80, 0x17, 0x9c, 0x89, 0x5f,
0x4f, 0x45, 0x5e, 0x48, 0xef, 0x94, 0x03, 0x6a, 0x9d, 0x12, 0x24, 0x8a, 0x15, 0x11, 0xb8, 0x0a,
0x45, 0x41, 0x46, 0x0e, 0xe3, 0xaa, 0x12, 0x88, 0xcf, 0x18, 0x9d, 0x0b, 0x9e, 0x8d, 0x30, 0xeb,
0x3e, 0x53, 0x82, 0x44, 0x71, 0x27, 0xac, 0x14, 0x9f, 0x29, 0x81, 0x0e, 0x61, 0xd3, 0xda, 0x5f,
0xd3, 0xdc, 0x86, 0x1a, 0x4b, 0xaf, 0x87, 0x83, 0x3c, 0x70, 0xba, 0x5e, 0xcf, 0x67, 0x5a, 0xc2,
0x92, 0x4a, 0xe3, 0xe9, 0x38, 0x91, 0x2a, 0x17, 0x55, 0x15, 0x40, 0x1f, 0xc3, 0x1a, 0xd6, 0x97,
0xf4, 0xb2, 0x7a, 0x57, 0x3e, 0xd2, 0xdf, 0x3a, 0xd0, 0x7c, 0xcd, 0x67, 0x48, 0x24, 0x27, 0x9f,
0x42, 0xc3, 0x64, 0x1f, 0x17, 0xb5, 0xf6, 0x3f, 0x96, 0x91, 0x2e, 0x17, 0xec, 0x19, 0xed, 0x61,
0x52, 0x64, 0x73, 0x56, 0x2e, 0xde, 0xf9, 0x0c, 0xda, 0x0b, 0x2a, 0xb9, 0xd3, 0xa5, 0x98, 0x9b,
0x78, 0x5e, 0x8a, 0xb9, 0xf4, 0xf2, 0x8a, 0xc7, 0x53, 0x81, 0x51, 0xf2, 0x99, 0x12, 0x7e, 0xea,
0xfe, 0xc4, 0xa1, 0xa7, 0x40, 0xfa, 0x99, 0xe0, 0x85, 0xc0, 0x4d, 0x5e, 0x8b, 0x3c, 0xe7, 0xe7,
0xe2, 0xbe, 0x58, 0x7b, 0x76, 0xac, 0xcb, 0xb8, 0xba, 0x56, 0x5c, 0xe9, 0x0b, 0x20, 0x03, 0x11,
0x8b, 0x42, 0xe8, 0xde, 0x70, 0x87, 0x5d, 0x19, 0x07, 0x4d, 0xe2, 0xfe, 0xc5, 0xe4, 0x29, 0xf8,
0xb2, 0xd3, 0xe0, 0x6e, 0xad, 0xfd, 0xb6, 0x0c, 0x51, 0xd9, 0x7e, 0x18, 0xaa, 0x30, 0x21, 0x68,
0x6e, 0xf4, 0xb2, 0x40, 0xae, 0x1e, 0xab, 0x00, 0x69, 0xf6, 0xed, 0x75, 0x22, 0x32, 0x5d, 0x1c,
0x4a, 0xa0, 0x7f, 0x29, 0x39, 0xa0, 0x57, 0x1f, 0x18, 0x88, 0x85, 0xa2, 0xfb, 0xbe, 0x66, 0xe6,
0x21, 0xb3, 0x8e, 0x64, 0x66, 0x37, 0xab, 0x55, 0xe4, 0xfc, 0x0f, 0x23, 0xf7, 0x7b, 0x07, 0xc8,
0xc9, 0x64, 0xb4, 0x4c, 0xee, 0xd5, 0x2a, 0xca, 0xc8, 0xb4, 0xb5, 0xbf, 0x2d, 0xb7, 0xbf, 0xa9,
0x65, 0xab, 0x9c, 0x7c, 0x0e, 0x35, 0x65, 0x5d, 0x07, 0xf5, 0x51, 0x49, 0x5d, 0xc1, 0x4c, 0xab,
0xe9, 0x67, 0xd0, 0xb2, 0x60, 0xec, 0x79, 0xaa, 0x57, 0xab, 0xe8, 0x68, 0x49, 0x3a, 0x71, 0x5a,
0x56, 0x5b, 0x93, 0x29, 0x81, 0x7e, 0x6e, 0x2a, 0xe2, 0xa1, 0x01, 0xa6, 0x21, 0x7c, 0xac, 0x2c,
0xbc, 0xbc, 0xe2, 0x51, 0xcc, 0xcf, 0xe2, 0xff, 0xa9, 0x68, 0x17, 0x72, 0x15, 0x40, 0x1d, 0xdf,
0x1d, 0x0e, 0xf4, 0xc1, 0x37, 0x22, 0x9d, 0x42, 0xd5, 0x43, 0xde, 0xf0, 0xb1, 0xd0, 0xd6, 0xf0,
0xb9, 0x4c, 0xb1, 0x7b, 0x67, 0x8a, 0xa5, 0xff, 0x91, 0xb8, 0x96, 0xb7, 0xa0, 0x87, 0xfe, 0x4b,
0xe1, 0xee, 0xc4, 0xd3, 0x1f, 0x42, 0xed, 0x28, 0xbc, 0x10, 0x63, 0x4e, 0xbe, 0x07, 0x75, 0x64,
0x2e, 0x72, 0xdd, 0x06, 0x9a, 0x65, 0x8d, 0x33, 0xa3, 0x91, 0x15, 0xa1, 0xfd, 0x5b, 0x45, 0x73,
0x61, 0x2b, 0x77, 0xb9, 0xc6, 0x9e, 0x43, 0x5d, 0xf3, 0xc5, 0x2a, 0xbb, 0x71, 0x88, 0x8c, 0x96,
0x3c, 0x85, 0x1a, 0x7a, 0x97, 0x07, 0x7e, 0x45, 0x04, 0x11, 0xa6, 0x15, 0xf4, 0x10, 0xbc, 0x13,
0x36, 0x94, 0x95, 0x80, 0xec, 0x0d, 0x0d, 0x2d, 0x49, 0x72, 0x3f, 0x4b, 0xf3, 0x42, 0xc7, 0x1e,
0x9f, 0x25, 0xf6, 0x2e, 0xcd, 0xd4, 0xc1, 0x6c, 0x33, 0x7c, 0xa6, 0x7f, 0x74, 0xc0, 0x7f, 0x93,
0x8e, 0x04, 0xd9, 0x00, 0x77, 0x38, 0xd0, 0x46, 0xdc, 0xe1, 0x80, 0x3c, 0x46, 0xfb, 0x3a, 0xde,
0x75, 0xb9, 0xff, 0x09, 0x1b, 0x32, 0xdc, 0xf3, 0x09, 0x34, 0x87, 0xf9, 0xbb, 0x2c, 0x1a, 0xf3,
0x6c, 0xae, 0xe7, 0x8d, 0x0a, 0xc0, 0xae, 0x54, 0xc8, 0x92, 0xf6, 0x55, 0xda, 0x51, 0x20, 0x4f,
0xa1, 0xfe, 0x05, 0x7b, 0xd7, 0x97, 0x26, 0xd7, 0x16, 0x4d, 0x1a, 0x9c, 0x7e, 0x0e, 0x1d, 0xc9,
0x04, 0xd7, 0x9b, 0xca, 0xda, 0x86, 0x9a, 0xc4, 0x4a, 0x66, 0x5a, 0xaa, 0x36, 0x71, 0xad, 0x4d,
0xe8, 0x2b, 0x65, 0xe1, 0xf0, 0x4a, 0x24, 0x85, 0x55, 0x9b, 0x28, 0xa3, 0x81, 0x36, 0x53, 0x02,
0x79, 0xa2, 0xbc, 0xd6, 0xee, 0x35, 0x24, 0x17, 0x29, 0x33, 0x44, 0xe9, 0x1c, 0xc0, 0x30, 0x99,
0xe6, 0xe5, 0x5a, 0x67, 0xd5, 0x5a, 0x42, 0x4d, 0xf9, 0xe8, 0xee, 0x03, 0x52, 0xaf, 0x10, 0x66,
0x0a, 0xeb, 0x07, 0x55, 0x61, 0xa9, 0x7c, 0x3e, 0x2a, 0xf3, 0xae, 0xf6, 0xa8, 0xca, 0xeb, 0x02,
0x5a, 0x16, 0xbe, 0xb2, 0xc6, 0x9e, 0x97, 0xc5, 0xe1, 0x56, 0xc6, 0x10, 0xd1, 0xc6, 0xb4, 0xfa,
0xee, 0x6e, 0x4c, 0x23, 0xdd, 0x52, 0xee, 0xd8, 0xa9, 0x07, 0x8f, 0x16, 0x0f, 0xbc, 0xb9, 0x65,
0x97, 0xe1, 0x7b, 0xb6, 0xfa, 0x83, 0x03, 0xed, 0x7e, 0x3c, 0xcd, 0x0b, 0x91, 0x95, 0x31, 0x6d,
0x6a, 0xa0, 0x4c, 0x6d, 0x05, 0xac, 0xce, 0x2e, 0xd9, 0x85, 0x35, 0x19, 0x71, 0x75, 0xb8, 0xed,
0x44, 0x28, 0xd8, 0xca, 0x84, 0x7f, 0x5b, 0x26, 0xe8, 0x29, 0x34, 0x0e, 0x8e, 0x86, 0x5f, 0x64,
0xe9, 0x74, 0xb2, 0xd2, 0x63, 0x33, 0xd6, 0xba, 0xd6, 0x58, 0xdb, 0x51, 0x23, 0x9a, 0xf2, 0x0a,
0xa7, 0xb2, 0x8e, 0x9a, 0xca, 0x7c, 0x8d, 0xf0, 0x19, 0x3d, 0x82, 0x4d, 0xe5, 0xae, 0xec, 0x38,
0x0f, 0x69, 0x8b, 0x66, 0x6e, 0xf2, 0xaa, 0xb9, 0x49, 0x1a, 0x55, 0x5d, 0xf7, 0xff, 0x69, 0xf4,
0x9f, 0x2e, 0x6c, 0x32, 0x91, 0x47, 0xdf, 0x88, 0x61, 0x92, 0x17, 0xd9, 0x34, 0x34, 0x17, 0xc7,
0xcf, 0xd3, 0x33, 0x9d, 0x0b, 0x8f, 0x29, 0xe1, 0xee, 0x53, 0x42, 0x28, 0xd4, 0xed, 0x26, 0x60,
0x2f, 0x30, 0x0a, 0xf2, 0x02, 0xea, 0x47, 0xe9, 0x34, 0x0b, 0xcb, 0xca, 0xc7, 0xce, 0xad, 0xf6,
0x57, 0x0a, 0x66, 0x16, 0x90, 0x2f, 0x81, 0x1c, 0x67, 0x3c, 0xc9, 0x63, 0x2e, 0x29, 0x99, 0xd7,
0x1a, 0xd5, 0x40, 0x66, 0x69, 0x17, 0x2c, 0xac, 0x78, 0x8d, 0xec, 0xd9, 0x47, 0x38, 0xa8, 0x23,
0xbf, 0x0d, 0xc3, 0x4f, 0x9f, 0x13, 0xfb, 0x90, 0x7f, 0xba, 0x54, 0xa1, 0x41, 0x0d, 0x5f, 0xd9,
0xc4, 0xcb, 0xdc, 0x56, 0xb0, 0xc5, 0x75, 0xf4, 0x77, 0x0e, 0xac, 0xdb, 0x6c, 0xee, 0x69, 0x17,
0x65, 0xfa, 0xdc, 0xfb, 0xe7, 0x3b, 0x93, 0x3e, 0x7f, 0xd5, 0x2c, 0xbd, 0x66, 0xcf, 0x7c, 0x29,
0x7c, 0xe7, 0x96, 0xe0, 0x3c, 0x88, 0x4e, 0x17, 0x5a, 0xef, 0x78, 0x56, 0x44, 0xd2, 0x98, 0xbe,
0xa7, 0xd7, 0x98, 0x0d, 0x51, 0x01, 0x8f, 0x6f, 0x14, 0x51, 0x3f, 0x1d, 0x4f, 0x64, 0xb5, 0x3e,
0xa8, 0x98, 0x64, 0x9b, 0xce, 0xb2, 0x34, 0x33, 0x11, 0x40, 0x81, 0x1e, 0x40, 0xe3, 0x38, 0x9d,
0xa4, 0x71, 0x7a, 0x3e, 0xbf, 0xa7, 0x65, 0x04, 0x50, 0x57, 0x57, 0x83, 0x6a, 0x51, 0x4d, 0x66,
0x44, 0xfa, 0x91, 0xac, 0xf7, 0x90, 0xc7, 0xe1, 0x34, 0xe6, 0x85, 0xc0, 0x2f, 0x02, 0x04, 0xbf,
0x4a, 0xf9, 0x48, 0x75, 0x05, 0x7d, 0xb4, 0xe8, 0x2f, 0x75, 0x01, 0x72, 0x74, 0xc7, 0xba, 0x82,
0x5e, 0x86, 0xf6, 0xac, 0xa5, 0x24, 0xf2, 0x63, 0x68, 0x59, 0xab, 0xed, 0x01, 0xce, 0x82, 0x99,
0xbd, 0x86, 0xfe, 0xcd, 0x59, 0x78, 0xe7, 0xc6, 0x9d, 0xab, 0xb7, 0xba, 0x52, 0x41, 0x6a, 0x30,
0x2d, 0x49, 0xd7, 0x0f, 0x67, 0x61, 0x3c, 0xcd, 0xa5, 0x4a, 0x5f, 0xb8, 0x25, 0x20, 0x5d, 0x97,
0x1f, 0x7d, 0xe9, 0xd4, 0x0c, 0x37, 0x46, 0x94, 0x9f, 0x87, 0x03, 0xc1, 0x47, 0x71, 0x94, 0x08,
0xac, 0x17, 0x8f, 0x95, 0x32, 0x79, 0xa1, 0x7a, 0xac, 0x29, 0xf4, 0xad, 0x25, 0xe2, 0xa8, 0x53,
0x9d, 0x37, 0xa7, 0x04, 0x3a, 0xcb, 0x2a, 0xba, 0x05, 0x44, 0x55, 0xc0, 0xcb, 0xb3, 0x34, 0x33,
0xb7, 0x2d, 0xed, 0x9b, 0xe6, 0x22, 0xa3, 0x7f, 0xdf, 0x25, 0x5e, 0x45, 0xd6, 0xb5, 0x23, 0x4b,
0x7f, 0x01, 0x1b, 0x7a, 0xb6, 0x13, 0x19, 0x16, 0xb4, 0x0c, 0x00, 0x13, 0x61, 0x2a, 0xc7, 0x44,
0xf3, 0x1d, 0x57, 0x01, 0xd2, 0x0e, 0x0e, 0xba, 0xe6, 0x76, 0xd2, 0x12, 0xce, 0x46, 0xd1, 0x79,
0x22, 0x46, 0x78, 0x63, 0x78, 0x4c, 0x4b, 0xf4, 0x4f, 0x2e, 0x6c, 0xa9, 0xa1, 0x33, 0x39, 0x17,
0x79, 0x51, 0x6d, 0x83, 0x63, 0x35, 0xf6, 0xff, 0x72, 0xac, 0xc6, 0x1b, 0xe0, 0x19, 0x6c, 0xf4,
0x63, 0xc1, 0xb3, 0x8a, 0x83, 0xda, 0x68, 0x09, 0x95, 0xe7, 0x06, 0x11, 0x7d, 0x3d, 0xab, 0x21,
0xd4, 0x86, 0xc8, 0x01, 0x34, 0xb4, 0x6b, 0xa6, 0x21, 0x3e, 0xc3, 0x5b, 0x6a, 0x05, 0x1b, 0x33,
0xdf, 0xe6, 0xfa, 0xab, 0xd3, 0x88, 0x3b, 0x6f, 0xa1, 0xbd, 0xa0, 0x5a, 0xf1, 0xd5, 0xd9, 0xb3,
0xbf, 0x3a, 0x5b, 0xfb, 0xc4, 0x1a, 0x97, 0xb5, 0x75, 0xfb, 0x4b, 0xb4, 0x0f, 0xdf, 0x5e, 0x45,
0x20, 0x27, 0x2f, 0xc0, 0x93, 0x44, 0xd5, 0x30, 0x1c, 0xdc, 0x46, 0x94, 0xc9, 0x45, 0xf4, 0xaf,
0x8e, 0x0e, 0xaa, 0xd0, 0x7a, 0xf3, 0xf7, 0xe0, 0x13, 0xdb, 0xc8, 0xd3, 0xd2, 0xc8, 0xd2, 0xb2,
0xbd, 0xd2, 0x51, 0xb9, 0x7a, 0xe7, 0x6b, 0x68, 0xac, 0x72, 0xcf, 0x57, 0xee, 0xfd, 0x68, 0xd1,
0xbd, 0xc7, 0xb7, 0x31, 0xcb, 0x6d, 0x2f, 0xf7, 0x60, 0x5b, 0xdd, 0xa6, 0x03, 0x5e, 0xf0, 0x5f,
0x65, 0x7c, 0x2c, 0xee, 0xbc, 0x52, 0x0f, 0x3a, 0xff, 0x78, 0xbf, 0xeb, 0xfc, 0xeb, 0xfd, 0xae,
0xf3, 0xef, 0xf7, 0xbb, 0xce, 0x9f, 0xff, 0xb3, 0xfb, 0xad, 0xb3, 0x1a, 0xfe, 0x76, 0xfb, 0xe4,
0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xa4, 0xc1, 0x04, 0xc8, 0x99, 0x13, 0x00, 0x00,
}
func (m *IndexMeta) Marshal() (dAtA []byte, err error) {
@ -5458,6 +5507,40 @@ func (m *ShardedIngestRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil
}
func (m *DeleteDataframeMessage) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *DeleteDataframeMessage) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *DeleteDataframeMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.XXX_unrecognized != nil {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if len(m.Index) > 0 {
i -= len(m.Index)
copy(dAtA[i:], m.Index)
i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index)))
i--
dAtA[i] = 0xa
}
return len(dAtA) - i, nil
}
func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int {
offset -= sovPrivate(v)
base := offset
@ -6580,6 +6663,22 @@ func (m *ShardedIngestRequest) Size() (n int) {
return n
}
func (m *DeleteDataframeMessage) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.Index)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func sovPrivate(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7
}
@ -13468,6 +13567,89 @@ func (m *ShardedIngestRequest) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *DeleteDataframeMessage) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: DeleteDataframeMessage: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: DeleteDataframeMessage: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthPrivate
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Index = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipPrivate(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0

4
pb/private.proto
View file

@ -270,3 +270,7 @@ message ShardIngestOperations {
message ShardedIngestRequest {
map <uint64, ShardIngestOperations> Ops = 1;
}
message DeleteDataframeMessage {
string Index = 1;
}

859
pb/public.pb.go
View file

File diff suppressed because it is too large Load diff

10
pb/public.proto
View file

@ -164,6 +164,8 @@ message QueryResult {
RowMatrix RowMatrix = 15;
GroupCounts GroupCounts = 16;
DistinctTimestamp DistinctTimestamp = 17;
DataFrame DataFrame = 18;
ArrowTable ArrowTable = 19;
}
message ImportRequest {
@ -260,3 +262,11 @@ message GroupCounts{
string Aggregate = 1;
repeated GroupCount Groups = 2;
}
message DataFrame{
bytes Data =1;
}
message ArrowTable{
bytes Data =1;
}

39
pql/ast.go
View file

@ -107,6 +107,13 @@ func (q *Query) startConditional() {
}
}
func (q *Query) condAddTimestamp(val string) {
// TODO(twg) 2022/09/14 reviist this ugly hack
tsv := parseTimestamp2(val)
secsinstring := fmt.Sprintf("%d", tsv.Unix())
q.condAdd(secsinstring)
}
func (q *Query) condAdd(val string) {
q.conditional = append(q.conditional, val)
}
@ -281,21 +288,27 @@ func (q *Query) endList() {
func (q *Query) addGT() {
q.lastCallStackElem().lastCond = GT
}
func (q *Query) addLT() {
q.lastCallStackElem().lastCond = LT
}
func (q *Query) addGTE() {
q.lastCallStackElem().lastCond = GTE
}
func (q *Query) addLTE() {
q.lastCallStackElem().lastCond = LTE
}
func (q *Query) addEQ() {
q.lastCallStackElem().lastCond = EQ
}
func (q *Query) addNEQ() {
q.lastCallStackElem().lastCond = NEQ
}
func (q *Query) addBTWN() {
q.lastCallStackElem().lastCond = BETWEEN
}
@ -497,7 +510,8 @@ var callInfoByFunc = map[string]callInfo{
"rows": nil,
},
},
"Shift": {allowUnknown: false,
"Shift": {
allowUnknown: false,
prototypes: map[string]interface{}{
"n": int64(0),
},
@ -615,6 +629,19 @@ var callInfoByFunc = map[string]callInfo{
"sort-desc": false,
},
},
"Apply": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_ivy": stringOrVariable,
"_ivyReduce": stringOrVariable,
},
},
"Arrow": {
allowUnknown: false,
prototypes: map[string]interface{}{
"header": interfaceOrVariable,
},
},
}
// We want to allow case-insensitive names, but we want to continue using
@ -1074,7 +1101,6 @@ func (c *Call) ExpandVars(vars map[string]interface{}) ([]*Call, error) {
default:
return nil, fmt.Errorf("variable: requires single value for argument, got: %+v", newArg)
}
}
}
@ -1378,6 +1404,15 @@ func parseNum(val string) interface{} {
return ival
}
func parseTimestamp2(val string) time.Time {
val = strings.Replace(val, "\"", "", -1)
tsval, err := time.Parse("2006-01-02T15:04:05Z", val)
if err != nil {
panic(fmt.Sprintf("%s: %s", invalidTimestampError, err))
}
return tsval
}
func parseTimestamp(val string) time.Time {
tsval, err := time.Parse(time.RFC3339Nano, val)
if err != nil {

68
pql/parser_test.go
View file

@ -3,6 +3,7 @@
package pql_test
import (
"fmt"
"reflect"
"strings"
"testing"
@ -224,6 +225,14 @@ func TestParser_Parse(t *testing.T) {
} else if !strings.Contains(err.Error(), "not a valid timestamp") {
t.Fatalf("expected error for invalid timestamp, got: %s", err.Error())
}
// str := `Row('2024-04-24T23:24:24Z' < x < '2024-04-24T24:24:24Z')`
str := `Row('2024-04-24T23:24:24Z' < x < '2024-04-24T24:24:24Z')`
_, err = pql.ParseString(str)
if err == nil {
t.Fatal("no error parsing invalid date")
} else if !strings.Contains(err.Error(), "not a valid timestamp") {
t.Fatalf("expected error for invalid timestamp, got: %s on string '%v'", err.Error(), str)
}
})
t.Run("VariousSpaces", func(t *testing.T) {
@ -239,6 +248,65 @@ func TestParser_Parse(t *testing.T) {
t.Fatalf("unexpected call: %#v", q.Calls[0])
}
})
t.Run("Apply", func(t *testing.T) {
twos := "2022-02-22T22:22:22Z"
date, _ := time.Parse(time.RFC3339, twos)
defaultFilter := `Row(x>'2022-02-22T22:22:22Z')`
// ivy := `(+/++A+B*B`
// ivy := `(+/ sqrt(A*A + B*B)/rho x 1-2 ? || )`
// ivy := `text*here`
defRes := []*pql.Call{
{
Name: "Row",
Args: map[string]interface{}{
"x": &pql.Condition{Op: pql.GT, Value: date},
},
},
}
programs := []struct {
prog string
description string
filter string
children []*pql.Call
}{
{"(+/ sqrt(x*x + y*y))/rho x )", "average distance from x to y", defaultFilter, defRes},
{"rho sample", "size of input values", defaultFilter, defRes},
{"(1 drop sample)", " first element removed", defaultFilter, defRes},
{"(-1 drop sample)", " last element removed", defaultFilter, defRes},
{"(1 drop sample) - (-1 drop sample)", "subtract those", defaultFilter, defRes},
{"_ > 0", "which ones of the last difference are greater than zero", defaultFilter, defRes},
{"+/ _", "sum list from previous calc", defaultFilter, defRes},
{"(1 drop sample) > (-1 drop sample)", "compare array", defaultFilter, defRes},
{"1 2 3 +.* 2 3 4", "inner product with + then * and equals 20", defaultFilter, defRes},
{"(1 drop sample) +.> (-1 drop sample)", "inner product with + then > ", defaultFilter, defRes},
{"op inc x = (1 drop sample) +.> (-1 drop sample)", "function of previous ", defaultFilter, defRes},
{"op sum3 x = (-2 drop x) +.> (-1 drop 1 drop x) + 2 drop x", "function of previous ", defaultFilter, defRes},
{"op grid3 x = 3 ((rho x)-2) rho (-2 drop x), (-1 drop 1 drop x), 2 drop x", "make 3x2 grid from array", defaultFilter, defRes},
{"x[ up x ]", "sort the array", defaultFilter, defRes},
{"x[ up x ]", "no filter", "", nil},
}
for i := range programs {
ivy := programs[i].prog
filter := programs[i].filter
children := programs[i].children
qstring := fmt.Sprintf(`Apply(%s,"%s")`, filter, ivy)
if filter == "" {
qstring = fmt.Sprintf(`Apply("%s")`, ivy)
}
q, err := pql.ParseString(qstring)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "Apply",
Children: children,
Args: map[string]interface{}{"_ivy": ivy},
},
) {
t.Fatalf("unexpected call: %#v", q.Calls[0])
}
}
})
}
func TestUnquote(t *testing.T) {

11
pql/pql.peg
View file

@ -8,6 +8,7 @@ type PQL Peg {
Calls <- sp (Call sp)* !.
Call <- "Set" {p.startCall("Set")} open col comma args (comma time)? close {p.endCall()}
/ "Clear" {p.startCall("Clear")} open col comma args close {p.endCall()}
/ "Apply" {p.startCall("Apply")} open (Call comma)? ivyprogram (comma ivyprogram2)? close {p.endCall()}
/ "ClearRow" {p.startCall("ClearRow")} open arg close {p.endCall()}
/ "Store" {p.startCall("Store")} open Call comma arg close {p.endCall()}
/ "TopN" {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
@ -19,6 +20,10 @@ Call <- "Set" {p.startCall("Set")} open col comma args (comma time)? close {p.e
/ "Sum" {p.startCall("Sum")} open posfield (comma allargs)? close {p.endCall()}
/ "Range" {p.startCall("Range")} open field eq value comma 'from='? {p.addField("from")} timefmt {p.addVal(text)} comma 'to='? sp {p.addField("to")} timefmt {p.addVal(text)} close {p.endCall()}
/ < IDENT > { p.startCall(text) } open allargs comma? close { p.endCall() }
ivyExpr <- ( '(' / '_' / '/' / '[' / ']' /'.' / '=' / '&' / '<'/ '>' /',' / ')' / '^' / '!' / '|' / [*+] / '-' / '?' / [[A-Z]] / [0-9] / '#' / [ \t\n] )*
ivyprogram <- '"'? <ivyExpr> '"' { p.addPosStr("_ivy", text) }
ivyprogram2 <- '"'? <ivyExpr> '"' { p.addPosStr("_ivyReduce", text) }
allargs <- Call (comma Call)* (comma args)? / args / sp
args <- arg (comma args)? sp
arg <- field eq value
@ -32,8 +37,10 @@ COND <- '><' { p.addBTWN() }
/ '<' { p.addLT() }
/ '>' { p.addGT() }
conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()}
condint <- < decimal > sp {p.condAdd(text)}
conditional <- {p.startConditional()} condintOrTime condLT condfield condLT condintOrTime {p.endConditional()}
condintOrTime <- condint / timefmtS
timefmtS <- <timestampfmt> sp { p.condAddTimestamp(text) }
condint <- < decimal > sp {p.condAdd(text)}
condLT <- <('<=' / '<')> sp {p.condAdd(text)}
condfield <- <fieldExpr> sp {p.condAdd(text)}

5228
pql/pql.peg.go
View file

File diff suppressed because it is too large Load diff

37
row.go
View file

@ -66,7 +66,7 @@ func (r *Row) Clone() (clone *Row) {
if seg.data != nil {
segClone.data = seg.data.Clone() // *roaring.Bitmap
}
//segClone.InvalidateCount() // not needed?
// segClone.InvalidateCount() // not needed?
clone.Segments = append(clone.Segments, segClone)
}
return clone
@ -144,10 +144,11 @@ func (r *Row) ToRows(callback func(*pb.RowResponse) error) error {
Headers: ci,
Columns: []*pb.ColumnResponse{
{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: x}},
}}); err != nil {
},
}); err != nil {
return errors.Wrap(err, "calling callback")
}
ci = nil //only send on the first
ci = nil // only send on the first
}
} else {
// Column IDs
@ -159,10 +160,11 @@ func (r *Row) ToRows(callback func(*pb.RowResponse) error) error {
Headers: ci,
Columns: []*pb.ColumnResponse{
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: x}},
}}); err != nil {
},
}); err != nil {
return errors.Wrap(err, "calling callback")
}
ci = nil //only send on the first
ci = nil // only send on the first
}
}
return nil
@ -186,7 +188,6 @@ func (r *Row) IsEmpty() bool {
if r.Segments[i].n > 0 {
return false
}
}
return true
}
@ -480,6 +481,19 @@ func (r *Row) Columns() []uint64 {
return a
}
func (r *Row) ShardColumns() []int64 {
// We occasionally hit cases where we want to call Columns on something
// that might not exist, but a nil slice would be fine.
if r == nil {
return nil
}
a := make([]int64, 0, r.Count())
for i := range r.Segments {
a = append(a, r.Segments[i].ShardColumns()...)
}
return a
}
// Includes returns true if the row contains the given column.
func (r *Row) Includes(col uint64) bool {
shard := col / ShardWidth
@ -647,6 +661,17 @@ func (s *RowSegment) Columns() []uint64 {
return a
}
// Columns returns a list of all columns set in the segment, normalized from 0-shardwidth-1
func (s *RowSegment) ShardColumns() []int64 {
a := make([]int64, 0, s.Count())
itr := s.data.Iterator()
mask := uint64(ShardWidth - 1)
for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
a = append(a, int64(mask&v))
}
return a
}
// Count returns the number of set columns in the row.
func (s *RowSegment) Count() uint64 { return s.n }

19
server.go
View file

@ -98,6 +98,8 @@ type Server struct { // nolint: maligned
writeLogReader computer.WriteLogReader
writeLogWriter computer.WriteLogWriter
snapshotReadWriter computer.SnapshotReadWriter
dataframeEnabled bool
}
type ExecutionPlannerFn func(executor Executor, api *API, sql string) sql3.CompilePlanner
@ -399,8 +401,8 @@ func OptServerMaxQueryMemory(v int64) ServerOption {
func OptServerDisCo(disCo disco.DisCo,
noder disco.Noder,
sharder disco.Sharder,
schemator disco.Schemator) ServerOption {
schemator disco.Schemator,
) ServerOption {
return func(s *Server) error {
s.disCo = disCo
s.noder = noder
@ -467,6 +469,14 @@ func OptServerIsComputeNode(is bool) ServerOption {
}
}
// OptServerIsDataframeEnabled specifies if experimental dataframe support available
func OptServerIsDataframeEnabled(is bool) ServerOption {
return func(s *Server) error {
s.dataframeEnabled = is
return nil
}
}
// NewServer returns a new instance of Server.
func NewServer(opts ...ServerOption) (*Server, error) {
cluster := newCluster()
@ -537,6 +547,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
executorOpts = append(executorOpts, optExecutorWorkerPoolSize(s.executorPoolSize))
}
s.executor = newExecutor(executorOpts...)
s.executor.dataframeEnabled = s.dataframeEnabled
path, err := expandDirName(s.dataDir)
if err != nil {
@ -1030,6 +1041,10 @@ func (s *Server) receiveMessage(m Message) error {
if err != nil {
return errors.Wrapf(err, "handling transaction message: %v", obj)
}
case *DeleteDataframeMessage:
if err := s.holder.DeleteDataframe(obj.Index); err != nil {
return err
}
}
return nil

6
server/config.go
View file

@ -77,7 +77,7 @@ type Config struct {
BindGRPC string `toml:"bind-grpc"`
// Listener is an already-bound listener to use for http.
//Listener *net.TCPListener
// Listener *net.TCPListener
Listener net.Listener
// GRPCListener is an already-bound listener to use for gRPC.
@ -250,6 +250,10 @@ type Config struct {
} `toml:"datadog"`
Auth Auth
Dataframe struct {
Enable bool `toml:"enable"`
} `toml:"dataframe"`
}
type Auth struct {

7
server/grpc.go
View file

@ -626,7 +626,8 @@ func (r ResultUint64) ToRows(callback func(*pb.RowResponse) error) error {
Headers: []*pb.ColumnInfo{{Name: "count", Datatype: "uint64"}},
Columns: []*pb.ColumnResponse{
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(r)}},
}})
},
})
}
// ResultBool is a wrapper around a bool result type
@ -645,7 +646,8 @@ func (r ResultBool) ToRows(callback func(*pb.RowResponse) error) error {
Headers: []*pb.ColumnInfo{{Name: "result", Datatype: "bool"}},
Columns: []*pb.ColumnResponse{
{ColumnVal: &pb.ColumnResponse_BoolVal{BoolVal: bool(r)}},
}})
},
})
}
// Normally we wouldn't need this wrapper, but since pilosa returns
@ -750,7 +752,6 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
fields = append(fields, field)
break
}
}
} else {
fields = append(fields, field)

2
server/grpc_internal_test.go
View file

@ -88,7 +88,7 @@ func TestGetTokensFromMetadata(t *testing.T) {
),
NewServerTransportStream(),
)
err := a.SetGRPCMetadata(ctx, test.md, test.access, test.refresh)
_, err := a.SetGRPCMetadata(ctx, test.md, test.access, test.refresh)
if err != nil {
t.Errorf("unexpected error setting GRPC metadata: %v", err)
}

1
server/server.go
View file

@ -607,6 +607,7 @@ func (m *Command) setupServer() error {
pilosa.OptServerWriteLogReader(wlr),
pilosa.OptServerWriteLogWriter(wlw),
pilosa.OptServerSnapshotReadWriter(snap),
pilosa.OptServerIsDataframeEnabled(m.Config.Dataframe.Enable),
}
if m.Config.LookupDBDSN != "" {