diff --git a/Makefile b/Makefile index 4ca3b029e..5794843ce 100644 --- a/Makefile +++ b/Makefile @@ -234,6 +234,18 @@ bg-rbf: @echo " log.bg-rbf green: \c"; cat log.bg-rbf | grep PASS |wc -l @echo " log.bg-rbf red: \c"; cat log.bg-rbf | grep '\-\-\- FAIL' |wc -l +rbf-lm: + mv log.rbf-lm log.rbf-lm.prev || true + PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-lm + @echo " log.rbf-lm green: \c"; cat log.rbf-lm | grep PASS |wc -l + @echo " log.rbf-lm red: \c"; cat log.rbf-lm | grep '\-\-\- FAIL' |wc -l + +lm-rbf: + mv log.lm-rbf log.lm-rbf.prev || true + PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.lm-rbf + @echo " log.lm-rbf green: \c"; cat log.lm-rbf | grep PASS |wc -l + @echo " log.lm-rbf red: \c"; cat log.lm-rbf | grep '\-\-\- FAIL' |wc -l + lm-rr: mv log.lm-rr log.lm-rr.prev || true PILOSA_TXSRC=lmdb_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.lm-rr diff --git a/api.go b/api.go index 287dbc66c..d8cdef042 100644 --- a/api.go +++ b/api.go @@ -1022,6 +1022,9 @@ type ImportOptions struct { Clear bool IgnoreKeyCheck bool Presorted bool + + // test Tx atomicity if > 0 + SimPowerLossAfter int } // ImportOption is a functional option type for API.Import. @@ -1052,8 +1055,71 @@ func OptImportOptionsPresorted(b bool) ImportOption { } } -// Import bulk imports data into a particular index,field,shard. +var ErrAborted = fmt.Errorf("error: update was aborted") + +func (api *API) ImportAtomicRecord(ctx context.Context, req *AtomicRecord, opts ...ImportOption) error { + simPowerLoss := false + lossAfter := -1 + var opt ImportOptions + for _, setter := range opts { + if setter != nil { + err := setter(&opt) + if err != nil { + return errors.Wrap(err, "ImportAtomicRecord ImportOptions") + } + } + } + if opt.SimPowerLossAfter > 0 { + simPowerLoss = true + lossAfter = opt.SimPowerLossAfter + } + + idx, err := api.Index(ctx, req.Index) + if err != nil { + return errors.Wrap(err, "getting index") + } + + // the whole point is to run this part of the import atomically. + // So make a Tx. + tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx}) + defer tx.Rollback() + + tot := 0 + + // BSIs (Values) + for _, ivr := range req.Ivr { + tot++ + if simPowerLoss && tot > lossAfter { + return ErrAborted + } + opts0 := append(opts, OptImportOptionsClear(ivr.Clear)) + err := api.ImportValueWithTx(ctx, tx, ivr, opts0...) + if err != nil { + return errors.Wrap(err, "ImportAtomicRecord ImportValueWithTx") + } + } + + // other bits, non-BSI + for _, ir := range req.Ir { + tot++ + if simPowerLoss && tot > lossAfter { + return ErrAborted + } + opts0 := append(opts, OptImportOptionsClear(ir.Clear)) + err := api.ImportWithTx(ctx, tx, ir, opts0...) + if err != nil { + return errors.Wrap(err, "ImportAtomicRecord ImportWithTx") + } + } + return tx.Commit() +} + func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOption) error { + return api.ImportWithTx(ctx, nil, req, opts...) +} + +// Import bulk imports data into a particular index,field,shard. +func (api *API) ImportWithTx(ctx context.Context, tx Tx, req *ImportRequest, opts ...ImportOption) error { span, _ := tracing.StartSpanFromContext(ctx, "API.Import") defer span.Finish() @@ -1156,39 +1222,43 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp timestamps[i] = &t } + isLocalTx := false + if tx == nil { + isLocalTx = true + tx = index.Txf.NewTx(Txo{Write: true, Index: index}) + defer tx.Rollback() + } + // Import columnIDs into existence field. if !options.Clear { - if err := func() error { - tx := index.Txf.NewTx(Txo{Write: true, Index: index}) - defer tx.Rollback() - - if err := importExistenceColumns(tx, index, req.ColumnIDs); err != nil { - api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) - return err - } - return tx.Commit() - }(); err != nil { + if err := importExistenceColumns(tx, index, req.ColumnIDs); err != nil { + api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + return err + } + if err != nil { return errors.Wrap(err, "importing existence columns") } } - tx := index.Txf.NewTx(Txo{Write: true, Index: index}) - defer tx.Rollback() - // Import into fragment. err = field.Import(tx, req.RowIDs, req.ColumnIDs, timestamps, opts...) if err != nil { api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) - } else { - err = tx.Commit() + return errors.Wrap(err, "importing") } - return errors.Wrap(err, "importing") + if isLocalTx { + err = tx.Commit() + } + return errors.Wrap(err, "committing") +} +func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts ...ImportOption) error { + return api.ImportValueWithTx(ctx, nil, req, opts...) } // ImportValue bulk imports values into a particular field. -func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts ...ImportOption) error { +func (api *API) ImportValueWithTx(ctx context.Context, tx Tx, req *ImportValueRequest, opts ...ImportOption) error { span, _ := tracing.StartSpanFromContext(ctx, "API.ImportValue") defer span.Finish() @@ -1198,7 +1268,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . index, field, err := api.indexField(req.Index, req.Field, req.Shard) if err != nil { - return errors.Wrap(err, "getting index and field") + return errors.Wrap(err, fmt.Sprintf("getting index '%v' and field '%v'; shard=%v", req.Index, req.Field, req.Shard)) } if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { @@ -1258,11 +1328,15 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . sort.Sort(req) } + isLocalTx := false // if we're importing into a specific shard if req.Shard != math.MaxUint64 { // Obtain transaction. - tx := index.Txf.NewTx(Txo{Write: true, Index: index}) - defer tx.Rollback() + if tx == nil { + isLocalTx = true + tx = index.Txf.NewTx(Txo{Write: true, Index: index}) + defer tx.Rollback() + } // Check that column IDs match the stated shard. if s1, s2 := req.ColumnIDs[0]/ShardWidth, req.ColumnIDs[len(req.ColumnIDs)-1]/ShardWidth; s1 != s2 && s2 != req.Shard { @@ -1293,7 +1367,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) } } - if err == nil { + if err == nil && isLocalTx { err = tx.Commit() } return errors.Wrap(err, "importing value") diff --git a/badger_test.go b/badger_test.go index 6c271bb0d..c1627f1db 100644 --- a/badger_test.go +++ b/badger_test.go @@ -1275,6 +1275,7 @@ func getTestBitmapAsRawRoaring(bitsToSet ...uint64) []byte { return buf.Bytes() } +/* func TestBadger_AutoCommit(t *testing.T) { // setup @@ -1330,6 +1331,7 @@ func TestBadger_BigWritesAvoidTxnTooLargeWithAutoCommit(t *testing.T) { err := tx.Commit() panicOn(err) } +*/ func TestBadger_DeleteIndex(t *testing.T) { @@ -1394,7 +1396,7 @@ func TestBadger_DeleteIndex(t *testing.T) { } func TestBadger_DeleteIndex_over100k(t *testing.T) { - + t.Skip("test big and long running, skip") // setup dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex_over100k") defer clean() diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 29cb2ecf7..a1453cd62 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -304,6 +304,14 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeTransactionMessage(msg, mt) return nil + case *pilosa.AtomicRecord: + msg := &internal.AtomicRecord{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling AtomicRecord") + } + s.decodeAtomicRecord(msg, mt) + return nil default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -375,6 +383,8 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message { return s.encodeTranslateIDsResponse(mt) case *pilosa.TransactionMessage: return s.encodeTransactionMessage(mt) + case *pilosa.AtomicRecord: + return s.encodeAtomicRecord(mt) } return nil } @@ -413,6 +423,7 @@ func (s Serializer) encodeImportRequest(m *pilosa.ImportRequest) *internal.Impor RowKeys: m.RowKeys, ColumnKeys: m.ColumnKeys, Timestamps: m.Timestamps, + Clear: m.Clear, } } @@ -428,6 +439,7 @@ func (s Serializer) encodeImportValueRequest(m *pilosa.ImportValueRequest) *inte Values: m.Values, FloatValues: m.FloatValues, StringValues: m.StringValues, + Clear: m.Clear, } } @@ -873,6 +885,20 @@ func (s Serializer) encodeTransactionMessage(msg *pilosa.TransactionMessage) *in } } +func (s Serializer) encodeAtomicRecord(msg *pilosa.AtomicRecord) *internal.AtomicRecord { + ar := &internal.AtomicRecord{ + Index: msg.Index, + Shard: msg.Shard, + } + for _, ivr := range msg.Ivr { + ar.Ivr = append(ar.Ivr, s.encodeImportValueRequest(ivr)) + } + for _, ir := range msg.Ir { + ar.Ir = append(ar.Ir, s.encodeImportRequest(ir)) + } + return ar +} + func (s Serializer) encodeTransaction(trns *pilosa.Transaction) *internal.Transaction { if trns == nil { return nil @@ -1179,6 +1205,7 @@ func (s Serializer) decodeImportRequest(pb *internal.ImportRequest, m *pilosa.Im m.Timestamps = pb.Timestamps m.IndexCreatedAt = pb.IndexCreatedAt m.FieldCreatedAt = pb.FieldCreatedAt + m.Clear = pb.Clear } func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportValueRequest) { @@ -1192,6 +1219,7 @@ func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m m.StringValues = pb.StringValues m.IndexCreatedAt = pb.IndexCreatedAt m.FieldCreatedAt = pb.FieldCreatedAt + m.Clear = pb.Clear } func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) { @@ -1295,6 +1323,22 @@ func decodeTransactionMessage(pb *internal.TransactionMessage, m *pilosa.Transac decodeTransaction(pb.Transaction, m.Transaction) } +func (s Serializer) decodeAtomicRecord(pb *internal.AtomicRecord, m *pilosa.AtomicRecord) { + m.Index = pb.Index + m.Shard = pb.Shard + m.Ivr = make([]*pilosa.ImportValueRequest, len(pb.Ivr)) + m.Ir = make([]*pilosa.ImportRequest, len(pb.Ir)) + + for i, ivr := range pb.Ivr { + m.Ivr[i] = &pilosa.ImportValueRequest{} + s.decodeImportValueRequest(ivr, m.Ivr[i]) + } + for i, ir := range pb.Ir { + m.Ir[i] = &pilosa.ImportRequest{} + s.decodeImportRequest(ir, m.Ir[i]) + } +} + func decodeTransaction(pb *internal.Transaction, trns *pilosa.Transaction) { trns.ID = pb.ID trns.Active = pb.Active diff --git a/fragment.go b/fragment.go index 523aac338..09a4bde3e 100644 --- a/fragment.go +++ b/fragment.go @@ -622,7 +622,7 @@ func (f *fragment) rowFromStorage(tx Tx, rowID uint64) (*Row, error) { // setBit sets a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. func (f *fragment) setBit(tx Tx, rowID, columnID uint64) (changed bool, err error) { - f.mu.Lock() + f.mu.Lock() // controls access to the file. defer f.mu.Unlock() var wp *io.Writer if f.storage != nil { diff --git a/fragment_internal_test.go b/fragment_internal_test.go index a41cee1a9..8ef6d5909 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -5323,19 +5323,11 @@ func check(t *testing.T, tx Tx, f *fragment, exp map[uint64]map[uint64]struct{}) func TestImportValueConcurrent(t *testing.T) { f, idx := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) - types := idx.Txf.TxTypes() - for _, ty := range types { - switch ty { - case roaringTxn: - t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " + - "blueGreenTx because the lack of transactional consistency " + - "from Roaring-per-file will create false comparison " + - "failures.")) - case lmdbTxn: - t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " + - "lmdb since only a single writer is allowed at once.")) - } - } + + // produces false positives under blue_green because of races + // between commits, and is probematic under single writer + // backends like rbf and lmdb. Marking as roaring-only. + roaringOnlyTest(t) // Since eg.Go gets called multiple times below, each // time needs its own Tx. So close the default one and diff --git a/handler.go b/handler.go index bb5aa245d..26e1bd186 100644 --- a/handler.go +++ b/handler.go @@ -113,6 +113,7 @@ var NopHandler Handler = nopHandler{} // ImportValueRequest describes the import request structure // for a value (BSI) import. +// Note: no RowIDs here. have to convert BSI Values into RowIDs internally. type ImportValueRequest struct { Index string IndexCreatedAt int64 @@ -121,11 +122,25 @@ type ImportValueRequest struct { // if Shard is MaxUint64 (an impossible shard value), this // indicates that the column IDs may come from multiple shards. Shard uint64 - ColumnIDs []uint64 + ColumnIDs []uint64 // e.g. weather stationID ColumnKeys []string - Values []int64 + Values []int64 // e.g. temperature, humidity, barometric pressure FloatValues []float64 StringValues []string + Clear bool // only works for ImportAtomicRecord() at the moment. +} + +// AtomicRecord applies all its Ivr and Ivr atomically, in a Tx. +// The top level Shard has to agree with Ivr[i].Shard and the Iv[i].Shard +// for all i included (in Ivr and Ir). The same goes for the top level Index: all records +// have to be writes to the same Index. These requirements are checked. +// +type AtomicRecord struct { + Index string + Shard uint64 + + Ivr []*ImportValueRequest // BSI values + Ir []*ImportRequest // other field types, e.g. single bit } func (ivr *ImportValueRequest) Len() int { return len(ivr.ColumnIDs) } @@ -176,7 +191,7 @@ func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreate } // ImportColumnAttrsRequest describes the import request structure -// for a ColumnAttr import +// for a ColumnAttr import. type ImportColumnAttrsRequest struct { AttrKey string ColumnIDs []uint64 @@ -187,7 +202,7 @@ type ImportColumnAttrsRequest struct { } // ImportRequest describes the import request structure -// for an import. +// for an import. BSIs use the ImportValueRequest instead. type ImportRequest struct { Index string IndexCreatedAt int64 @@ -199,6 +214,7 @@ type ImportRequest struct { RowKeys []string ColumnKeys []string Timestamps []int64 + Clear bool // only works for ImportAtomicRecord() at the moment. } // ValidateWithTimestamp ensures that the payload of the request is valid. diff --git a/http/handler.go b/http/handler.go index 890a4a31d..f0514bda1 100644 --- a/http/handler.go +++ b/http/handler.go @@ -204,6 +204,7 @@ func (h *Handler) populateValidators() { h.validators["PostField"] = queryValidationSpecRequired() h.validators["DeleteField"] = queryValidationSpecRequired() h.validators["PostImport"] = queryValidationSpecRequired().Optional("clear", "ignoreKeyCheck") + h.validators["PostImportAtomicRecord"] = queryValidationSpecRequired().Optional("simPowerLossAfter") h.validators["PostImportRoaring"] = queryValidationSpecRequired().Optional("remote", "clear") h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns", "profile") h.validators["GetInfo"] = queryValidationSpecRequired() @@ -343,6 +344,7 @@ func newRouter(handler *Handler) *mux.Router { router.Handle("/debug/vars", expvar.Handler()).Methods("GET") router.Handle("/metrics", promhttp.Handler()) router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") + router.HandleFunc("/import-atomic-record", handler.handlePostImportAtomicRecord).Methods("POST").Name("PostImportAtomicRecord") router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET").Name("GetIndexes") router.HandleFunc("/index", handler.handlePostIndex).Methods("POST").Name("PostIndex") router.HandleFunc("/index/", handler.handlePostIndex).Methods("POST").Name("PostIndex") @@ -2020,6 +2022,61 @@ func GetHTTPClient(t *tls.Config) *http.Client { return &http.Client{Transport: transport} } +// handlePostImportAtomicRecord handles /import-atomic-record requests +func (h *Handler) handlePostImportAtomicRecord(w http.ResponseWriter, r *http.Request) { + + // Verify that request is only communicating over protobufs. + if error, code := validateProtobufHeader(r); error != "" { + http.Error(w, error, code) + return + } + + // Read entire body. + body, err := readBody(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + // Unmarshal request based on field type. + + q := r.URL.Query() + sLoss := q.Get("simPowerLossAfter") + loss := 0 + if sLoss != "" { + l, err := strconv.ParseInt(sLoss, 10, 64) + loss = int(l) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } + } + opt := func(o *pilosa.ImportOptions) error { + o.SimPowerLossAfter = loss + return nil + } + + req := &pilosa.AtomicRecord{} + if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.api.ImportAtomicRecord(r.Context(), req, opt); err != nil { + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Write response. + _, err = w.Write(importOk) + if err != nil { + h.logger.Printf("writing import response: %v", err) + } +} + // handlePostImport handles /import requests. func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. diff --git a/internal/public.pb.go b/internal/public.pb.go index 0cf303b0b..8125f04ab 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1751,6 +1751,7 @@ type ImportRequest struct { Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps,proto3" json:"Timestamps,omitempty"` IndexCreatedAt int64 `protobuf:"varint,9,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` FieldCreatedAt int64 `protobuf:"varint,10,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` + Clear bool `protobuf:"varint,11,opt,name=Clear,proto3" json:"Clear,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1859,6 +1860,13 @@ func (m *ImportRequest) GetFieldCreatedAt() int64 { return 0 } +func (m *ImportRequest) GetClear() bool { + if m != nil { + return m.Clear + } + return false +} + type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -1870,6 +1878,7 @@ type ImportValueRequest struct { StringValues []string `protobuf:"bytes,9,rep,name=StringValues,proto3" json:"StringValues,omitempty"` IndexCreatedAt int64 `protobuf:"varint,10,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` FieldCreatedAt int64 `protobuf:"varint,11,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` + Clear bool `protobuf:"varint,12,opt,name=Clear,proto3" json:"Clear,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1978,6 +1987,84 @@ func (m *ImportValueRequest) GetFieldCreatedAt() int64 { return 0 } +func (m *ImportValueRequest) GetClear() bool { + if m != nil { + return m.Clear + } + return false +} + +type AtomicRecord struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Shard uint64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` + Ivr []*ImportValueRequest `protobuf:"bytes,3,rep,name=Ivr,proto3" json:"Ivr,omitempty"` + Ir []*ImportRequest `protobuf:"bytes,4,rep,name=Ir,proto3" json:"Ir,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *AtomicRecord) Reset() { *m = AtomicRecord{} } +func (m *AtomicRecord) String() string { return proto.CompactTextString(m) } +func (*AtomicRecord) ProtoMessage() {} +func (*AtomicRecord) Descriptor() ([]byte, []int) { + return fileDescriptor_413a91106d7bcce8, []int{27} +} +func (m *AtomicRecord) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *AtomicRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_AtomicRecord.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *AtomicRecord) XXX_Merge(src proto.Message) { + xxx_messageInfo_AtomicRecord.Merge(m, src) +} +func (m *AtomicRecord) XXX_Size() int { + return m.Size() +} +func (m *AtomicRecord) XXX_DiscardUnknown() { + xxx_messageInfo_AtomicRecord.DiscardUnknown(m) +} + +var xxx_messageInfo_AtomicRecord proto.InternalMessageInfo + +func (m *AtomicRecord) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *AtomicRecord) GetShard() uint64 { + if m != nil { + return m.Shard + } + return 0 +} + +func (m *AtomicRecord) GetIvr() []*ImportValueRequest { + if m != nil { + return m.Ivr + } + return nil +} + +func (m *AtomicRecord) GetIr() []*ImportRequest { + if m != nil { + return m.Ir + } + return nil +} + type TranslateKeysRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -1991,7 +2078,7 @@ func (m *TranslateKeysRequest) Reset() { *m = TranslateKeysRequest{} } func (m *TranslateKeysRequest) String() string { return proto.CompactTextString(m) } func (*TranslateKeysRequest) ProtoMessage() {} func (*TranslateKeysRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{27} + return fileDescriptor_413a91106d7bcce8, []int{28} } func (m *TranslateKeysRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2052,7 +2139,7 @@ func (m *TranslateKeysResponse) Reset() { *m = TranslateKeysResponse{} } func (m *TranslateKeysResponse) String() string { return proto.CompactTextString(m) } func (*TranslateKeysResponse) ProtoMessage() {} func (*TranslateKeysResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{28} + return fileDescriptor_413a91106d7bcce8, []int{29} } func (m *TranslateKeysResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2101,7 +2188,7 @@ func (m *TranslateIDsRequest) Reset() { *m = TranslateIDsRequest{} } func (m *TranslateIDsRequest) String() string { return proto.CompactTextString(m) } func (*TranslateIDsRequest) ProtoMessage() {} func (*TranslateIDsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{29} + return fileDescriptor_413a91106d7bcce8, []int{30} } func (m *TranslateIDsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2162,7 +2249,7 @@ func (m *TranslateIDsResponse) Reset() { *m = TranslateIDsResponse{} } func (m *TranslateIDsResponse) String() string { return proto.CompactTextString(m) } func (*TranslateIDsResponse) ProtoMessage() {} func (*TranslateIDsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{30} + return fileDescriptor_413a91106d7bcce8, []int{31} } func (m *TranslateIDsResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2210,7 +2297,7 @@ func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestVi func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequestView) ProtoMessage() {} func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{31} + return fileDescriptor_413a91106d7bcce8, []int{32} } func (m *ImportRoaringRequestView) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2269,7 +2356,7 @@ func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} } func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequest) ProtoMessage() {} func (*ImportRoaringRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{32} + return fileDescriptor_413a91106d7bcce8, []int{33} } func (m *ImportRoaringRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2356,7 +2443,7 @@ func (m *ImportColumnAttrsRequest) Reset() { *m = ImportColumnAttrsReque func (m *ImportColumnAttrsRequest) String() string { return proto.CompactTextString(m) } func (*ImportColumnAttrsRequest) ProtoMessage() {} func (*ImportColumnAttrsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{33} + return fileDescriptor_413a91106d7bcce8, []int{34} } func (m *ImportColumnAttrsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2455,6 +2542,7 @@ func init() { proto.RegisterType((*QueryResult)(nil), "internal.QueryResult") proto.RegisterType((*ImportRequest)(nil), "internal.ImportRequest") proto.RegisterType((*ImportValueRequest)(nil), "internal.ImportValueRequest") + proto.RegisterType((*AtomicRecord)(nil), "internal.AtomicRecord") proto.RegisterType((*TranslateKeysRequest)(nil), "internal.TranslateKeysRequest") proto.RegisterType((*TranslateKeysResponse)(nil), "internal.TranslateKeysResponse") proto.RegisterType((*TranslateIDsRequest)(nil), "internal.TranslateIDsRequest") @@ -2467,105 +2555,109 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1555 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x73, 0xdb, 0x44, - 0x14, 0x8f, 0x2c, 0x39, 0xb6, 0x9f, 0x93, 0x34, 0xdd, 0xa6, 0x45, 0x53, 0xda, 0xd4, 0xa3, 0x09, - 0x60, 0x38, 0xa4, 0x93, 0xd2, 0x76, 0x7a, 0x01, 0xda, 0xd4, 0x29, 0xd1, 0x94, 0x84, 0xb2, 0xce, - 0x84, 0x1b, 0x33, 0x8a, 0xbd, 0xa4, 0x1a, 0x64, 0xcb, 0xc8, 0x32, 0x4e, 0x2e, 0xcc, 0xf0, 0x19, - 0xb8, 0xf0, 0x11, 0xb8, 0xf2, 0x15, 0x38, 0x71, 0xe4, 0xce, 0x85, 0x29, 0x7c, 0x8b, 0x5e, 0x98, - 0xf7, 0x56, 0xab, 0x5d, 0x29, 0x4a, 0x9a, 0xe9, 0x70, 0xdb, 0xf7, 0x67, 0xdf, 0xbe, 0xf7, 0x7b, - 0x6f, 0xdf, 0x3e, 0x09, 0x96, 0x26, 0xb3, 0xa3, 0x28, 0x1c, 0x6c, 0x4e, 0x92, 0x38, 0x8d, 0x59, - 0x33, 0x1c, 0xa7, 0x22, 0x19, 0x07, 0x91, 0x37, 0x05, 0x9b, 0xc7, 0x73, 0xe6, 0x42, 0xe3, 0x69, - 0x1c, 0xcd, 0x46, 0xe3, 0xa9, 0x6b, 0x75, 0xec, 0xae, 0xc3, 0x15, 0xc9, 0x18, 0x38, 0xcf, 0xc5, - 0xe9, 0xd4, 0xb5, 0x3b, 0x76, 0xb7, 0xc5, 0x69, 0xcd, 0x36, 0xa0, 0xfe, 0x24, 0x4d, 0x93, 0xa9, - 0x5b, 0xeb, 0xd8, 0xdd, 0xf6, 0xbd, 0x95, 0x4d, 0x65, 0x6e, 0x13, 0xd9, 0x5c, 0x0a, 0xd1, 0x26, - 0x8f, 0x83, 0x24, 0x1c, 0x1f, 0xbb, 0x4e, 0xc7, 0xea, 0x2e, 0x71, 0x45, 0x7a, 0x7b, 0xd0, 0xea, - 0x87, 0xc7, 0x63, 0x31, 0xc4, 0xa3, 0xef, 0x80, 0xfd, 0x22, 0xc6, 0x63, 0xad, 0x6e, 0xfb, 0xde, - 0xb2, 0x36, 0xc5, 0xe3, 0x39, 0x47, 0x09, 0x2a, 0xec, 0x8b, 0x63, 0xb7, 0x56, 0xa9, 0xb0, 0x2f, - 0x8e, 0xbd, 0x47, 0xb0, 0xc2, 0xe3, 0xb9, 0x3f, 0x14, 0xe3, 0x34, 0xfc, 0x36, 0x14, 0x09, 0x39, - 0xcd, 0xe3, 0xb9, 0x8a, 0x85, 0xd6, 0x79, 0x20, 0x35, 0x1d, 0x88, 0x77, 0x13, 0x16, 0xfd, 0xde, - 0x17, 0xe1, 0x34, 0x65, 0xab, 0x60, 0xfb, 0x3d, 0xb5, 0x01, 0x97, 0x9e, 0x0f, 0x57, 0x77, 0x4e, - 0xd2, 0x24, 0x18, 0xa4, 0x62, 0xe8, 0xf7, 0x24, 0x1c, 0x6c, 0x05, 0x6a, 0x7e, 0x8f, 0x7c, 0x75, - 0x78, 0xcd, 0xef, 0xb1, 0x0d, 0x70, 0x0e, 0x83, 0x48, 0x01, 0xb1, 0xaa, 0x9d, 0x93, 0x66, 0x39, - 0x49, 0xbd, 0xa3, 0x82, 0xa9, 0xbd, 0x20, 0x4d, 0xc2, 0x13, 0x76, 0x03, 0x16, 0x9f, 0x85, 0x22, - 0x1a, 0xca, 0x43, 0x5b, 0x3c, 0xa3, 0xd8, 0x03, 0x9d, 0x0a, 0x69, 0xf5, 0x5d, 0x6d, 0xf5, 0x8c, - 0x43, 0x79, 0x9e, 0xbc, 0xdb, 0xd0, 0x78, 0x2e, 0x4e, 0x29, 0x16, 0x15, 0xa9, 0x65, 0x44, 0xfa, - 0x97, 0x05, 0xd7, 0xf2, 0xdd, 0x07, 0xc1, 0x51, 0x24, 0x0e, 0x83, 0x68, 0x26, 0xd8, 0x86, 0x8a, - 0xdb, 0xaa, 0xf2, 0x7f, 0x77, 0x81, 0xb0, 0x60, 0x1f, 0xe4, 0xd8, 0xa1, 0xda, 0x55, 0xad, 0x96, - 0x1d, 0xb9, 0xbb, 0x90, 0x55, 0xc6, 0x2d, 0x68, 0x6e, 0xf7, 0x7d, 0x32, 0xed, 0xda, 0x1d, 0xab, - 0x6b, 0xef, 0x2e, 0xf0, 0x9c, 0xc3, 0x6e, 0x42, 0x63, 0x6f, 0x96, 0x8a, 0x13, 0xbf, 0x47, 0x15, - 0xe1, 0xec, 0x2e, 0x70, 0xc5, 0xc0, 0x9d, 0xb4, 0x7c, 0x2e, 0x4e, 0xdd, 0x7a, 0xc7, 0xea, 0xb6, - 0x70, 0xa7, 0xe2, 0xb0, 0x35, 0x70, 0xb6, 0xe3, 0x38, 0x72, 0x17, 0x3b, 0x56, 0xb7, 0x89, 0xa7, - 0x21, 0xb5, 0xdd, 0x80, 0x3a, 0x19, 0xf6, 0x7e, 0x84, 0xb5, 0x62, 0x70, 0x59, 0xba, 0x18, 0xd8, - 0x68, 0xcf, 0xca, 0xec, 0x21, 0xc1, 0x56, 0x29, 0x85, 0xb5, 0xec, 0x7c, 0x4c, 0xe2, 0x03, 0x58, - 0x24, 0x33, 0xb2, 0xc8, 0xdb, 0xf7, 0x6e, 0x57, 0x00, 0xae, 0x21, 0xe3, 0x99, 0xf2, 0x76, 0x8b, - 0x10, 0xff, 0x32, 0xf1, 0x7b, 0xde, 0x27, 0x65, 0x70, 0x29, 0x97, 0x98, 0x88, 0xfd, 0x60, 0x24, - 0xe4, 0xf9, 0x9c, 0xd6, 0xc8, 0x3b, 0x38, 0x9d, 0x08, 0x72, 0xa0, 0xc5, 0x69, 0xed, 0xfd, 0x64, - 0xc1, 0x4a, 0x71, 0x3f, 0xfa, 0x64, 0x54, 0xc7, 0x05, 0x3e, 0x91, 0x56, 0x5e, 0x3c, 0x8f, 0xca, - 0xc5, 0xb3, 0x7e, 0xde, 0xbe, 0x72, 0xfd, 0x7c, 0x0a, 0xce, 0x8b, 0x20, 0x4c, 0xce, 0x54, 0xf8, - 0xaa, 0x84, 0xd0, 0x26, 0x77, 0x6d, 0x99, 0x8b, 0xfa, 0xd3, 0x78, 0x36, 0x4e, 0x25, 0x86, 0x5c, - 0x12, 0xde, 0x0e, 0xb4, 0x70, 0xbf, 0x0c, 0xdc, 0x93, 0xc6, 0xb2, 0xb2, 0x32, 0xfa, 0x03, 0x72, - 0xb9, 0x3c, 0x68, 0x0d, 0xea, 0xa4, 0x9c, 0x21, 0x21, 0x09, 0x6f, 0x17, 0x00, 0xa5, 0x53, 0x69, - 0x67, 0x03, 0xea, 0x44, 0x65, 0x20, 0x94, 0x0d, 0x49, 0xe1, 0x39, 0x96, 0x6e, 0x43, 0xdd, 0x1f, - 0xa7, 0x0f, 0xef, 0xa3, 0x58, 0x16, 0x24, 0x7a, 0x63, 0xf3, 0xac, 0x64, 0x66, 0xd0, 0x94, 0xd0, - 0xc5, 0x73, 0x6d, 0xc0, 0x32, 0x0c, 0x20, 0x17, 0xdb, 0x4a, 0x4f, 0xc5, 0x49, 0x04, 0x5e, 0x5b, - 0x1e, 0xcf, 0x35, 0x24, 0x19, 0xc5, 0xde, 0x53, 0xa7, 0x38, 0x14, 0xf3, 0x15, 0xe3, 0x2a, 0xa1, - 0x17, 0xea, 0xd8, 0x6f, 0x00, 0x3e, 0x4f, 0xe2, 0xd9, 0x84, 0x40, 0x63, 0x5d, 0xa8, 0x13, 0x95, - 0xc5, 0xc7, 0xf4, 0x26, 0xe5, 0x1b, 0x97, 0x0a, 0xd5, 0xa0, 0x63, 0x72, 0xfa, 0xb3, 0x91, 0xbc, - 0x69, 0x1c, 0x97, 0x58, 0x4a, 0xcd, 0xc3, 0x20, 0xca, 0xc5, 0x87, 0x41, 0x94, 0xc5, 0x8d, 0xcb, - 0xa2, 0x19, 0x5b, 0x99, 0xb9, 0x09, 0xcd, 0x67, 0x51, 0x1c, 0xa4, 0xa8, 0x8c, 0xb6, 0x2c, 0x9e, - 0xd3, 0x6c, 0x0b, 0xa0, 0x27, 0x06, 0xe1, 0x28, 0x88, 0x50, 0xea, 0x94, 0x1b, 0x40, 0x26, 0xe3, - 0x86, 0x92, 0xf7, 0x00, 0x1a, 0x19, 0x55, 0x8d, 0x3d, 0x72, 0xfb, 0x83, 0x20, 0x12, 0xca, 0x0b, - 0x22, 0xbc, 0xaf, 0x61, 0x59, 0x16, 0x23, 0x3e, 0x1f, 0x7d, 0x91, 0x5e, 0xa2, 0x14, 0x2f, 0xf5, - 0x10, 0x79, 0xbf, 0x5a, 0xe0, 0xe0, 0x4a, 0x19, 0xb0, 0xb4, 0x01, 0xf3, 0x36, 0x3a, 0xf2, 0x36, - 0xb2, 0x0e, 0xb4, 0xfb, 0x29, 0xbe, 0x53, 0xba, 0x8d, 0xb5, 0xb8, 0xc9, 0x42, 0xbc, 0xfc, 0x71, - 0xaa, 0xd3, 0x6d, 0xf3, 0x9c, 0x66, 0xb7, 0xa0, 0x85, 0xbd, 0x49, 0x0a, 0xb1, 0x91, 0x35, 0xb9, - 0x66, 0xb0, 0x75, 0x00, 0x85, 0xec, 0x4c, 0x50, 0x37, 0xb3, 0xb8, 0xc1, 0xf1, 0xee, 0x42, 0x03, - 0x3d, 0xdd, 0x0b, 0x26, 0x3a, 0x36, 0xeb, 0xa2, 0xd8, 0x5e, 0x5b, 0xb0, 0xf4, 0xd5, 0x4c, 0x24, - 0xa7, 0x5c, 0x7c, 0x3f, 0x13, 0xd3, 0x14, 0xb1, 0x25, 0x5a, 0xd5, 0x32, 0x11, 0x58, 0xb5, 0xfd, - 0x97, 0x41, 0x32, 0x94, 0x48, 0x39, 0x3c, 0xa3, 0x30, 0x56, 0x8d, 0xf9, 0x94, 0x62, 0x6d, 0x72, - 0x93, 0x45, 0xf5, 0x2e, 0x46, 0x71, 0xaa, 0x82, 0xc9, 0x28, 0xd6, 0x85, 0x2b, 0x3b, 0x27, 0x83, - 0x68, 0x36, 0x14, 0x3c, 0x9e, 0xcb, 0xdd, 0xd4, 0x9c, 0x79, 0x99, 0xcd, 0xde, 0xc7, 0xe6, 0x46, - 0x2c, 0xd5, 0x9a, 0x1a, 0xa4, 0x58, 0xe2, 0xb2, 0x2d, 0x58, 0xda, 0x19, 0x1d, 0x89, 0xe1, 0x50, - 0x0c, 0x7b, 0x41, 0x1a, 0xb8, 0x4d, 0x8a, 0xbb, 0xf4, 0xe0, 0x17, 0x54, 0xbc, 0x9f, 0x2d, 0x58, - 0xce, 0xa2, 0x9f, 0x4e, 0xe2, 0xf1, 0x54, 0x60, 0x8a, 0x77, 0x92, 0x44, 0xa5, 0x78, 0x27, 0x49, - 0xd8, 0x5d, 0x68, 0x70, 0x31, 0x9d, 0x45, 0xa9, 0xaa, 0x92, 0xeb, 0xda, 0xa2, 0xda, 0x3b, 0x8b, - 0x52, 0xae, 0xb4, 0xd8, 0x67, 0xb0, 0x52, 0xa8, 0x43, 0xf5, 0x2c, 0xbc, 0xa3, 0xf7, 0x15, 0xe4, - 0xbc, 0xa4, 0xee, 0xbd, 0x76, 0xa0, 0x6d, 0x58, 0xce, 0x8b, 0x0c, 0xf1, 0x59, 0xce, 0x8a, 0xec, - 0x0e, 0xcd, 0x5d, 0xe7, 0x4c, 0x3d, 0xd8, 0x93, 0x96, 0xc0, 0xda, 0xcf, 0xca, 0xd2, 0xda, 0xd7, - 0x8d, 0xd0, 0xbe, 0xa8, 0x11, 0xe2, 0x14, 0xf7, 0x32, 0x18, 0x1f, 0x8b, 0x21, 0x95, 0x65, 0x93, - 0x2b, 0x92, 0x6d, 0xea, 0xae, 0x40, 0x79, 0x2c, 0xf4, 0x1a, 0x25, 0xe1, 0xba, 0x73, 0xc8, 0x2e, - 0x87, 0x93, 0x41, 0x43, 0xd6, 0x8b, 0xa4, 0xd8, 0x43, 0x68, 0xeb, 0xf6, 0x35, 0xcd, 0x52, 0xb4, - 0xa6, 0x4d, 0x69, 0x21, 0x37, 0x15, 0xd9, 0xe3, 0xf2, 0x88, 0xe6, 0xb6, 0xc8, 0x0b, 0xb7, 0x10, - 0xb9, 0x21, 0xe7, 0xe5, 0x91, 0x6e, 0xcb, 0x98, 0x19, 0x5d, 0xa0, 0xcd, 0xd7, 0xf4, 0xe6, 0x5c, - 0xc4, 0x8d, 0xc9, 0xf2, 0xbe, 0xf9, 0x96, 0xb8, 0x6d, 0xda, 0xb3, 0x56, 0x44, 0x4e, 0xca, 0xb8, - 0xf9, 0xe6, 0x6c, 0x19, 0x0f, 0x99, 0xbb, 0x54, 0x3e, 0x28, 0x17, 0x71, 0xe3, 0xb9, 0xf3, 0x2b, - 0xe6, 0x3b, 0x77, 0x99, 0xb6, 0x56, 0x0f, 0x6f, 0x52, 0x85, 0x57, 0x4c, 0x85, 0x8f, 0xcb, 0x93, - 0x80, 0xbb, 0x52, 0x06, 0xaa, 0x28, 0xe7, 0x25, 0x7d, 0xef, 0xb7, 0x1a, 0x2c, 0xfb, 0xa3, 0x49, - 0x9c, 0xa4, 0x46, 0x4b, 0xf0, 0xc7, 0x43, 0x71, 0xa2, 0x5a, 0x02, 0x11, 0xd5, 0xaf, 0x26, 0xb5, - 0x66, 0x6c, 0x0d, 0xd4, 0x0a, 0x1c, 0x2e, 0x09, 0xa3, 0x1c, 0x9c, 0x42, 0x39, 0xdc, 0x82, 0x96, - 0xac, 0x7d, 0x14, 0xd5, 0x49, 0xa4, 0x19, 0xf2, 0x03, 0x60, 0x4e, 0x83, 0x63, 0x83, 0x46, 0x51, - 0x45, 0x62, 0x1b, 0x94, 0x6a, 0x24, 0x6c, 0x92, 0xd0, 0xe0, 0xa0, 0xfc, 0x20, 0x1c, 0x89, 0x69, - 0x1a, 0x8c, 0x26, 0xd8, 0x57, 0xec, 0xae, 0xcd, 0x0d, 0x0e, 0xb6, 0x14, 0x0a, 0xe2, 0x69, 0x22, - 0x82, 0x54, 0x0c, 0x9f, 0xa4, 0x54, 0x4e, 0x36, 0x2f, 0x71, 0x51, 0x8f, 0xc2, 0xd2, 0x7a, 0x20, - 0xf5, 0x8a, 0x5c, 0xef, 0xf7, 0x1a, 0x30, 0x89, 0x99, 0x1c, 0xf1, 0xfe, 0x37, 0xe0, 0x2e, 0x06, - 0xa8, 0x08, 0x43, 0xe3, 0x0c, 0x0c, 0x37, 0xf2, 0xc1, 0x54, 0x42, 0x90, 0x51, 0xd8, 0xb5, 0xf5, - 0x9b, 0x21, 0xf1, 0xb3, 0xb8, 0xc9, 0x62, 0x1e, 0x2c, 0x19, 0x0f, 0x16, 0xde, 0x36, 0xb4, 0x5d, - 0xe0, 0x55, 0x80, 0x08, 0x97, 0x04, 0xb1, 0x5d, 0x09, 0xe2, 0x21, 0xac, 0x1d, 0x24, 0xc1, 0x78, - 0x1a, 0x05, 0xa9, 0x40, 0xf7, 0xdf, 0x06, 0xc5, 0x8a, 0xaf, 0x4d, 0xef, 0x43, 0xb8, 0x5e, 0xb2, - 0xab, 0x7b, 0x3d, 0xc2, 0x6a, 0xeb, 0x6f, 0xb6, 0x3e, 0x5c, 0xcb, 0x55, 0xfd, 0xde, 0x5b, 0x79, - 0x70, 0xd6, 0xe8, 0x47, 0x46, 0x5c, 0x64, 0x34, 0x3b, 0xbe, 0xca, 0xd7, 0x6d, 0x70, 0xb3, 0xbb, - 0x27, 0x3f, 0x75, 0x33, 0x0f, 0x0e, 0x43, 0x31, 0x3f, 0xef, 0x6b, 0x80, 0xde, 0xba, 0x1a, 0x7d, - 0x20, 0xd3, 0xda, 0xfb, 0xd7, 0x82, 0xb5, 0x2a, 0x23, 0x34, 0xbc, 0x45, 0x22, 0x90, 0xaf, 0x5b, - 0x93, 0x4b, 0x82, 0x3d, 0x82, 0xfa, 0x0f, 0xa1, 0x98, 0xab, 0xd7, 0xcd, 0x33, 0x06, 0xcf, 0x73, - 0x3c, 0xe1, 0x72, 0x03, 0x96, 0xd7, 0x93, 0x41, 0x1a, 0xc6, 0x63, 0x35, 0xca, 0x4a, 0x0a, 0xcf, - 0xd9, 0x8e, 0xe2, 0xc1, 0x77, 0xf2, 0x23, 0x8d, 0x4b, 0xa2, 0xa2, 0x5c, 0xea, 0x97, 0x2c, 0x97, - 0xc5, 0xea, 0x3b, 0x67, 0x29, 0xac, 0x8c, 0x71, 0xe3, 0x8d, 0x19, 0x93, 0x77, 0x4c, 0xcd, 0x8d, - 0x74, 0xc7, 0x5c, 0x39, 0x33, 0xe9, 0xd1, 0x50, 0x91, 0x38, 0xa7, 0xe1, 0x92, 0xbe, 0xd0, 0x1d, - 0xca, 0x52, 0x4e, 0xbf, 0xe1, 0x66, 0x9e, 0x0d, 0x76, 0xb1, 0x2a, 0xd8, 0xed, 0xd5, 0x3f, 0x5e, - 0xad, 0x5b, 0x7f, 0xbe, 0x5a, 0xb7, 0xfe, 0x7e, 0xb5, 0x6e, 0xfd, 0xf2, 0xcf, 0xfa, 0xc2, 0xd1, - 0x22, 0xfd, 0x61, 0xf9, 0xf8, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xac, 0xd8, 0x28, 0xa3, 0x71, - 0x11, 0x00, 0x00, + // 1620 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0xdb, 0x46, + 0x16, 0x37, 0x45, 0xca, 0x92, 0x9e, 0x64, 0xc7, 0x99, 0x38, 0x59, 0x22, 0xeb, 0x38, 0x02, 0xe1, + 0xdd, 0x68, 0xf7, 0xe0, 0xc0, 0xd9, 0x24, 0xc8, 0x65, 0x77, 0x63, 0x47, 0xce, 0x9a, 0xc8, 0xda, + 0x9b, 0x1d, 0x19, 0xde, 0xdb, 0x02, 0xb4, 0x34, 0x75, 0x88, 0x52, 0xa2, 0x4a, 0x51, 0x91, 0x7d, + 0x29, 0xd0, 0xcf, 0x90, 0x4b, 0x3f, 0x42, 0x3f, 0x47, 0x2f, 0xed, 0xb1, 0xc7, 0x02, 0xbd, 0x14, + 0x69, 0xbf, 0x45, 0x2e, 0xc5, 0x7b, 0xc3, 0xd1, 0x0c, 0x29, 0xda, 0x31, 0x82, 0xde, 0xe6, 0xfd, + 0x99, 0x37, 0xf3, 0x7e, 0xef, 0xc7, 0x37, 0x4f, 0x82, 0xd6, 0x78, 0x7a, 0x1a, 0x85, 0xfd, 0xed, + 0x71, 0x12, 0xa7, 0x31, 0xab, 0x87, 0xa3, 0x54, 0x24, 0xa3, 0x20, 0xf2, 0x26, 0x60, 0xf3, 0x78, + 0xc6, 0x5c, 0xa8, 0xbd, 0x88, 0xa3, 0xe9, 0x70, 0x34, 0x71, 0xad, 0xb6, 0xdd, 0x71, 0xb8, 0x12, + 0x19, 0x03, 0xe7, 0x95, 0xb8, 0x98, 0xb8, 0x76, 0xdb, 0xee, 0x34, 0x38, 0xad, 0xd9, 0x16, 0x54, + 0x77, 0xd3, 0x34, 0x99, 0xb8, 0x95, 0xb6, 0xdd, 0x69, 0x3e, 0x5a, 0xdd, 0x56, 0xe1, 0xb6, 0x51, + 0xcd, 0xa5, 0x11, 0x63, 0xf2, 0x38, 0x48, 0xc2, 0xd1, 0x99, 0xeb, 0xb4, 0xad, 0x4e, 0x8b, 0x2b, + 0xd1, 0x3b, 0x84, 0x46, 0x2f, 0x3c, 0x1b, 0x89, 0x01, 0x1e, 0x7d, 0x1f, 0xec, 0xd7, 0x31, 0x1e, + 0x6b, 0x75, 0x9a, 0x8f, 0x56, 0x74, 0x28, 0x1e, 0xcf, 0x38, 0x5a, 0xd0, 0xe1, 0x48, 0x9c, 0xb9, + 0x95, 0x52, 0x87, 0x23, 0x71, 0xe6, 0x3d, 0x83, 0x55, 0x1e, 0xcf, 0xfc, 0x81, 0x18, 0xa5, 0xe1, + 0x67, 0xa1, 0x48, 0xe8, 0xd2, 0x3c, 0x9e, 0xa9, 0x5c, 0x68, 0x3d, 0x4f, 0xa4, 0xa2, 0x13, 0xf1, + 0xee, 0xc2, 0xb2, 0xdf, 0xfd, 0x77, 0x38, 0x49, 0xd9, 0x1a, 0xd8, 0x7e, 0x57, 0x6d, 0xc0, 0xa5, + 0xe7, 0xc3, 0xcd, 0xfd, 0xf3, 0x34, 0x09, 0xfa, 0xa9, 0x18, 0xf8, 0x5d, 0x09, 0x07, 0x5b, 0x85, + 0x8a, 0xdf, 0xa5, 0xbb, 0x3a, 0xbc, 0xe2, 0x77, 0xd9, 0x16, 0x38, 0x27, 0x41, 0xa4, 0x80, 0x58, + 0xd3, 0x97, 0x93, 0x61, 0x39, 0x59, 0xbd, 0xd3, 0x5c, 0xa8, 0xc3, 0x20, 0x4d, 0xc2, 0x73, 0x76, + 0x07, 0x96, 0x5f, 0x86, 0x22, 0x1a, 0xc8, 0x43, 0x1b, 0x3c, 0x93, 0xd8, 0x13, 0x5d, 0x0a, 0x19, + 0xf5, 0x8f, 0x3a, 0xea, 0xc2, 0x85, 0xe6, 0x75, 0xf2, 0xee, 0x41, 0xed, 0x95, 0xb8, 0xa0, 0x5c, + 0x54, 0xa6, 0x96, 0x91, 0xe9, 0x4f, 0x16, 0xdc, 0x9a, 0xef, 0x3e, 0x0e, 0x4e, 0x23, 0x71, 0x12, + 0x44, 0x53, 0xc1, 0xb6, 0x54, 0xde, 0x56, 0xd9, 0xfd, 0x0f, 0x96, 0x08, 0x0b, 0xf6, 0x60, 0x8e, + 0x1d, 0xba, 0xdd, 0xd4, 0x6e, 0xd9, 0x91, 0x07, 0x4b, 0x19, 0x33, 0x36, 0xa0, 0xbe, 0xd7, 0xf3, + 0x29, 0xb4, 0x6b, 0xb7, 0xad, 0x8e, 0x7d, 0xb0, 0xc4, 0xe7, 0x1a, 0x76, 0x17, 0x6a, 0x87, 0xd3, + 0x54, 0x9c, 0xfb, 0x5d, 0x62, 0x84, 0x73, 0xb0, 0xc4, 0x95, 0x02, 0x77, 0xd2, 0xf2, 0x95, 0xb8, + 0x70, 0xab, 0x6d, 0xab, 0xd3, 0xc0, 0x9d, 0x4a, 0xc3, 0xd6, 0xc1, 0xd9, 0x8b, 0xe3, 0xc8, 0x5d, + 0x6e, 0x5b, 0x9d, 0x3a, 0x9e, 0x86, 0xd2, 0x5e, 0x0d, 0xaa, 0x14, 0xd8, 0xfb, 0x12, 0xd6, 0xf3, + 0xc9, 0x65, 0xe5, 0x62, 0x60, 0x63, 0x3c, 0x2b, 0x8b, 0x87, 0x02, 0x5b, 0xa3, 0x12, 0x56, 0xb2, + 0xf3, 0xb1, 0x88, 0x4f, 0x60, 0x99, 0xc2, 0x48, 0x92, 0x37, 0x1f, 0xdd, 0x2b, 0x01, 0x5c, 0x43, + 0xc6, 0x33, 0xe7, 0xbd, 0x06, 0x21, 0xfe, 0x9f, 0xc4, 0xef, 0x7a, 0x7f, 0x2f, 0x82, 0x4b, 0xb5, + 0xc4, 0x42, 0x1c, 0x05, 0x43, 0x21, 0xcf, 0xe7, 0xb4, 0x46, 0xdd, 0xf1, 0xc5, 0x58, 0xd0, 0x05, + 0x1a, 0x9c, 0xd6, 0xde, 0x57, 0x16, 0xac, 0xe6, 0xf7, 0xe3, 0x9d, 0x0c, 0x76, 0x5c, 0x71, 0x27, + 0xf2, 0x9a, 0x93, 0xe7, 0x59, 0x91, 0x3c, 0x9b, 0x97, 0xed, 0x2b, 0xf2, 0xe7, 0x1f, 0xe0, 0xbc, + 0x0e, 0xc2, 0x64, 0x81, 0xe1, 0x6b, 0x12, 0x42, 0x9b, 0xae, 0x6b, 0xcb, 0x5a, 0x54, 0x5f, 0xc4, + 0xd3, 0x51, 0x2a, 0x31, 0xe4, 0x52, 0xf0, 0xf6, 0xa1, 0x81, 0xfb, 0x65, 0xe2, 0x9e, 0x0c, 0x96, + 0xd1, 0xca, 0xe8, 0x0f, 0xa8, 0xe5, 0xf2, 0xa0, 0x75, 0xa8, 0x92, 0x73, 0x86, 0x84, 0x14, 0xbc, + 0x03, 0x00, 0xb4, 0x4e, 0x64, 0x9c, 0x2d, 0xa8, 0x92, 0x94, 0x81, 0x50, 0x0c, 0x24, 0x8d, 0x97, + 0x44, 0xba, 0x07, 0x55, 0x7f, 0x94, 0x3e, 0x7d, 0x8c, 0x66, 0x49, 0x48, 0xbc, 0x8d, 0xcd, 0x33, + 0xca, 0x4c, 0xa1, 0x2e, 0xa1, 0x8b, 0x67, 0x3a, 0x80, 0x65, 0x04, 0x40, 0x2d, 0xb6, 0x95, 0xae, + 0xca, 0x93, 0x04, 0xfc, 0x6c, 0x79, 0x3c, 0xd3, 0x90, 0x64, 0x12, 0xfb, 0x93, 0x3a, 0xc5, 0xa1, + 0x9c, 0x6f, 0x18, 0x9f, 0x12, 0xde, 0x42, 0x1d, 0xfb, 0x7f, 0x80, 0x7f, 0x25, 0xf1, 0x74, 0x4c, + 0xa0, 0xb1, 0x0e, 0x54, 0x49, 0xca, 0xf2, 0x63, 0x7a, 0x93, 0xba, 0x1b, 0x97, 0x0e, 0xe5, 0xa0, + 0x63, 0x71, 0x7a, 0xd3, 0xa1, 0xfc, 0xd2, 0x38, 0x2e, 0x91, 0x4a, 0xf5, 0x93, 0x20, 0x9a, 0x9b, + 0x4f, 0x82, 0x28, 0xcb, 0x1b, 0x97, 0xf9, 0x30, 0xb6, 0x0a, 0x73, 0x17, 0xea, 0x2f, 0xa3, 0x38, + 0x48, 0xd1, 0x19, 0x63, 0x59, 0x7c, 0x2e, 0xb3, 0x1d, 0x80, 0xae, 0xe8, 0x87, 0xc3, 0x20, 0x42, + 0xab, 0x53, 0x6c, 0x00, 0x99, 0x8d, 0x1b, 0x4e, 0xde, 0x13, 0xa8, 0x65, 0x52, 0x39, 0xf6, 0xa8, + 0xed, 0xf5, 0x83, 0x48, 0xa8, 0x5b, 0x90, 0xe0, 0xfd, 0x0f, 0x56, 0x24, 0x19, 0xf1, 0xf9, 0xe8, + 0x89, 0xf4, 0x1a, 0x54, 0xbc, 0xd6, 0x43, 0xe4, 0x7d, 0x63, 0x81, 0x83, 0x2b, 0x15, 0xc0, 0xd2, + 0x01, 0xcc, 0xaf, 0xd1, 0x91, 0x5f, 0x23, 0x6b, 0x43, 0xb3, 0x97, 0xe2, 0x3b, 0xa5, 0xdb, 0x58, + 0x83, 0x9b, 0x2a, 0xc4, 0xcb, 0x1f, 0xa5, 0xba, 0xdc, 0x36, 0x9f, 0xcb, 0x6c, 0x03, 0x1a, 0xd8, + 0x9b, 0xa4, 0x11, 0x1b, 0x59, 0x9d, 0x6b, 0x05, 0xdb, 0x04, 0x50, 0xc8, 0x4e, 0x05, 0x75, 0x33, + 0x8b, 0x1b, 0x1a, 0xef, 0x21, 0xd4, 0xf0, 0xa6, 0x87, 0xc1, 0x58, 0xe7, 0x66, 0x5d, 0x95, 0xdb, + 0x07, 0x0b, 0x5a, 0xff, 0x9d, 0x8a, 0xe4, 0x82, 0x8b, 0x2f, 0xa6, 0x62, 0x92, 0x22, 0xb6, 0x24, + 0x2b, 0x2e, 0x93, 0x80, 0xac, 0xed, 0xbd, 0x09, 0x92, 0x81, 0x44, 0xca, 0xe1, 0x99, 0x84, 0xb9, + 0x6a, 0xcc, 0x27, 0x94, 0x6b, 0x9d, 0x9b, 0x2a, 0xe2, 0xbb, 0x18, 0xc6, 0xa9, 0x4a, 0x26, 0x93, + 0x58, 0x07, 0x6e, 0xec, 0x9f, 0xf7, 0xa3, 0xe9, 0x40, 0xf0, 0x78, 0x26, 0x77, 0x53, 0x73, 0xe6, + 0x45, 0x35, 0xfb, 0x33, 0x36, 0x37, 0x52, 0xa9, 0xd6, 0x54, 0x23, 0xc7, 0x82, 0x96, 0xed, 0x40, + 0x6b, 0x7f, 0x78, 0x2a, 0x06, 0x03, 0x31, 0xe8, 0x06, 0x69, 0xe0, 0xd6, 0x29, 0xef, 0xc2, 0x83, + 0x9f, 0x73, 0xf1, 0xde, 0x59, 0xb0, 0x92, 0x65, 0x3f, 0x19, 0xc7, 0xa3, 0x89, 0xc0, 0x12, 0xef, + 0x27, 0x89, 0x2a, 0xf1, 0x7e, 0x92, 0xb0, 0x87, 0x50, 0xe3, 0x62, 0x32, 0x8d, 0x52, 0xc5, 0x92, + 0xdb, 0x3a, 0xa2, 0xda, 0x3b, 0x8d, 0x52, 0xae, 0xbc, 0xd8, 0x3f, 0x61, 0x35, 0xc7, 0x43, 0xf5, + 0x2c, 0xfc, 0x41, 0xef, 0xcb, 0xd9, 0x79, 0xc1, 0xdd, 0xfb, 0xe0, 0x40, 0xd3, 0x88, 0x3c, 0x27, + 0x19, 0xe2, 0xb3, 0x92, 0x91, 0xec, 0x3e, 0xcd, 0x5d, 0x97, 0x4c, 0x3d, 0xd8, 0x93, 0x5a, 0x60, + 0x1d, 0x65, 0xb4, 0xb4, 0x8e, 0x74, 0x23, 0xb4, 0xaf, 0x6a, 0x84, 0x38, 0xc5, 0xbd, 0x09, 0x46, + 0x67, 0x62, 0x40, 0xb4, 0xac, 0x73, 0x25, 0xb2, 0x6d, 0xdd, 0x15, 0xa8, 0x8e, 0xb9, 0x5e, 0xa3, + 0x2c, 0x5c, 0x77, 0x0e, 0xd9, 0xe5, 0x70, 0x32, 0xa8, 0x49, 0xbe, 0x48, 0x89, 0x3d, 0x85, 0xa6, + 0x6e, 0x5f, 0x93, 0xac, 0x44, 0xeb, 0x3a, 0x94, 0x36, 0x72, 0xd3, 0x91, 0x3d, 0x2f, 0x8e, 0x68, + 0x6e, 0x83, 0x6e, 0xe1, 0xe6, 0x32, 0x37, 0xec, 0xbc, 0x38, 0xd2, 0xed, 0x18, 0x33, 0xa3, 0x0b, + 0xb4, 0xf9, 0x96, 0xde, 0x3c, 0x37, 0x71, 0x63, 0xb2, 0x7c, 0x6c, 0xbe, 0x25, 0x6e, 0x93, 0xf6, + 0xac, 0xe7, 0x91, 0x93, 0x36, 0x6e, 0xbe, 0x39, 0x3b, 0xc6, 0x43, 0xe6, 0xb6, 0x8a, 0x07, 0xcd, + 0x4d, 0xdc, 0x78, 0xee, 0xfc, 0x92, 0xf9, 0xce, 0x5d, 0xa1, 0xad, 0xe5, 0xc3, 0x9b, 0x74, 0xe1, + 0x25, 0x53, 0xe1, 0xf3, 0xe2, 0x24, 0xe0, 0xae, 0x16, 0x81, 0xca, 0xdb, 0x79, 0xc1, 0xdf, 0xfb, + 0xae, 0x02, 0x2b, 0xfe, 0x70, 0x1c, 0x27, 0xa9, 0xd1, 0x12, 0xfc, 0xd1, 0x40, 0x9c, 0xab, 0x96, + 0x40, 0x42, 0xf9, 0xab, 0x49, 0xad, 0x19, 0x5b, 0x03, 0xb5, 0x02, 0x87, 0x4b, 0xc1, 0xa0, 0x83, + 0x93, 0xa3, 0xc3, 0x06, 0x34, 0x24, 0xf7, 0xd1, 0x54, 0x25, 0x93, 0x56, 0xc8, 0x1f, 0x00, 0x33, + 0x1a, 0x1c, 0x6b, 0x34, 0x8a, 0x2a, 0x11, 0xdb, 0xa0, 0x74, 0x23, 0x63, 0x9d, 0x8c, 0x86, 0x06, + 0xed, 0xc7, 0xe1, 0x50, 0x4c, 0xd2, 0x60, 0x38, 0xc6, 0xbe, 0x62, 0x77, 0x6c, 0x6e, 0x68, 0xb0, + 0xa5, 0x50, 0x12, 0x2f, 0x12, 0x11, 0xa4, 0x62, 0xb0, 0x9b, 0x12, 0x9d, 0x6c, 0x5e, 0xd0, 0xa2, + 0x1f, 0xa5, 0xa5, 0xfd, 0x40, 0xfa, 0xe5, 0xb5, 0xf4, 0x2c, 0x46, 0x22, 0x48, 0x88, 0x24, 0x75, + 0x2e, 0x05, 0xef, 0xc7, 0x0a, 0x30, 0x89, 0xa4, 0x1c, 0xfc, 0x7e, 0x37, 0x38, 0xaf, 0x86, 0x2d, + 0x0f, 0x4e, 0x6d, 0x01, 0x9c, 0x3b, 0xf3, 0x71, 0x55, 0x02, 0x93, 0x49, 0xd8, 0xcb, 0xf5, 0x4b, + 0x22, 0x51, 0xb5, 0xb8, 0xa9, 0x62, 0x1e, 0xb4, 0x8c, 0x67, 0x0c, 0xbf, 0x41, 0x8c, 0x9d, 0xd3, + 0x95, 0x40, 0x0b, 0xd7, 0x84, 0xb6, 0x79, 0x35, 0xb4, 0x2d, 0x13, 0xda, 0x77, 0x16, 0xb4, 0x76, + 0xd3, 0x78, 0x18, 0xf6, 0xb9, 0xe8, 0xc7, 0xc9, 0xe0, 0x72, 0x50, 0x25, 0x7c, 0x15, 0x13, 0xbe, + 0x6d, 0xb0, 0xfd, 0xb7, 0x49, 0xd6, 0x0a, 0x37, 0x8c, 0x41, 0x6b, 0xa1, 0x56, 0x1c, 0x1d, 0xd9, + 0x03, 0xa8, 0xf8, 0x09, 0x31, 0x37, 0xd7, 0xc4, 0x73, 0x1f, 0x09, 0xaf, 0xf8, 0x89, 0x77, 0x02, + 0xeb, 0xc7, 0x49, 0x30, 0x9a, 0x44, 0x41, 0x2a, 0x10, 0xea, 0x4f, 0xa9, 0x78, 0xc9, 0xef, 0x65, + 0xef, 0x2f, 0x70, 0xbb, 0x10, 0x57, 0xbf, 0x56, 0x48, 0x01, 0x5b, 0xff, 0xea, 0xec, 0xc1, 0xad, + 0xb9, 0xab, 0xdf, 0xfd, 0xa4, 0x1b, 0x2c, 0x06, 0xfd, 0xab, 0x91, 0x17, 0x05, 0xcd, 0x8e, 0x2f, + 0xbb, 0xeb, 0x1e, 0xb8, 0x19, 0x30, 0xf2, 0xc7, 0x7a, 0x76, 0x83, 0x93, 0x50, 0xcc, 0x2e, 0xfb, + 0x3d, 0x43, 0xaf, 0x75, 0x85, 0x7e, 0xe2, 0xd3, 0xda, 0xfb, 0xd5, 0x82, 0xf5, 0xb2, 0x20, 0x9a, + 0x0c, 0x96, 0x41, 0x06, 0xf6, 0x0c, 0xaa, 0x6f, 0x43, 0x31, 0x53, 0xef, 0xb3, 0xb7, 0x50, 0xa2, + 0x85, 0x9b, 0x70, 0xb9, 0x01, 0x3f, 0x85, 0xdd, 0x7e, 0x1a, 0xc6, 0x23, 0x35, 0x8c, 0x4b, 0x09, + 0xcf, 0xd9, 0x8b, 0xe2, 0xfe, 0xe7, 0xf2, 0x67, 0x26, 0x97, 0x42, 0x09, 0xb5, 0xab, 0xd7, 0xa4, + 0xf6, 0x72, 0x19, 0xb5, 0xbd, 0x6f, 0x2d, 0x85, 0x95, 0x31, 0x30, 0x7d, 0xb4, 0x62, 0x9a, 0xd0, + 0xb6, 0x22, 0xb4, 0x2b, 0xa7, 0x3e, 0x3d, 0xdc, 0x2a, 0x11, 0x27, 0x4d, 0x5c, 0xd2, 0x7f, 0x0c, + 0x0e, 0x55, 0x69, 0x2e, 0x7f, 0xa4, 0x8b, 0x2c, 0x26, 0xbb, 0x5c, 0x96, 0xec, 0xde, 0xda, 0xf7, + 0xef, 0x37, 0xad, 0x1f, 0xde, 0x6f, 0x5a, 0x3f, 0xbf, 0xdf, 0xb4, 0xbe, 0xfe, 0x65, 0x73, 0xe9, + 0x74, 0x99, 0xfe, 0x23, 0xfa, 0xdb, 0x6f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x58, 0x5b, 0x7b, 0xe3, + 0x33, 0x12, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -4143,6 +4235,16 @@ func (m *ImportRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.Clear { + i-- + if m.Clear { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x58 + } if m.FieldCreatedAt != 0 { i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) i-- @@ -4272,6 +4374,16 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.Clear { + i-- + if m.Clear { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x60 + } if m.FieldCreatedAt != 0 { i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) i-- @@ -4369,6 +4481,73 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *AtomicRecord) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *AtomicRecord) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *AtomicRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Ir) > 0 { + for iNdEx := len(m.Ir) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Ir[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } + } + if len(m.Ivr) > 0 { + for iNdEx := len(m.Ivr) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Ivr[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + } + if m.Shard != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) + i-- + dAtA[i] = 0x10 + } + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *TranslateKeysRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -5529,6 +5708,9 @@ func (m *ImportRequest) Size() (n int) { if m.FieldCreatedAt != 0 { n += 1 + sovPublic(uint64(m.FieldCreatedAt)) } + if m.Clear { + n += 2 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5587,6 +5769,40 @@ func (m *ImportValueRequest) Size() (n int) { if m.FieldCreatedAt != 0 { n += 1 + sovPublic(uint64(m.FieldCreatedAt)) } + if m.Clear { + n += 2 + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *AtomicRecord) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if m.Shard != 0 { + n += 1 + sovPublic(uint64(m.Shard)) + } + if len(m.Ivr) > 0 { + for _, e := range m.Ivr { + l = e.Size() + n += 1 + l + sovPublic(uint64(l)) + } + } + if len(m.Ir) > 0 { + for _, e := range m.Ir { + l = e.Size() + n += 1 + l + sovPublic(uint64(l)) + } + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -10139,6 +10355,26 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { break } } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Clear", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Clear = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -10584,6 +10820,199 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { break } } + case 12: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Clear", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Clear = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *AtomicRecord) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: AtomicRecord: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: AtomicRecord: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) + } + m.Shard = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Shard |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ivr", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Ivr = append(m.Ivr, &ImportValueRequest{}) + if err := m.Ivr[len(m.Ivr)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ir", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Ir = append(m.Ir, &ImportRequest{}) + if err := m.Ir[len(m.Ir)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/internal/public.proto b/internal/public.proto index 2bace4f67..630392e24 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -175,6 +175,7 @@ message ImportRequest { repeated int64 Timestamps = 6; int64 IndexCreatedAt = 9; int64 FieldCreatedAt = 10; + bool Clear = 11; } message ImportValueRequest { @@ -188,6 +189,14 @@ message ImportValueRequest { repeated string StringValues = 9; int64 IndexCreatedAt = 10; int64 FieldCreatedAt = 11; + bool Clear = 12; +} + +message AtomicRecord { + string Index = 1; + uint64 Shard = 2; + repeated ImportValueRequest Ivr = 3; + repeated ImportRequest Ir = 4; } message TranslateKeysRequest { diff --git a/lmdb_test.go b/lmdb_test.go index 89ba52f11..97b050a38 100644 --- a/lmdb_test.go +++ b/lmdb_test.go @@ -102,11 +102,8 @@ func mustOpenEmptyLMDBWrapper(path string) (w *LMDBWrapper, cleaner func()) { } return w, func() { - w.Close() // stop any started background GC goroutine. - os.RemoveAll(fn) - if FileExists(fn + "-lock") { - os.RemoveAll(fn) - } + w.Close() + panicOn(w.DeleteDBPath(fn)) } } diff --git a/pg/protocol.go b/pg/protocol.go index 57dc2168f..c55decbd7 100644 --- a/pg/protocol.go +++ b/pg/protocol.go @@ -38,7 +38,7 @@ type Protocol uint32 const ( // ProtocolPostgres30 is version 3.0 of the Postgres wire protocol. - ProtocolPostgres30 Protocol = (3 << 16) | 0 + ProtocolPostgres30 Protocol = (3 << 16) // ProtocolCancel is the protocol used for query cancellation. ProtocolCancel Protocol = (1234 << 16) | 5678 diff --git a/row.go b/row.go index ac0ef56e3..9170926a7 100644 --- a/row.go +++ b/row.go @@ -474,7 +474,7 @@ func (r *Row) MarshalJSON() ([]byte, error) { func (r *Row) Columns() []uint64 { a := make([]uint64, 0, r.Count()) for i := range r.segments { - a = append(a, r.segments[i].Columns()...) // Accessing Tx memory that is now invalid. + a = append(a, r.segments[i].Columns()...) } return a } diff --git a/tx_test.go b/tx_test.go new file mode 100644 index 000000000..300e3444b --- /dev/null +++ b/tx_test.go @@ -0,0 +1,245 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa_test + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/http" + "github.com/pilosa/pilosa/v2/server" + "github.com/pilosa/pilosa/v2/test" +) + +func queryIRABit(m0api *pilosa.API, acctOwnerID uint64, iraField string, iraRowID uint64, index string) (bit bool) { + query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID) // acctOwnerID) + res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query}) + panicOn(err) + cols := res.Results[0].(*pilosa.Row).Columns() + for i := range cols { + if cols[i] == acctOwnerID { + return true + } + } + return false +} + +func mustQueryAcct(m0api *pilosa.API, acctOwnerID uint64, fieldAcct0, index string) (acctBal int64) { + query := fmt.Sprintf("FieldValue(field=%v, column=%v)", fieldAcct0, acctOwnerID) + res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query}) + panicOn(err) + + if len(res.Results) == 0 { + return 0 + } + valCount := res.Results[0].(pilosa.ValCount) + return valCount.Val +} + +func queryBalances(m0api *pilosa.API, acctOwnerID uint64, fldAcct0, fldAcct1, index string) (acct0bal, acct1bal int64) { + + acct0bal = mustQueryAcct(m0api, acctOwnerID, fldAcct0, index) + acct1bal = mustQueryAcct(m0api, acctOwnerID, fldAcct1, index) + return +} +func skipForRoaring(t *testing.T) { + if strings.Contains(os.Getenv("PILOSA_TXSRC"), "roaring") { + t.Skip("skip if roaring pseudo-txn involved -- won't show transactional rollback") + } +} + +func TestAPI_ImportAIR(t *testing.T) { + skipForRoaring(t) + c := test.MustRunCluster(t, 1, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID("node0"), + pilosa.OptServerClusterHasher(&offsetModHasher{}), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + )}, + ) + defer c.Close() + + m0 := c[0] + m0api := m0.API + + ctx := context.Background() + index := "i" + + fieldAcct0 := "acct0" + fieldAcct1 := "acct1" + + transferUSD := int64(100) + _ = transferUSD + opts := pilosa.OptFieldTypeInt(-1000, 1000) + + _, err := m0api.CreateIndex(ctx, index, pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = m0api.CreateField(ctx, index, fieldAcct0, opts) + if err != nil { + t.Fatalf("creating fieldAcct0: %v", err) + } + _, err = m0api.CreateField(ctx, index, fieldAcct1, opts) + if err != nil { + t.Fatalf("creating fieldAcct1: %v", err) + } + + iraField := "ira" // set field. + iraRowID := uint64(3) + _, err = m0api.CreateField(ctx, index, iraField) + if err != nil { + t.Fatalf("creating fieldIRA: %v", err) + } + + acctOwnerID := uint64(78) // ColumnID + shard := acctOwnerID / ShardWidth + + // setup 500 USD in acct1 and 700 USD in acct2. + // transfer 100 USD. + // should see 400 USD in acct, and 800 USD in acct2. + // + + // setup initial balances + + createAIRUpdate := func(acct0bal, acct1bal int64) (air *pilosa.AtomicRecord) { + ivr0 := &pilosa.ImportValueRequest{ + Index: index, + Field: fieldAcct0, + Shard: shard, + ColumnIDs: []uint64{acctOwnerID}, + Values: []int64{acct0bal}, + } + ivr1 := &pilosa.ImportValueRequest{ + Index: index, + Field: fieldAcct1, + Shard: shard, + ColumnIDs: []uint64{acctOwnerID}, + Values: []int64{acct1bal}, + } + + ir0 := &pilosa.ImportRequest{ + Index: index, + Field: iraField, + Shard: shard, + ColumnIDs: []uint64{acctOwnerID}, + RowIDs: []uint64{iraRowID}, + } + + air = &pilosa.AtomicRecord{ + Index: index, + Shard: shard, + Ivr: []*pilosa.ImportValueRequest{ + ivr0, ivr1, + }, + Ir: []*pilosa.ImportRequest{ir0}, + } + return + } + + expectedBalStartingAcct0 := int64(500) + expectedBalStartingAcct1 := int64(700) + + air := createAIRUpdate(expectedBalStartingAcct0, expectedBalStartingAcct1) + + if err := m0api.ImportAtomicRecord(ctx, air); err != nil { + t.Fatal(err) + } + + iraBit := queryIRABit(m0api, acctOwnerID, iraField, iraRowID, index) + if !iraBit { + panic("IRA bit should have been set") + } + + startingBalanceAcct0, startingBalanceAcct1 := queryBalances(m0api, acctOwnerID, fieldAcct0, fieldAcct1, index) + //vv("starting balance: acct0=%v, acct1=%v", startingBalanceAcct0, startingBalanceAcct1) + + if startingBalanceAcct0 != expectedBalStartingAcct0 { + panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", expectedBalStartingAcct0, startingBalanceAcct0)) + } + if startingBalanceAcct1 != expectedBalStartingAcct1 { + panic(fmt.Sprintf("expected %v, observed %v starting acct1 balance", expectedBalStartingAcct1, startingBalanceAcct1)) + } + + //vv("sad path: transferUSD %v from %v -> %v, with power loss half-way through", transferUSD, fieldAcct0, fieldAcct1) + + opt := func(o *pilosa.ImportOptions) error { + o.SimPowerLossAfter = 1 + return nil + } + expectedBalEndingAcct0 := expectedBalStartingAcct0 - 100 + expectedBalEndingAcct1 := expectedBalStartingAcct1 + 100 + + air = createAIRUpdate(expectedBalEndingAcct0, expectedBalEndingAcct1) + + err = m0api.ImportAtomicRecord(ctx, air, opt) + if err != pilosa.ErrAborted { + panic(fmt.Sprintf("expected ErrTxnAborted but got err='%#v'", err)) + } + + b0, b1 := queryBalances(m0api, acctOwnerID, fieldAcct0, fieldAcct1, index) + //vv("after power failure tx, balance: acct0=%v, acct1=%v", b0, b1) + + if b0 != expectedBalStartingAcct0 { + panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", expectedBalStartingAcct0, b0)) + } + if b1 != expectedBalStartingAcct1 { + panic(fmt.Sprintf("expected %v, observed %v starting acct1 balance", expectedBalStartingAcct1, b1)) + } + //vv("good: with power loss half-way, no change in account balances; acct0=%v; acct1=%v", b0, b1) + + // next part of the test, just make sure we do the update. + //vv("happy path: transferUSD %v from %v -> %v, with no interruption.", transferUSD, fieldAcct0, fieldAcct1) + + // happy path with no power failure half-way through. + + err = m0api.ImportAtomicRecord(ctx, air) + panicOn(err) + + eb0, eb1 := queryBalances(m0api, acctOwnerID, fieldAcct0, fieldAcct1, index) + + // should have been applied this time. + if eb0 != expectedBalEndingAcct0 || + eb1 != expectedBalEndingAcct1 { + panic(fmt.Sprintf("problem: transaction did not get committed/applied. transferUSD=%v, but we see: startingBalanceAcct0=%v -> endingBalanceAcct0=%v; startingBalanceAcct1=%v -> endingBalanceAcct1=%v", transferUSD, startingBalanceAcct0, eb0, startingBalanceAcct1, eb1)) + } + //vv("ending balance: acct0=%v, acct1=%v", eb0, eb1) + + // clear all the bits + air.Ivr[0].Clear = true + air.Ivr[1].Clear = true + air.Ir[0].Clear = true + + err = m0api.ImportAtomicRecord(ctx, air) + panicOn(err) + + eb0, eb1 = queryBalances(m0api, acctOwnerID, fieldAcct0, fieldAcct1, index) + if eb0 != 0 || + eb1 != 0 { + panic("problem: bits did not clear") + } + //vv("cleared balances: acct0=%v, acct1=%v", eb0, eb1) + + iraBit = queryIRABit(m0api, acctOwnerID, iraField, iraRowID, index) + if iraBit { + panic("IRA bit should have been cleared") + } + +}