From 538768ea9db5d6a352afa533f4e4353e5a14359a Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 23 Sep 2019 14:23:17 -0500 Subject: [PATCH 1/8] handle truncated/damaged .available.shards The available shards file is just a hint to save us a bit of time later; we don't need it to run and it can get updated pretty easily later. If we have problems reading it, we should just report the error, nuke the file, and continue without it. --- field.go | 24 +++++++++++++----- field_internal_test.go | 57 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/field.go b/field.go index dfc098237..341695816 100644 --- a/field.go +++ b/field.go @@ -316,18 +316,30 @@ func (f *Field) mergeRemoteAvailableShards(b *roaring.Bitmap) { // loadAvailableShards reads remoteAvailableShards data for the field, if any. func (f *Field) loadAvailableShards() error { - bm := roaring.NewBitmap() // Read data from meta file. path := filepath.Join(f.path, ".available.shards") buf, err := ioutil.ReadFile(path) + // doesn't exist: this is fine if os.IsNotExist(err) { return nil - } else if err != nil { - return errors.Wrap(err, "reading available shards") - } else { - if err := bm.UnmarshalBinary(buf); err != nil { - return errors.Wrap(err, "unmarshaling") + } + // some other problem: + if err != nil { + f.logger.Printf("available shards file present but unreadable, discarding: %v", err) + err = os.Remove(path) + if err != nil { + return errors.Wrap(err, "deleting corrupt available shards list") } + return nil + } + bm := roaring.NewBitmap() + if err = bm.UnmarshalBinary(buf); err != nil { + f.logger.Printf("available shards file corrupt, discarding: %v", err) + err = os.Remove(path) + if err != nil { + return errors.Wrap(err, "deleting corrupt available shards list") + } + return nil } // Merge bitmap from file into field. f.mergeRemoteAvailableShards(bm) diff --git a/field_internal_test.go b/field_internal_test.go index 3fe3dc83f..41cb5832b 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -19,6 +19,7 @@ import ( "io/ioutil" "math" "os" + "path/filepath" "reflect" "testing" "time" @@ -366,6 +367,62 @@ func TestField_PersistAvailableShards(t *testing.T) { } +func TestField_CorruptAvailableShards(t *testing.T) { + f := MustOpenField(OptFieldTypeDefault()) + + // bm represents remote available shards. + bm := roaring.NewBitmap(1, 2, 3) + + if err := f.AddRemoteAvailableShards(bm); err != nil { + t.Fatal(err) + } + + path := filepath.Join(f.path, ".available.shards") + + avail, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + t.Fatal(err) + } + n, err := avail.Write([]byte{23}) + if err != nil || n != 1 { + t.Fatal(err) + } + avail.Close() + + // Reload field and verify that shard data is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) { + t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice()) + } +} + +func TestField_TruncatedAvailableShards(t *testing.T) { + f := MustOpenField(OptFieldTypeDefault()) + + // bm represents remote available shards. + bm := roaring.NewBitmap(1, 2, 3) + + if err := f.AddRemoteAvailableShards(bm); err != nil { + t.Fatal(err) + } + + path := filepath.Join(f.path, ".available.shards") + + avail, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + t.Fatal(err) + } + avail.Close() + + // Reload field and verify that shard data is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) { + t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice()) + } +} + // Ensure that persisting available shards having a smaller footprint (for example, // when going from a bitmap to a smaller, RLE representation) succeeds. func TestField_PersistAvailableShardsFootprint(t *testing.T) { From 66544660336915373f06d69d261293b737eccc21 Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 24 Sep 2019 12:24:22 -0500 Subject: [PATCH 2/8] partially implement truncation of fragments for corrupt ops log Which is to say don't actually implement it, because openStorage is too messy right now, but this is the rest of the framework, and now I'm going to digress into fixing openStorage. --- fragment.go | 21 +++++++++++++++---- roaring/roaring.go | 40 +++++++++++++++++++++++++++++++++++++ roaring/unmarshal_binary.go | 6 +++--- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/fragment.go b/fragment.go index fa66847b4..85d5cb385 100644 --- a/fragment.go +++ b/fragment.go @@ -406,11 +406,24 @@ func (f *fragment) openStorage(unmarshalData bool) error { // *either* or *both* of old and new storage data might be in use. // So we call the thing that should unconditionally unmap both of them... if err := f.storage.UnmarshalBinary(data); err != nil { - _, e2 := f.storage.RemapRoaringStorage(nil) - if e2 != nil { - return fmt.Errorf("unmarshal storage: file=%s, err=%s, clearing old mapping also failed: %v", f.file.Name(), err, e2) + // roaring can report advisory-only errors... + _, ok := err.(roaring.AdvisoryError) + if !ok { + name := f.file.Name() + f.file.Close() + f.file = nil + _, e2 := f.storage.RemapRoaringStorage(nil) + if e2 != nil { + return fmt.Errorf("unmarshal storage: file=%s, err=%s, clearing old mapping also failed: %v", name, err, e2) + } + return fmt.Errorf("unmarshal storage: file=%s, err=%s", name, err) + } else { + f.Logger.Printf("warning: unmarshal storage, file=%s, err=%v", f.file.Name(), err) + } + trunc, ok := err.(roaring.FileShouldBeTruncatedError) + if ok { + f.Logger.Printf("should probably truncate file %s to %d bytes, but can't yet", f.file.Name(), trunc.SuggestedLength()) } - return fmt.Errorf("unmarshal storage: file=%s, err=%s", f.file.Name(), err) } f.rowCache = &simpleCache{make(map[uint64]*Row)} f.ops, f.opN = f.storage.Ops() diff --git a/roaring/roaring.go b/roaring/roaring.go index 86ee64fc5..046e1f859 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -77,6 +77,46 @@ var containerTypeNames = map[byte]string{ var fullContainer = NewContainerRun([]interval16{{start: 0, last: maxContainerVal}}).Freeze() +// AdvisoryError is used for the special case where we probably want to *report* +// an error reading a file, but don't want to actually count the file as not +// being read. For instance, a partial ops-log entry is *probably* harmless; +// we probably crashed while writing (?) and as such didn't report the write +// as successful. We hope. +type AdvisoryError interface { + error + AdvisoryOnly() +} + +type advisoryError struct { + e error +} + +func (a advisoryError) Error() string { + return a.e.Error() +} + +// This marks the error as safe to ignore. +func (a advisoryError) AdvisoryOnly() { +} + +type FileShouldBeTruncatedError interface { + AdvisoryError + SuggestedLength() int64 +} + +type fileShouldBeTruncatedError struct { + advisoryError + offset int64 +} + +func (f *fileShouldBeTruncatedError) SuggestedLength() int64 { + return f.offset +} + +func newFileShouldBeTruncatedError(err error, offset int64) *fileShouldBeTruncatedError { + return &fileShouldBeTruncatedError{advisoryError: advisoryError{e: err}, offset: offset} +} + type Containers interface { // Get returns nil if the key does not exist. Get(key uint64) *Container diff --git a/roaring/unmarshal_binary.go b/roaring/unmarshal_binary.go index 86f30df0b..c4feb5bd1 100644 --- a/roaring/unmarshal_binary.go +++ b/roaring/unmarshal_binary.go @@ -205,15 +205,15 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { // Unmarshal the op and apply it. var opr op if err := opr.UnmarshalBinary(buf); err != nil { - // FIXME(benbjohnson): return error with position so file can be trimmed. - return err + return newFileShouldBeTruncatedError(err, int64(opsOffset)) } opr.apply(b) // Increase the op count. b.ops++ b.opN += opr.count() + opsOffset += opr.size() // Move the buffer forward. - buf = buf[opr.size():] + buf = buf[opsOffset:] } return nil From b25eb8f5962302418544ab19cce1bf8a7a5afe31 Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 9 Oct 2019 13:25:00 -0500 Subject: [PATCH 3/8] Sources and Generations: tracking mmapped files This code represents an attempt at providing reliable tracking of whether any bitmaps still in use have access to a given block of mmapped data, allowing us to unmap the data when nothing is using it anymore. The basic approach is as follows: Each mmap is associated with a new object, called a "generation". A generation reflects a particular instance of a given file being mapped. When a bitmap is built from an mmapped data source, the bitmap is given a pointer to the generation as its Source. When bitmap operations combine containers from other bitmaps, they produce new bitmaps that are tagged with the combined set of sources. When we snapshot a file, or for some other reason wish to remap it, the corresponding bitmap has all its containers updated to use the new storage, and the bitmap's source is changed. However, previously-handed-out containers might still have references to the old storage. Those containers would be in bitmaps with the old source. After a bunch of study of trying to reference-count and track this, I realized: We don't actually need to do that, because we already have something suitable for determining whether anything can reach a given object. It's the garbage collector. So we set a finalizer on the generation object, which handles unmapping. There's additional sanity-checks here to confirm things like "we thought this generation should be expiring", and we track timestamps. We could also have things check whether a given bitmap's source was marked as obsolete "a while ago", but that isn't implemented yet. There's a debug version of this which tracks finalization, creation, and ending timestamps, and has a call to provide diagnostics for this. Identical generation IDs get separated out with random suffixes in this case -- there's sometimes a second or third instance of the same name due to a holder closing and reopening, but this basically only happens in testing. Note that generations are still used even when there's no mmapping, but unless debugging is turned on, they shouldn't propagate much -- we don't consider a generation to be the source of a bitmap unless the bitmap actually mapped things from that generation's mmapped storage, or debugging is on. There's a couple of other, possibly more subtle, changes and bug fixes that got caught by the testing on this: * If a fragment is partially opened and then opening some later part fails, we close the earlier parts before returning the error so we aren't leaving it partially open. * Several operations on segments which were requesting that a frozen copy of a bitmap be created are now actually *replacing* their bitmap with the frozen bitmap, rather than discarding it. * intersectRunRun, if it decides to create an array or bitmap, will yield that container instead of discarding it. And why all of this? Why, so we can actually implement the thing where when a fragment has a valid roaring bitmap, but the ops log is corrupt, we can truncate the corrupt part of the ops log and reopen it. Which I did. When the generationdebug build tag is in use, every generation has a finalizer all the time. When it's not, they only get finalizers when we expect them to be done -- say, when closing a fragment. This is because finalizers appear to be possibly-expensive. There's some logical cleanup to openStorage here, dividing part of its work into applyStorage and importStorage, which have a common case for handling "there's no data in this file". --- cluster_internal_test.go | 1 + fragment.go | 607 +++++++++++++--------------------- fragment_internal_test.go | 29 +- generation.go | 407 +++++++++++++++++++++++ generation_debug.go | 160 +++++++++ generation_nodebug.go | 37 +++ generation_test.go | 39 +++ holder_test.go | 19 ++ roaring/container_stash.go | 3 - roaring/generation_debug.go | 19 ++ roaring/generation_nodebug.go | 19 ++ roaring/roaring.go | 71 +++- roaring/source.go | 98 ++++++ roaring/unmarshal_binary.go | 6 +- row.go | 10 +- utils_internal_test.go | 3 + view.go | 2 +- 17 files changed, 1141 insertions(+), 389 deletions(-) create mode 100644 generation.go create mode 100644 generation_debug.go create mode 100644 generation_nodebug.go create mode 100644 generation_test.go create mode 100644 roaring/generation_debug.go create mode 100644 roaring/generation_nodebug.go create mode 100644 roaring/source.go diff --git a/cluster_internal_test.go b/cluster_internal_test.go index dad42d31f..fe519ed04 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -158,6 +158,7 @@ func TestFragSources(t *testing.T) { c5.addNodeBasicSorted(node3) idx := newIndexWithTempPath("i") + defer idx.Close() field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault()) if err != nil { t.Fatal(err) diff --git a/fragment.go b/fragment.go index 85d5cb385..852b11389 100644 --- a/fragment.go +++ b/fragment.go @@ -43,7 +43,6 @@ import ( "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/shardwidth" "github.com/pilosa/pilosa/v2/stats" - "github.com/pilosa/pilosa/v2/syswrap" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" ) @@ -110,9 +109,8 @@ type fragment struct { // File-backed storage path string flags byte // user-defined flags passed to roaring - file *os.File + gen generation storage *roaring.Bitmap - storageData []byte totalOpN int64 // total opN values totalOps int64 // total ops (across all snapshots) opN int // number of ops since snapshot (may be approximate for imports) @@ -254,6 +252,10 @@ func (f *fragment) Open() error { // Fill cache with rows persisted to disk. f.Logger.Debugf("open cache for index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard) if err := f.openCache(); err != nil { + e2 := f.closeStorage() + if e2 != nil { + return errors.Wrapf(err, "closing storage: %v, after opening cache", e2) + } return errors.Wrap(err, "opening cache") } @@ -273,48 +275,103 @@ func (f *fragment) Open() error { return nil } -func (f *fragment) reopen() (mustClose bool, err error) { - if f.file == nil { - // Open the data file to be mmap'd and used as an ops log. - f.file, mustClose, err = syswrap.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - return mustClose, fmt.Errorf("open file: %s", err) - } - f.storage.OpWriter = f.file +// emptyStorage is the common case for importStorage/applyStorage where they +// get no data. It tries to write the current storage to the provided file, +// which is assumed to be the file they didn't get any data from. +func (f *fragment) emptyStorage(file *os.File) (bool, error) { + // No data. We'll mark this for no mapping, clear any existing + // mapped containers, and set the Source to nil. We also have no + // ops. + f.opN = 0 + f.ops = 0 + f.storage.SetOps(0, 0) + + f.storage.PreferMapping(false) + _, err := f.storage.RemapRoaringStorage(nil) + f.storage.SetSource(nil) + if err != nil { + return false, fmt.Errorf("applying/importing storage: no data, and clearing old mapping also failed: %v", err) } - return mustClose, nil + // Write the existing storage out to the file so it's + // a valid Roaring file thereafter. nothing to unmarshal. + // In the unlikely event that this happened even though we + // had significant data, we're not mapping it, but that's + // harmless even if it's not maximally efficient. + bi := bufio.NewWriter(file) + if _, err = f.storage.WriteTo(bi); err != nil { + return false, fmt.Errorf("init storage file: %s", err) + } + bi.Flush() + return false, nil } -// openStorage opens the storage bitmap. Usually you also want to read in -// the storage, but in the case where we just wrote that file, such as -// unprotectedWriteToFragment, we could also just... not. If we didn't -// have existing storage, we probably need to unmarshal the data. If the -// file we're asked to open is empty, we probably don't. -// -// If we already had mapped storage previously, we want to unmap that, and -// possibly remap it from the file, but we don't need a full unmarshal, just -// an update of mapped pointers. -// -// unmarshalData is somewhat overloaded. it tells us whether or not we -// need to actually create a bitmap from the data (if the data exists to -// do this from). -// -// usually unmarshalData is only set to false when we're in the middle of -// a snapshot, and unprotectedWriteToFragment just wrote the in-memory data -// out. -// -// If we have existing storage data, and we successfully get new data, -// we will unmap the existing storage data. -// -// This function's design is probably a problem -- it is trying to handle -// both cases where there was existing data before, and cases where we -// just wrote the data. -func (f *fragment) openStorage(unmarshalData bool) error { - oldStorageData := f.storageData - // there's a few places where we might encounter an error, but need - // to continue past it through other error checks, before returning it. - var lastError error +// importStorage attempts to import data from storage -- for instance, +// reading in a roaring bitmap from media. +func (f *fragment) importStorage(data []byte, file *os.File, newGen generation, mapped bool) (bool, error) { + f.storage.PreferMapping(mapped) + if len(data) == 0 { + return f.emptyStorage(file) + } + // UnmarshalBinary will have remapped the storage to newGen if it + // succeeded, or if it fails but the error is advisory-only. So we + // optimistically set the source here, but if there's a non-advisory + // error, we'll unmap it and then set the source to nil. + f.storage.SetSource(newGen) + if err := f.storage.UnmarshalBinary(data); err != nil { + // roaring can report advisory-only errors... + cause := errors.Cause(err) + _, ok := cause.(roaring.AdvisoryError) + if !ok { + _, e2 := f.storage.RemapRoaringStorage(nil) + f.storage.SetSource(nil) + if e2 != nil { + return false, fmt.Errorf("unmarshal storage: file=%s, err=%s, clearing old mapping also failed: %v", file.Name(), err, e2) + } + return false, fmt.Errorf("unmarshal storage: file=%s, err=%s", file.Name(), err) + } + f.Logger.Printf("warning: unmarshal storage, file=%s, err=%v", file.Name(), err) + trunc, ok := cause.(roaring.FileShouldBeTruncatedError) + if ok { + // generation code looks for a FileShouldBeTruncatedError + return false, trunc + } + } + f.ops, f.opN = f.storage.Ops() + // For now, we assume that UnmarshalBinary will have mapped at least + // one container if we told it the storage was mapped and it didn't + // error out. This might be wrong in occasional trivial cases, but + // it should be harmless. + return mapped, nil +} + +// applyStorage applies storage to a fragment that may already have +// usable data. For instance, this would try to remap existing containers +// to use a new storage as backing store. +func (f *fragment) applyStorage(data []byte, file *os.File, newGen generation, mapped bool) (bool, error) { + if len(data) == 0 { + return f.emptyStorage(file) + } + // Tell storage to prefer mapping if and only if we think the data + // is mmapped and valid. + f.storage.PreferMapping(mapped) + f.storage.SetSource(newGen) + // RemapRoaringStorage will fix any mapped containers to point either + // to the provided data (if PreferMapping was called with true and + // data is provided and there's a corresponding container) or to + // allocated storage, so when it's done, there's nothing in it that + // is mapped to anything *other than* the provided data. + return f.storage.RemapRoaringStorage(data) +} + +// openStorage opens the storage bitmap. +// +// This has been massively reworked recently, and now hands a lot of +// file management off to the generation object and the Done method +// of that object. Similarly, the bitmap mapping/remapping +// logic is now mostly in importStorage (reading in a bitmap) and applyStorage +// (remapping an existing bitmap to match a new backing store). +func (f *fragment) openStorage(unmarshalData bool) error { // Create a roaring bitmap to serve as storage for the shard. if f.storage == nil { f.storage = roaring.NewFileBitmap() @@ -323,152 +380,23 @@ func (f *fragment) openStorage(unmarshalData bool) error { // unmarshal this data in order to have any. unmarshalData = true } - // Open the data file to be mmap'd and used as an ops log. - file, mustClose, err := syswrap.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - return fmt.Errorf("open file: %s", err) - } - f.file = file - if mustClose { - defer f.safeClose() - } - - // Lock the underlying file. - if err := syscall.Flock(int(f.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { - return fmt.Errorf("flock: %s", err) - } - - // data is the data we would unmarshal from, if we're unmarshalling; it might - // be obtained by calling ReadAll on a file. - // - // newStorageData is the data we should map things to. it is set only if - // mmapped; if we didn't mmap (say, we couldn't), we won't want to unmap - // the ioutil byte slice. (Theoretically, we shouldn't be using the mapped - // flag in that case...) - var data []byte - var newStorageData []byte - - // If the file is empty then initialize it with an empty bitmap. - fi, err := f.file.Stat() - if err != nil { - return errors.Wrap(err, "statting file before") - } else if fi.Size() == 0 { - bi := bufio.NewWriter(f.file) - var err error - if _, err = f.storage.WriteTo(bi); err != nil { - return fmt.Errorf("init storage file: %s", err) - } - bi.Flush() - _, err = f.file.Stat() - if err != nil { - return errors.Wrap(err, "statting file after") - } - // there's nothing here, we're not going to try to unmarshal it. - unmarshalData = false - f.rowCache = &simpleCache{make(map[uint64]*Row)} - } else { - // Mmap the underlying file so it can be zero copied. - data, err = syswrap.Mmap(int(f.file.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) - if err == syswrap.ErrMaxMapCountReached { - f.Logger.Debugf("maximum number of maps reached, reading file instead") - if unmarshalData { - data, err = ioutil.ReadAll(file) - if err != nil { - return errors.Wrap(err, "failure file readall") - } - } - } else if err != nil { - return errors.Wrap(err, "mmap failed") - } else { - newStorageData = data - } - } - + f.rowCache = &simpleCache{make(map[uint64]*Row)} + var storageOp func([]byte, *os.File, generation, bool) (bool, error) if unmarshalData { - f.storageData = newStorageData - // We're about to either re-read the bitmap, or fail to do so - // and unconditionally unmap the existing stuff. Either way, we - // want to unmap the old storage data after we're done here, but - // we can't unmap it yet because it's still live until sometime - // later, but we can't unmap it later, because we could return - // early... this is what defer is for. - if oldStorageData != nil { - defer func() { - unmapErr := syswrap.Munmap(oldStorageData) - if unmapErr != nil { - f.Logger.Printf("unmap of old storage failed: %s", err) - } - }() - } - // set the preference for mapping based on whether the data's mmapped - f.storage.PreferMapping(newStorageData != nil) - // so we have a problem here: if this fails, it's unclear whether - // *either* or *both* of old and new storage data might be in use. - // So we call the thing that should unconditionally unmap both of them... - if err := f.storage.UnmarshalBinary(data); err != nil { - // roaring can report advisory-only errors... - _, ok := err.(roaring.AdvisoryError) - if !ok { - name := f.file.Name() - f.file.Close() - f.file = nil - _, e2 := f.storage.RemapRoaringStorage(nil) - if e2 != nil { - return fmt.Errorf("unmarshal storage: file=%s, err=%s, clearing old mapping also failed: %v", name, err, e2) - } - return fmt.Errorf("unmarshal storage: file=%s, err=%s", name, err) - } else { - f.Logger.Printf("warning: unmarshal storage, file=%s, err=%v", f.file.Name(), err) - } - trunc, ok := err.(roaring.FileShouldBeTruncatedError) - if ok { - f.Logger.Printf("should probably truncate file %s to %d bytes, but can't yet", f.file.Name(), trunc.SuggestedLength()) - } - } - f.rowCache = &simpleCache{make(map[uint64]*Row)} - f.ops, f.opN = f.storage.Ops() + storageOp = f.importStorage } else { - // we're moving to new storage, so instead of using the OpN - // derived from reading that storage, we notify the bitmap that - // OpN is now effectively zero. - f.opN = 0 - f.ops = 0 - f.storage.SetOps(0, 0) - // if oldStorageData is nil, this just tries to unmap any bits that - // are currently mapped. otherwise, it will point them at this - // storage (if the containers match). - var mappedAny bool - mappedAny, lastError = f.storage.RemapRoaringStorage(newStorageData) - if oldStorageData != nil { - unmapErr := syswrap.Munmap(oldStorageData) - if unmapErr != nil { - f.Logger.Printf("unmap of old storage failed: %s", err) - } - } - if mappedAny { - // Advise the kernel that the mmap is accessed randomly. - if err := madvise(newStorageData, syscall.MADV_RANDOM); err != nil { - lastError = fmt.Errorf("madvise: %s", err) - } - } else { - // if we did map data, but for some reason none of it got used - // as backing store, we can unmap it, and set the slice to nil, - // so we don't keep the now-invalid slice in f.storageData. - if newStorageData != nil { - unmapErr := syswrap.Munmap(newStorageData) - if unmapErr != nil { - lastError = fmt.Errorf("unmapping unused storage data: %s", err) - } - newStorageData = nil - } - } - f.storageData = newStorageData + storageOp = f.applyStorage } - - // Attach the file to the bitmap to act as a write-ahead log. - f.storage.OpWriter = f.file - - return lastError + var err error + f.gen, err = newGeneration(f.gen, f.path, unmarshalData, storageOp, f.Logger) + if generationDebug { + // We might have already done this anyway, if we think we + // mapped stuff, but when debugging we want to do it + // unconditionally, because the test cases otherwise won't + // exercise this code well. + f.storage.SetSource(f.gen) + } + return err } // openCache initializes the cache from row ids persisted to disk. @@ -550,7 +478,7 @@ func (f *fragment) close() error { } // Close underlying storage. - if err := f.closeStorage(true); err != nil { + if err := f.closeStorage(); err != nil { f.Logger.Printf("fragment: error closing storage: err=%s, path=%s", err, f.path) return errors.Wrap(err, "closing storage") } @@ -561,54 +489,17 @@ func (f *fragment) close() error { return nil } -// safeClose is unprotected. -func (f *fragment) safeClose() error { - // Flush file, unlock & close. - if f.file != nil { - if err := f.file.Sync(); err != nil { - return fmt.Errorf("sync: %s", err) - } - if err := syscall.Flock(int(f.file.Fd()), syscall.LOCK_UN); err != nil { - return fmt.Errorf("unlock: %s", err) - } - if err := syswrap.CloseFile(f.file); err != nil { - return fmt.Errorf("close file: %s", err) - } - } - f.file = nil - f.storage.OpWriter = nil - - return nil -} - -// closeStorage attempts to close storage, including unmapping the old -// storage if includeMap is true. This would normally make sense if you're -// expecting to be done using the fragment, or to reload it. But it's also -// okay to just leave stuff mmapped; you don't have to keep the file -// descriptor open. So in some cases, we'll just leave the old mmapping -// in place, rather than regenerating everything from the new file. -func (f *fragment) closeStorage(includeMap bool) error { - // Clear the storage bitmap so it doesn't access the closed mmap. - - //f.storage = roaring.NewBitmap() - - // Unmap the file. - if includeMap && f.storageData != nil { - if err := syswrap.Munmap(f.storageData); err != nil { - return fmt.Errorf("munmap: %s", err) - } - f.storageData = nil - } - - if err := f.safeClose(); err != nil { - return err - } - +// closeStorage marks the current generation as done. It is not necessary +// to call this before openStorage. +func (f *fragment) closeStorage() error { // opN is determined by how many bit set/clear operations are in the storage // write log, so once the storage is closed it should be 0. Opening new // storage will set opN appropriately. f.opN = 0 + if f.gen != nil { + f.gen.Done() + } return nil } @@ -661,22 +552,17 @@ func (f *fragment) rowFromStorage(rowID uint64) *Row { func (f *fragment) setBit(rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - mustClose, err := f.reopen() - if err != nil { - return false, errors.Wrap(err, "reopening") - } - if mustClose { - defer f.safeClose() - } - - // handle mutux field type - if f.mutexVector != nil { - if err := f.handleMutex(rowID, columnID); err != nil { - return changed, errors.Wrap(err, "handling mutex") + err = f.gen.Transaction(&f.storage.OpWriter, func() error { + // handle mutux field type + if f.mutexVector != nil { + if err := f.handleMutex(rowID, columnID); err != nil { + return errors.Wrap(err, "handling mutex") + } } - } - - return f.unprotectedSetBit(rowID, columnID) + changed, err = f.unprotectedSetBit(rowID, columnID) + return err + }) + return changed, err } // handleMutex will clear an existing row and store the new row @@ -740,17 +626,14 @@ func (f *fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err // clearBit clears a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *fragment) clearBit(rowID, columnID uint64) (bool, error) { +func (f *fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - mustClose, err := f.reopen() - if err != nil { - return false, errors.Wrap(err, "reopening") - } - if mustClose { - defer f.safeClose() - } - return f.unprotectedClearBit(rowID, columnID) + err = f.gen.Transaction(&f.storage.OpWriter, func() error { + changed, err = f.unprotectedClearBit(rowID, columnID) + return err + }) + return changed, err } // unprotectedClearBit TODO should be replaced by an invocation of @@ -796,17 +679,14 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er // setRow replaces an existing row (specified by rowID) with the given // Row. This updates both the on-disk storage and the in-cache bitmap. -func (f *fragment) setRow(row *Row, rowID uint64) (bool, error) { +func (f *fragment) setRow(row *Row, rowID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - mustClose, err := f.reopen() - if err != nil { - return false, errors.Wrap(err, "reopening") - } - if mustClose { - defer f.safeClose() - } - return f.unprotectedSetRow(row, rowID) + err = f.gen.Transaction(&f.storage.OpWriter, func() error { + changed, err = f.unprotectedSetRow(row, rowID) + return err + }) + return changed, err } func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err error) { @@ -855,17 +735,14 @@ func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err // ClearRow clears a row for a given rowID within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *fragment) clearRow(rowID uint64) (bool, error) { +func (f *fragment) clearRow(rowID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - mustClose, err := f.reopen() - if err != nil { - return false, errors.Wrap(err, "reopening") - } - if mustClose { - defer f.safeClose() - } - return f.unprotectedClearRow(rowID) + err = f.gen.Transaction(&f.storage.OpWriter, func() error { + changed, err = f.unprotectedClearRow(rowID) + return err + }) + return changed, err } func (f *fragment) unprotectedClearRow(rowID uint64) (changed bool, err error) { @@ -991,67 +868,62 @@ func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64 func (f *fragment) setValueBase(columnID uint64, bitDepth uint, value int64, clear bool) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - mustClose, err := f.reopen() - if err != nil { - return false, errors.Wrap(err, "reopening") - } - if mustClose { - defer f.safeClose() - } + err = f.gen.Transaction(&f.storage.OpWriter, func() error { + // Convert value to an unsigned representation. + uvalue := uint64(value) + if value < 0 { + uvalue = uint64(-value) + } - // Convert value to an unsigned representation. - uvalue := uint64(value) - if value < 0 { - uvalue = uint64(-value) - } + for i := uint(0); i < bitDepth; i++ { + if uvalue&(1<= 0 || clear { + if c, err := f.unprotectedClearBit(uint64(bsiSignBit), columnID); err != nil { + return errors.Wrap(err, "clearing sign") + } else if c { + changed = true + } + } else { + if c, err := f.unprotectedSetBit(uint64(bsiSignBit), columnID); err != nil { + return errors.Wrap(err, "marking sign") + } else if c { + changed = true + } } - } else { - if c, err := f.unprotectedSetBit(uint64(bsiExistsBit), columnID); err != nil { - return changed, errors.Wrap(err, "marking not-null") - } else if c { - changed = true - } - } - // Mark sign bit (or clear). - if value >= 0 || clear { - if c, err := f.unprotectedClearBit(uint64(bsiSignBit), columnID); err != nil { - return changed, errors.Wrap(err, "clearing sign") - } else if c { - changed = true - } - } else { - if c, err := f.unprotectedSetBit(uint64(bsiSignBit), columnID); err != nil { - return changed, errors.Wrap(err, "marking sign") - } else if c { - changed = true - } - } - - return changed, nil + return nil + }) + return changed, err } // importSetValue is a more efficient SetValue just for imports. @@ -2079,52 +1951,46 @@ func (f *fragment) bulkImportStandard(rowIDs, columnIDs []uint64, options *Impor // snapshot of the fragment or just do in-memory updates while appending // operations to the op log. func (f *fragment) importPositions(set, clear []uint64, rowSet map[uint64]struct{}) error { - mustClose, err := f.reopen() - if err != nil { - return errors.Wrap(err, "reopening") - } - if mustClose { - defer f.safeClose() - } - - if len(set) > 0 { - f.stats.Count("ImportingN", int64(len(set)), 1) - changedN, err := f.storage.AddN(set...) // TODO benchmark Add/RemoveN behavior with sorted/unsorted positions - if err != nil { - return errors.Wrap(err, "adding positions") + err := f.gen.Transaction(&f.storage.OpWriter, func() error { + if len(set) > 0 { + f.stats.Count("ImportingN", int64(len(set)), 1) + changedN, err := f.storage.AddN(set...) // TODO benchmark Add/RemoveN behavior with sorted/unsorted positions + if err != nil { + return errors.Wrap(err, "adding positions") + } + f.stats.Count("ImportedN", int64(changedN), 1) + f.incrementOpN(changedN) } - f.stats.Count("ImportedN", int64(changedN), 1) - f.incrementOpN(changedN) - } - if len(clear) > 0 { - f.stats.Count("ClearingN", int64(len(clear)), 1) - changedN, err := f.storage.RemoveN(clear...) - if err != nil { - return errors.Wrap(err, "clearing positions") + if len(clear) > 0 { + f.stats.Count("ClearingN", int64(len(clear)), 1) + changedN, err := f.storage.RemoveN(clear...) + if err != nil { + return errors.Wrap(err, "clearing positions") + } + f.stats.Count("ClearedN", int64(changedN), 1) + f.incrementOpN(changedN) } - f.stats.Count("ClearedN", int64(changedN), 1) - f.incrementOpN(changedN) - } - // Update cache counts for all affected rows. - for rowID := range rowSet { - // Invalidate block checksum. - delete(f.checksums, int(rowID/HashBlockSize)) + // Update cache counts for all affected rows. + for rowID := range rowSet { + // Invalidate block checksum. + delete(f.checksums, int(rowID/HashBlockSize)) + + if f.CacheType != CacheTypeNone { + n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) + f.cache.BulkAdd(rowID, n) + } + + f.rowCache.Add(rowID, nil) + } if f.CacheType != CacheTypeNone { - n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) - f.cache.BulkAdd(rowID, n) + f.cache.Recalculate() } - - f.rowCache.Add(rowID, nil) - } - - if f.CacheType != CacheTypeNone { - f.cache.Recalculate() - } - - return nil + return nil + }) + return err } // bulkImportMutex performs a bulk import on a fragment while ensuring @@ -2210,7 +2076,6 @@ func (f *fragment) importValueSmallWrite(columnIDs []uint64, values []int64, bit } return nil }(); err != nil { - _ = f.closeStorage(true) _ = f.openStorage(true) return err } @@ -2258,7 +2123,6 @@ func (f *fragment) importValue(columnIDs []uint64, values []int64, bitDepth uint } return nil }(); err != nil { - _ = f.closeStorage(true) _ = f.openStorage(true) return err } @@ -2289,7 +2153,13 @@ func (f *fragment) importRoaring(ctx context.Context, data []byte, clear bool) e defer f.mu.Unlock() span.Finish() span, ctx = tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits") - changed, rowSet, err := f.storage.ImportRoaringBits(data, clear, true, rowSize) + var changed int + var rowSet map[uint64]int + err := f.gen.Transaction(&f.storage.OpWriter, func() (err error) { + changed, rowSet, err = f.storage.ImportRoaringBits(data, clear, true, rowSize) + return err + }) + span.Finish() if err != nil { return err @@ -2383,22 +2253,24 @@ func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) (n int64, err e if err != nil { return n, fmt.Errorf("create snapshot file: %s", err) } - defer file.Close() + // No deferred close, because we want to close it sooner than the + // end of this function. // Write storage to snapshot. bw := bufio.NewWriter(file) if n, err = bm.WriteTo(bw); err != nil { + file.Close() return n, fmt.Errorf("snapshot write to: %s", err) } if err := bw.Flush(); err != nil { + file.Close() return n, fmt.Errorf("flush: %s", err) } - // Close current storage. - if err := f.closeStorage(false); err != nil { - return n, fmt.Errorf("close storage: %s", err) - } + // we close the file here so we don't still have it open when trying + // to open it in a moment. + file.Close() // Move snapshot to data file location. if err := os.Rename(snapshotPath, f.path); err != nil { @@ -2598,11 +2470,6 @@ func (f *fragment) readStorageFromArchive(r io.Reader) error { return errors.Wrap(err, "copying") } - // Close current storage. - if err := f.closeStorage(true); err != nil { - return errors.Wrap(err, "closing") - } - // Move snapshot to data file location. if err := os.Rename(path, f.path); err != nil { return errors.Wrap(err, "renaming") diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 1582ecd3c..13d47181b 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1397,6 +1397,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { // Read into another fragment. f1 := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f1.Clean(t) if rn, err := f1.ReadFrom(&buf); err != nil { t.Fatal(err) } else if wn != rn { @@ -2233,7 +2234,20 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { b.Errorf("import error: %v", err) } b.StopTimer() - stat, _ := f.file.Stat() + var stat os.FileInfo + var statTarget io.Writer + err = f.gen.Transaction(&statTarget, func() error { + targetFile, ok := statTarget.(*os.File) + if ok { + stat, _ = targetFile.Stat() + } else { + b.Errorf("couldn't stat file") + } + return nil + }) + if err != nil { + b.Errorf("transaction error: %v", err) + } fileSize[name] = stat.Size() f.Clean(b) } @@ -2382,6 +2396,7 @@ func TestGetZipfRowsSliceRoaring(t *testing.T) { t.Fatalf("suspect distribution from getZipfRowsSliceRoaring") } } + f.Clean(t) } // getZipfRowsSliceRoaring generates a random fragment with the given number of @@ -2529,7 +2544,14 @@ func (f *fragment) sanityCheck(t testing.TB) { func (f *fragment) Clean(t testing.TB) { f.awaitSnapshot() f.sanityCheck(t) + if f.storage != nil && f.storage.Source != nil { + if f.storage.Source.Dead() { + t.Fatalf("cleaning up fragment %s, source %s, source already dead", f.path, f.storage.Source.ID()) + } + } errc := f.Close() + // prevent double-closes of generation during testing. + f.gen = nil errf := os.Remove(f.path) errp := os.Remove(f.cachePath()) if errc != nil || errf != nil { @@ -3272,7 +3294,7 @@ func TestImportClearRestart(t *testing.T) { f2.MaxOpN = maxOpN f2.CacheType = f.CacheType - err = f.closeStorage(true) + err = f.closeStorage() if err != nil { t.Fatalf("closing storage: %v", err) } @@ -3306,7 +3328,7 @@ func TestImportClearRestart(t *testing.T) { f3.MaxOpN = maxOpN f3.CacheType = f.CacheType - err = f2.closeStorage(true) + err = f2.closeStorage() if err != nil { t.Fatalf("f2 closing storage: %v", err) } @@ -3354,6 +3376,7 @@ func check(t *testing.T, f *fragment, exp map[uint64]map[uint64]struct{}) { func TestImportValueConcurrent(t *testing.T) { f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) + defer f.Clean(t) eg := &errgroup.Group{} for i := 0; i < 4; i++ { i := i diff --git a/generation.go b/generation.go new file mode 100644 index 000000000..34bf2b18b --- /dev/null +++ b/generation.go @@ -0,0 +1,407 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "runtime" + "sync" + "syscall" + "time" + + "github.com/pilosa/pilosa/v2/logger" + "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/syswrap" + "github.com/pkg/errors" +) + +// generation represents one "generation" of opening a data file. +// This is what determines when it's safe to unmap a data file, if it +// got mapped, and handles closing/reopening files if we need to +// manage file handle availability. It's an interface because this +// lets us write simpler code for specific cases, rather than handling +// the whole matrix of mapped/unmapped, staying open/being reopened, +// etcetera. +// +// You create a generation by calling newGeneration with a file +// path. If it succeeds in opening that path, it calls a provided +// setup function with the data from the generation, and a flag +// indicating whether the data is mmapped. If the setup function +// fails, newGeneration cleans things up and closes. Otherwise, +// it returns a generation. +// +// The generation itself uses runtime.SetFinalizer to clean up when +// the last reference to it goes away. You should store a pointer +// to the generation in any object which is reliant on the generation. +// +// When you anticipate a generation should be done (for instance, +// opening a new generation), the old one gets marked done, which +// stashes a timestamp in it. Later operations can check whether +// the timestamp is a while back, and if so, complain that something +// might be wrong. +// +// In some cases, we don't have enough open file limit to keep every +// file actually open. To address this, use the `Transaction` function, +// which ensures that the file is open, stores a reference to it in +// a provided `*io.Writer`, and then restores the previous value of +// the io.Writer when it's done. For instance, for a bitmap, this might +// be used with `&b.OpWriter`. +// +// newGeneration takes an optional previous generation; it calls +// that generation's Done function after running the provided setup, +// and bumps the generation count. +type generation interface { + // Transaction runs the given transaction with the generation's + // file open. If the **os.File parameter is + // non-nil, the generation's file will be open, and stored + // into that pointer, during the execution of func, after + // which the previous contents are restored. Otherwise + // the file may or may not be open during the operation. + Transaction(*io.Writer, func() error) error + // Done() should be called exactly once, to indicate that a + // generation is expected not to be in use for long -- for instance, + // when a new generation replaces it. + Done() + // Generation count. + Generation() int64 + // ID indicates the source -- path and generation number -- that + // this generation represents. + ID() string + Dead() bool +} + +type mmapGeneration struct { + mu sync.Mutex // mutex guards modifiers of generation, not of data + transMu sync.Mutex // guards transactions, specifically + path string + id string + file *os.File + data []byte + generation int64 // generation counter + dead bool // we think this generation is dead + deadSince time.Time // when this generation was marked dead + retries int // for cases where we're retrying + logger logger.Logger +} + +func (m *mmapGeneration) Dead() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.dead +} + +func (m *mmapGeneration) ID() string { + return m.id +} + +func (m *mmapGeneration) Generation() int64 { + return m.generation +} + +// Transaction runs an exclusive call, ensuring that the file is open if +// the *io.Writer parameter is present. +func (m *mmapGeneration) Transaction(fileP *io.Writer, fn func() error) (transactionErr error) { + m.transMu.Lock() + defer m.transMu.Unlock() + // HEY LOOK CAREFULLY AT THIS BIT: + // We can't just defer this unlock. We specifically want to be + // sure to unlock the regular mutex *before* this function is over, + // and if we error out trying to open the file, we want to do it + // even sooner. If we deferred this, the transaction would block + // *everything*, including things like sanity checks against the + // generation being Dead(), but also including the deferred + // re-close-the-file. + m.mu.Lock() + // if we've been asked for a file pointer, we need to ensure that + // our file is open, and that the file pointer to it is stored in + // the requested location, then revert that when we're done. + // if we aren't asked for a file pointer, nothing needs the file + // open. + if m.dead { + elapsed := time.Since(m.deadSince) + m.logger.Printf("WARNING: transaction against %s, which has been dead for %v\n", m.id, elapsed) + } + if fileP != nil { + if m.file == nil { + // we ignore the shouldClose response here; if this + // fragment was previously not being kept open, we're + // going to stick with that. + _, err := m.openFile() + if err != nil { + m.mu.Unlock() + return err + } + defer func() { + // report a close error if we have no other error to report + m.mu.Lock() + defer m.mu.Unlock() + err := m.closeFile() + if transactionErr == nil { + transactionErr = err + } + }() + } + var fileStash io.Writer + fileStash, *fileP = *fileP, m.file + defer func() { + *fileP = fileStash + }() + } + // We are done locking the generation itself for now. + m.mu.Unlock() + return fn() +} + +// Done marks the generation done, and closes its file, but may not unmap it. +// It's still conceptually possible to end up doing a Transaction against a +// done generation, but it's a red flag. +func (m *mmapGeneration) Done() { + if m == nil { + return + } + m.mu.Lock() + defer m.mu.Unlock() + if m.dead { + oops := fmt.Sprintf("generation %s, marked done again at %v, previously marked dead at %v", + m.id, time.Now(), m.deadSince) + panic(oops) + } + m.dead = true + m.deadSince = time.Now() + err := m.closeFile() + if err != nil { + m.logger.Printf("error closing generation %s: %v", m.id, err) + } + // If we're not debugging, the finalizer won't have been enabled + // previously. Finalizers have non-zero cost, so having them not be + // created until they're needed seems rewarding? + if !generationDebug { + runtime.SetFinalizer(m, generationFinalizer) + } + endGeneration(m.id) + // note, Done() doesn't close the file; only the finalizer actually + // does the shutdown. +} + +// Try to close the file if it's currently open. +func (m *mmapGeneration) closeFile() error { + var lastErr error + // report the most serious error encountered, but still close + // file even if something else failed. + if m.file != nil { + if err := m.file.Sync(); err != nil { + lastErr = fmt.Errorf("sync: %s", err) + } + if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_UN); err != nil { + lastErr = fmt.Errorf("unlock: %s", err) + } + if err := syswrap.CloseFile(m.file); err != nil { + lastErr = fmt.Errorf("close file: %s", err) + } + m.file = nil + } + return lastErr +} + +// openFile ensures the file is open and locked, or fails. If it does +// open the file, it will also report the "you need to close this file +// when you're done" flag from syswrap. +func (m *mmapGeneration) openFile() (shouldClose bool, err error) { + if m.file != nil { + return false, nil + } + m.file, shouldClose, err = syswrap.OpenFile(m.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + return false, err + } + // do we actually want this in every openFile? I don't know. + if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + m.file.Close() + m.file = nil + return false, fmt.Errorf("flock: %s", err) + } + return shouldClose, nil +} + +func generationFinalizer(m *mmapGeneration) { + m.mu.Lock() + if !m.dead { + m.logger.Printf("finalizing generation %s which isn't dead yet\n", + m.id) + } + m.mu.Unlock() + err := m.closeFile() + if err != nil { + m.logger.Printf("finalizing generation, closing file: %v\n", err) + } + if m.data != nil { + err := syswrap.Munmap(m.data) + if err != nil { + m.logger.Printf("finalizing generation, munmap: %v\n", err) + } + m.data = nil + } + finalizeGeneration(m.id) +} + +// Cancel closes a generation out entirely. It cancels any finalizer, +// unmaps any data, ends generation tracking, and closes any files. +// It does each of these separately whether or not the others need to be done, +// or succeed. It's used to handle failures from newGeneration; it makes sure +// the generation isn't holding any resources and doesn't need to be cleaned +// up otherwise. +// +// Mostly a helper function because there's several cases where newGeneration +// might fail. +func (m *mmapGeneration) Cancel() { + if m.data != nil { + _ = syswrap.Munmap(m.data) + m.data = nil + } + err := m.closeFile() + if err != nil { + m.logger.Printf("error cancelling generation %s: %v", m.id, err) + } + runtime.SetFinalizer(m, nil) + m.dead = true + m.deadSince = time.Now() + cancelGeneration(m.id) +} + +// newGeneration creates a new generation using the given file path. It +// then calls the provided setup function with the allocated storage, a +// file handle, the new generation, and a flag indicatting whether the storage +// is memory-mapped. If the setup function returns a non-nil error, the +// generation is cleaned up, and newGeneration fails. The setup function +// also returns a boolean indicating whether it used the mapping; if it +// didn't, newGeneration discards the mapping and returns a nil generation. +// +// If generationDebug is enabled, we track the generation even if no mapping +// is actually in use, so we can verify that the tracking is working. +// +// On failure, newGeneration returns nil values for generation and func, +// and an error. On success, the func returned is the close func to use +// when the generation is no longer needed by the caller. +func newGeneration(existing generation, path string, readData bool, setup func([]byte, *os.File, generation, bool) (bool, error), logger logger.Logger) (generation, error) { + m := mmapGeneration{path: path, logger: logger} + if existing != nil { + m.generation = existing.Generation() + 1 + // we might keep a previous generation around just for its generation count. + if !existing.Dead() { + defer existing.Done() + } + } + shouldClose, err := m.openFile() + if err != nil { + return nil, err + } + m.id = fmt.Sprintf("%s:%d", m.path, m.generation) + // possibly assign new generation ID if this one's been used, which can + // happen with reopens, especially during testing. + m.id = registerGeneration(m.id) + // if debugging, we always want the finalizer on so we notice if a + // generation is finalized without being closed. for non-debugging + // use, we only need it when the generation is closed. + if generationDebug { + runtime.SetFinalizer(&m, generationFinalizer) + } + // Mmap the underlying file so it can be zero copied. + var mapped bool + var data []byte + fi, err := m.file.Stat() + if err == nil && fi.Size() > 0 { + data, err = syswrap.Mmap(int(m.file.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) + if err == syswrap.ErrMaxMapCountReached { + // I have no idea where/how to display this message. + m.logger.Printf("maximum number of maps reached, reading file '%s' instead", m.path) + } else if err != nil { + m.Cancel() + return nil, errors.Wrap(err, "mmap failed") + } else { + mapped = true + } + } + if data == nil && readData { + data, err = ioutil.ReadAll(m.file) + if err != nil { + m.Cancel() + return nil, errors.Wrap(err, "failure file readall") + } + } + // if we got here, data's the expected data, so let's try to use it + mappedAny, err := setup(data, m.file, &m, mapped) + + // if the setup failed, we unmap data if we previously mapped it, + // and exit. Note that having no data, or having only trivial + // data (like a zero-container Roaring file) isn't "failed". + if err != nil { + m.Cancel() + // Unless, that is, we think the file probably ought to + // be truncated: For instance, if a bitmap has a corrupted + // ops log, we could truncate that part of it and retry. + if err, ok := err.(roaring.FileShouldBeTruncatedError); ok && m.retries < 1 { + m.logger.Printf("file %s read partially, but should-be-truncated at %d bytes\n", m.path, err.SuggestedLength()) + // close this generation, then try again. once. + m.retries++ + err := os.Truncate(m.path, err.SuggestedLength()) + if err != nil { + m.logger.Printf("truncating file failed [but retrying anyway]: %v\n", err) + } + return newGeneration(&m, path, readData, setup, logger) + } + return nil, err + } + + if mapped { + // when generationDebug is on, we want to track this even + // if it's not being used. + if generationDebug || mappedAny { + // Advise the kernel that the mmap is accessed randomly. + // We don't care much about errors with this. + _ = madvise(data, syscall.MADV_RANDOM) + // store the data, so we can unmap it when this generation + // gets finalized. + m.data = data + } else { + // unmap the data and don't stash the pointer in this + // generation. It's not being used. This generation + // doesn't need to exist, yay. + unmapErr := syswrap.Munmap(data) + if unmapErr != nil { + m.logger.Printf("error unmapping (probably harmless): %v", unmapErr) + } + } + } + // shouldClose comes from underlying syswrap.OpenFile, which checks + // a count of open files to hint at us when we need to start closing + // files to preserve open file descriptor limit. + if shouldClose { + err := m.closeFile() + if err != nil { + m.logger.Printf("closing file to preserve open files failed: %v\n", err) + } + } + // It's possible that the generation has no actual data to track, + // because nothing's mapped, in which case there won't be any bitmap + // sources following this, just the fragment source. (Bitmaps won't + // be attached to the source unless they're actually mapped to it, + // or generationDebug is true). That's okay. We pay a tiny cost + // for the finalizer, but we also get higher confidence that it really + // does get cleaned up. + return &m, nil +} diff --git a/generation_debug.go b/generation_debug.go new file mode 100644 index 000000000..a425deabe --- /dev/null +++ b/generation_debug.go @@ -0,0 +1,160 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build generationdebug + +package pilosa + +import ( + "fmt" + "math/rand" + "runtime" + "sort" + "sync" + "time" +) + +const generationDebug = true + +type lifespan struct { + from, to, finalized time.Time +} + +var knownGenerations map[string]lifespan +var knownGenerationLock sync.Mutex + +var timeZero time.Time + +func registerGeneration(id string) string { + knownGenerationLock.Lock() + defer knownGenerationLock.Unlock() + if knownGenerations == nil { + knownGenerations = make(map[string]lifespan) + } + newSpan := lifespan{from: time.Now()} + origId := id + + // if you have more than 65k of the same file open, maybe you have bigger + // problems than this. + for span, exists := knownGenerations[id]; exists; span, exists = knownGenerations[id] { + suffix := fmt.Sprintf("::%04x", rand.Int63n(65536)) + if span.finalized != timeZero { + fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v, finalized %v\n", + id, suffix, span.from, span.to, span.finalized) + } else { + if span.to != timeZero { + fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v\n", id, suffix, span.from, span.to) + } else { + fmt.Printf("new generation %s: adding %s, already exists, created %v", id, suffix, span.from) + } + } + id = origId + suffix + } + fmt.Printf("new generation %s\n", id) + knownGenerations[id] = newSpan + return id +} + +func endGeneration(id string) { + knownGenerationLock.Lock() + defer knownGenerationLock.Unlock() + span, exists := knownGenerations[id] + if !exists { + oops := fmt.Sprintf("ending generation %s: unknown", id) + panic(oops) + } + if span.finalized != timeZero || span.to != timeZero { + oops := fmt.Sprintf("ending generation %s: already died at %v, finalized at %v", id, span.to, span.finalized) + panic(oops) + } + span.to = time.Now() + knownGenerations[id] = span +} + +// cancelGeneration marks the generation as finalized. In principle it's +// only used in cases where we just started a generation but something +// went wrong. it's not fancier than this because of the weird cases +// where the same generation shows up again, such as when closing and +// reopening an index so we don't know about previous instances of the +// same files. +func cancelGeneration(id string) { + knownGenerationLock.Lock() + defer knownGenerationLock.Unlock() + span, exists := knownGenerations[id] + if exists { + span.finalized = time.Now() + span.to = span.finalized + knownGenerations[id] = span + } +} + +func finalizeGeneration(id string) { + knownGenerationLock.Lock() + defer knownGenerationLock.Unlock() + span, exists := knownGenerations[id] + if !exists { + oops := fmt.Sprintf("finalizing generation %s: unknown", id) + panic(oops) + } + if span.finalized != timeZero { + var oops string + if span.to != timeZero { + oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, but not dead", id, span.finalized) + } else { + oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, dead at %v", id, span.finalized, span.to) + } + panic(oops) + } + span.finalized = time.Now() + knownGenerations[id] = span +} + +func reportGenerations() []string { + runtime.GC() + knownGenerationLock.Lock() + defer knownGenerationLock.Unlock() + var surviving []string + times := make([]int64, 0, len(knownGenerations)) + for id, span := range knownGenerations { + if span.to == timeZero { + if span.finalized == timeZero { + surviving = append(surviving, fmt.Sprintf("%s: %v, not ended or finalized", id, span.from)) + } else { + surviving = append(surviving, fmt.Sprintf("%s: %v, finalized %v, not ended", id, span.from, span.finalized)) + } + } else { + if span.finalized == timeZero { + surviving = append(surviving, fmt.Sprintf("%s: %v to %v, not finalized", id, span.from, span.to)) + } else { + times = append(times, int64(span.finalized.Sub(span.to))) + } + } + } + if len(times) > 0 { + sort.Slice(times, func(i, j int) bool { return times[i] < times[j] }) + var total int64 + for _, d := range times { + total += d + } + var mean, median, p90, p99, worst int64 + mean = total / int64(len(times)) + median = times[len(times)/2] + p90 = times[(len(times)*9)/10] + p99 = times[(len(times)*99)/100] + worst = times[len(times)-1] + surviving = append(surviving, fmt.Sprintf("%d finalized spans. lag: mean %v, median %v, p90 %v, p99 %v, worst %v", + len(times), time.Duration(mean), time.Duration(median), time.Duration(p90), time.Duration(p99), time.Duration(worst))) + } + return surviving +} diff --git a/generation_nodebug.go b/generation_nodebug.go new file mode 100644 index 000000000..a2d2dd2f6 --- /dev/null +++ b/generation_nodebug.go @@ -0,0 +1,37 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !generationdebug + +package pilosa + +const generationDebug = false + +func registerGeneration(id string) string { + return id +} + +func endGeneration(id string) { +} + +func cancelGeneration(id string) { +} + +func finalizeGeneration(id string) { +} + +//lint:ignore U1000 this is conditional on a build flag, see generation_test.go. +func reportGenerations() []string { //nolint:unused,deadcode + return nil +} diff --git a/generation_test.go b/generation_test.go new file mode 100644 index 000000000..3dd86fd8f --- /dev/null +++ b/generation_test.go @@ -0,0 +1,39 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// +build generationdebug + +package pilosa + +import ( + "fmt" + "os" + "testing" +) + +func examineResults() { + results := reportGenerations() + if len(results) > 0 { + fmt.Printf("generations:\n") + for _, res := range results { + fmt.Printf(" %s\n", res) + } + } +} + +func TestMain(m *testing.M) { + ret := m.Run() + examineResults() + os.Exit(ret) +} diff --git a/holder_test.go b/holder_test.go index 3f88703e5..09ec03d97 100644 --- a/holder_test.go +++ b/holder_test.go @@ -197,7 +197,26 @@ func TestHolder_Open(t *testing.T) { t.Fatalf("unexpected error: %s", err) } }) + t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) { + h := test.MustOpenHolder() + defer h.Close() + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := field.SetBit(0, 0, nil); err != nil { + t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 20); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err != nil { + t.Fatalf("unexpected error: %s", err) + } + }) } func TestHolder_HasData(t *testing.T) { diff --git a/roaring/container_stash.go b/roaring/container_stash.go index fe03f9d2e..c1dee9fa4 100644 --- a/roaring/container_stash.go +++ b/roaring/container_stash.go @@ -267,9 +267,6 @@ func (c *Container) Freeze() *Container { if c.flags&flagFrozen != 0 { return c } - // unmapOrClone should unmap-in-place because the existing - // container isn't frozen (or we'd already have returned it). - c = c.unmapOrClone() c.flags |= flagFrozen return c } diff --git a/roaring/generation_debug.go b/roaring/generation_debug.go new file mode 100644 index 000000000..e5ac9f7b8 --- /dev/null +++ b/roaring/generation_debug.go @@ -0,0 +1,19 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build generationdebug + +package roaring + +const generationDebug = true diff --git a/roaring/generation_nodebug.go b/roaring/generation_nodebug.go new file mode 100644 index 000000000..4ce3f4ab1 --- /dev/null +++ b/roaring/generation_nodebug.go @@ -0,0 +1,19 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !generationdebug + +package roaring + +const generationDebug = false diff --git a/roaring/roaring.go b/roaring/roaring.go index 046e1f859..2c985358b 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -184,6 +184,7 @@ type ContainerIterator interface { // Bitmap represents a roaring bitmap. type Bitmap struct { Containers Containers + Source Source // User-defined flags. Flags byte @@ -258,6 +259,7 @@ func (b *Bitmap) Freeze() *Bitmap { // Create a copy of the bitmap structure. other := &Bitmap{ Containers: b.Containers.Freeze(), + Source: b.Source, } return other @@ -589,13 +591,21 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap { hi0, hi1 := highbits(start), highbits(end) citer, _ := b.Containers.Iterator(hi0) other := NewSliceBitmap() + mappedAny := false for citer.Next() { k, c := citer.Value() if k >= hi1 { break } + if c.Mapped() { + mappedAny = true + } other.Containers.Put(off+(k-hi0), c.Freeze()) } + // if b.Source != nil && mappedAny { + if b.Source != nil && (generationDebug || mappedAny) { + other.Source = b.Source + } return other } @@ -634,6 +644,7 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { // Intersect returns the intersection of b and other. func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { output := NewBitmap() + usedB, usedOther := false, false iiter, _ := b.Containers.Iterator(0) jiter, _ := other.Containers.Iterator(0) i, j := iiter.Next(), jiter.Next() @@ -647,12 +658,27 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj - output.Containers.Put(ki, intersect(ci, cj)) + newC := intersect(ci, cj) + if newC == ci { + usedB = true + } + if newC == cj { + usedOther = true + } + output.Containers.Put(ki, newC) i, j = iiter.Next(), jiter.Next() ki, ci = iiter.Value() kj, cj = jiter.Value() } } + switch { + case usedB && usedOther: + output.Source = MergeSources(b.Source, other.Source) + case usedB: + output.Source = b.Source + case usedOther: + output.Source = other.Source + } return output } @@ -680,25 +706,43 @@ func (b *Bitmap) UnionInPlace(others ...*Bitmap) { func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { iiter, _ := b.Containers.Iterator(0) jiter, _ := other.Containers.Iterator(0) + usedB, usedOther := false, false i, j := iiter.Next(), jiter.Next() ki, ci := iiter.Value() kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { target.Containers.Put(ki, ci.Freeze()) + usedB = true i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { target.Containers.Put(kj, cj.Freeze()) + usedOther = true j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj - target.Containers.Put(ki, union(ci, cj)) + newC := union(ci, cj) + target.Containers.Put(ki, newC) + if newC == ci { + usedB = true + } + if newC == cj { + usedOther = true + } i, j = iiter.Next(), jiter.Next() ki, ci = iiter.Value() kj, cj = jiter.Value() } } + switch { + case usedB && usedOther: + target.Source = MergeSources(b.Source, other.Source) + case usedB: + target.Source = b.Source + case usedOther: + target.Source = other.Source + } } // unionInPlace stores the union of b and others into b. The others will @@ -792,7 +836,14 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { bitmapIters = make(handledIters, 0, requiredSliceSize) } + var sources []Source + if b.Source != nil { + sources = append(sources, b.Source) + } for _, other := range others { + if other.Source != nil { + sources = append(sources, other.Source) + } otherIter, _ := other.Containers.Iterator(0) if otherIter.Next() { bitmapIters = append(bitmapIters, handledIter{ @@ -802,6 +853,8 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { }) } } + // new bitmap might have containers from any of those bitmaps in it + b.Source = MergeSources(sources...) // Loop until we've exhausted every iter. hasNext := true @@ -930,6 +983,7 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // Difference returns the difference of b and other. func (b *Bitmap) Difference(other *Bitmap) *Bitmap { output := NewBitmap() + output.Source = b.Source iiter, _ := b.Containers.Iterator(0) jiter, _ := other.Containers.Iterator(0) @@ -957,6 +1011,9 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap { // Xor returns the bitwise exclusive or of b and other. func (b *Bitmap) Xor(other *Bitmap) *Bitmap { output := NewBitmap() + // Xor can end up with containers from either parent if the other + // had no container or an empty container. + output.Source = MergeSources(b.Source, other.Source) iiter, _ := b.Containers.Iterator(0) jiter, _ := other.Containers.Iterator(0) @@ -1473,7 +1530,11 @@ func (b *Bitmap) RemapRoaringStorage(data []byte) (mappedAny bool, returnErr err var itrPointer *uint16 var itrErr error - if data != nil { + // If we got no data, we don't want to do the actual mapping, just + // the unmapping. If preferMapping is false, we also don't want to + // map to the data. We still need to do the UpdateEvery loop, we + // just won't have an iterator for it. + if data != nil && b.preferMapping { itr, err = newRoaringIterator(data) } // don't return early: we still have to do the unmapping @@ -3311,9 +3372,9 @@ func intersectRunRun(a, b *Container) *Container { output.setN(n) runs := output.runs() if n < ArrayMaxSize && int32(len(runs)) > n/2 { - output.runToArray() + output = output.runToArray() } else if len(runs) > runMaxSize { - output.runToBitmap() + output = output.runToBitmap() } return output } diff --git a/roaring/source.go b/roaring/source.go new file mode 100644 index 000000000..4cd7ae244 --- /dev/null +++ b/roaring/source.go @@ -0,0 +1,98 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roaring + +import ( + "strings" +) + +// A Source represents the source a given bitmap gets its data from, +// such as a memory-mapped file. When combining bitmaps, we might +// track them together in a single combined-source of some sort. +type Source interface { + ID() string + Dead() bool +} + +// MergeSources combines sources. If you have two bitmaps, and you're +// combining them, then the combination's source is a combination of +// those two sources. +func MergeSources(sources ...Source) Source { + sourceCount := 0 + totalCount := 0 + var lastSource Source + for _, s := range sources { + if s == nil { + continue + } + lastSource = s + if s, ok := s.(combinedSource); ok { + sourceCount++ + totalCount += len(s) + } else { + sourceCount++ + totalCount++ + } + } + // if there's no sources (this includes all sources being + // empty combinedSources), we don't have a source. + if totalCount == 0 { + return nil + } + // if there's exactly one source, combined or otherwise, that's + // fine, we'll just return it. + if sourceCount == 1 { + return lastSource + } + // make a new combinedSource, flattening any combinedSources + // already present. + newSources := make([]Source, 0, totalCount) + for _, s := range sources { + if s == nil { + continue + } + if s, ok := s.(combinedSource); ok { + newSources = append(newSources, s...) + } else { + newSources = append(newSources, s) + } + } + return combinedSource(newSources) +} + +// SetSource tells the bitmap what source to associate with new things it +// creates. This is possibly logically incorrect. +func (b *Bitmap) SetSource(s Source) { + b.Source = s +} + +type combinedSource []Source + +func (c combinedSource) ID() string { + ids := make([]string, len(c)) + for i := range c { + ids[i] = c[i].ID() + } + return strings.Join(ids, ",") +} + +func (c combinedSource) Dead() bool { + for i := range c { + if c[i].Dead() { + return true + } + } + return false +} diff --git a/roaring/unmarshal_binary.go b/roaring/unmarshal_binary.go index c4feb5bd1..e80e834cf 100644 --- a/roaring/unmarshal_binary.go +++ b/roaring/unmarshal_binary.go @@ -30,7 +30,9 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { return nil } statsHit("Bitmap/UnmarshalBinary") - b.opN = 0 // reset opN since we're reading new data. + // reset ops/opN since we're reading new data. + b.ops = 0 + b.opN = 0 fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2])) if fileMagic == MagicNumber { // if pilosa roaring return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring") @@ -213,7 +215,7 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { b.opN += opr.count() opsOffset += opr.size() // Move the buffer forward. - buf = buf[opsOffset:] + buf = data[opsOffset:] } return nil diff --git a/row.go b/row.go index 519416420..601d5101e 100644 --- a/row.go +++ b/row.go @@ -359,7 +359,7 @@ type rowSegment struct { } func (s *rowSegment) Freeze() { - s.data.Freeze() + s.data = s.data.Freeze() } /* @@ -392,7 +392,7 @@ func (s *rowSegment) IntersectionCount(other *rowSegment) uint64 { // Intersect returns the itersection of s and other. func (s *rowSegment) Intersect(other *rowSegment) *rowSegment { data := s.data.Intersect(other.data) - data.Freeze() + data = data.Freeze() return &rowSegment{ data: data, @@ -422,7 +422,7 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment { // Difference returns the diff of s and other. func (s *rowSegment) Difference(other *rowSegment) *rowSegment { data := s.data.Difference(other.data) - data.Freeze() + data = data.Freeze() return &rowSegment{ data: data, @@ -435,7 +435,7 @@ func (s *rowSegment) Difference(other *rowSegment) *rowSegment { // Xor returns the xor of s and other. func (s *rowSegment) Xor(other *rowSegment) *rowSegment { data := s.data.Xor(other.data) - data.Freeze() + data = data.Freeze() return &rowSegment{ data: data, @@ -452,7 +452,7 @@ func (s *rowSegment) Shift() (*rowSegment, error) { if err != nil { return nil, errors.Wrap(err, "shifting roaring data") } - data.Freeze() + data = data.Freeze() return &rowSegment{ data: data, diff --git a/utils_internal_test.go b/utils_internal_test.go index 76e70fb9e..fe8e0d278 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -301,6 +301,9 @@ func (t *ClusterCluster) Close() error { if err != nil { return err } + // Make sure open indexes get shut down too. we wouldn't do + // this normally for a cluster, but we want to for test cases. + c.holder.Close() } return nil } diff --git a/view.go b/view.go index 89484fa3f..69ae1b22f 100644 --- a/view.go +++ b/view.go @@ -483,7 +483,7 @@ func upgradeViewBSIv2(v *view, bitDepth uint) (ok bool, _ error) { if tmpPath, err := upgradeRoaringBSIv2(frag, bitDepth); err != nil { return ok, errors.Wrap(err, "upgrading bsi v2") - } else if err := frag.closeStorage(true); err != nil { + } else if err := frag.closeStorage(); err != nil { return ok, errors.Wrap(err, "closing after bsi v2 upgrade") } else if err := os.Rename(tmpPath, frag.path); err != nil { return ok, errors.Wrap(err, "renaming after bsi v2 upgrade") From 3b696da34a361fd41836b7e1fdc3e54b15911cb2 Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 1 May 2019 16:52:24 -0500 Subject: [PATCH 4/8] plugins and precomputed data So in some cases, when we do a query, the results of one part of the query are innately shared-across-nodes; for instance, a hypothetical Distinct query. More generally, we allow cross-index queries; calls can have "index=foo" in them. This patch lets us handle that without duplicating that query all over. Before we actually start doing the separate calls, we run the query once from the coordinating node, then patch the results in, and send relevant subsets over to each client, etcetera. Also provides slightly friendlier (and I hope faster) support for converting bitmaps to/from sets of rows. We also add an extension interface, and some fancy stuff to let us define new calls, which use this. They're sort of tied together because the first extension I wanted to implement needed precomputed calls. The extension API lets us create extensions using `pkg/plugin` (with all its associated limitations, unfortunately), then query them at load time for functionality. This also implies some revamping of the argument validation for PQL, like verifying that functions exist and knowing things about their argument types. So basically this is an overly intrusive patch, and would be better as separate patches, but they're hard to detangle. add trivial execution-time profiling What if you could ?profile=true on a query and get some numbers back? That'd be really cool. We already have tracing/spans, but right now, those only generate any data if you have something set up for them to trace to. Add a fancy wrapper that lets us generate our own tracing data, and dump it into the request response, if ?profile=true. add a sample extension, add missing features to extension interface Implement a naive probabilistic filter extension as an example of what an extension looks like. In the process, discover multiple omissions in the bitmap API. Well, I did *say* it was experimental. --- api.go | 1 + encoding/proto/proto.go | 65 +- executor.go | 475 +++++- ext/ext.go | 224 +++ ext/samples/.gitignore | 1 + ext/samples/some/some.go | 111 ++ extension.go | 82 + go.mod | 3 +- go.sum | 3 + handler.go | 4 + internal/private.pb.go | 2850 ++++++++++++++-------------------- internal/public.pb.go | 2962 +++++++++++++++++++----------------- internal/public.proto | 10 +- pql/ast.go | 268 +++- pql/parser.go | 8 + pql/parser_test.go | 16 +- pql/pqlpeg_test.go | 20 +- roaring/container_stash.go | 39 + roaring/roaring.go | 175 +++ row.go | 168 ++ server.go | 120 ++ 21 files changed, 4421 insertions(+), 3184 deletions(-) create mode 100644 ext/ext.go create mode 100644 ext/samples/.gitignore create mode 100644 ext/samples/some/some.go create mode 100644 extension.go diff --git a/api.go b/api.go index 68e282916..e028bc7ca 100644 --- a/api.go +++ b/api.go @@ -152,6 +152,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er ExcludeRowAttrs: req.ExcludeRowAttrs, // NOTE: Kept for Pilosa 1.x compat. ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat. ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat. + EmbeddedData: req.EmbeddedData, // precomputed values that needed to be passed with the request } resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) if err != nil { diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index a44440057..62e948c48 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -396,14 +396,19 @@ func encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) *internal.Import } func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest { - return &internal.QueryRequest{ + r := &internal.QueryRequest{ Query: m.Query, Shards: m.Shards, ColumnAttrs: m.ColumnAttrs, Remote: m.Remote, ExcludeRowAttrs: m.ExcludeRowAttrs, ExcludeColumns: m.ExcludeColumns, + EmbeddedData: make([]*internal.Row, len(m.EmbeddedData)), } + for i := range m.EmbeddedData { + r.EmbeddedData[i] = encodeRow(m.EmbeddedData[i]) + } + return r } func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { @@ -416,6 +421,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { pb.Results[i] = &internal.QueryResult{} switch result := m.Results[i].(type) { + case pilosa.SignedRow: + pb.Results[i].Type = queryResultTypeSignedRow + pb.Results[i].SignedRow = encodeSignedRow(result) case *pilosa.Row: pb.Results[i].Type = queryResultTypeRow pb.Results[i].Row = encodeRow(result) @@ -446,7 +454,7 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { case nil: pb.Results[i].Type = queryResultTypeNil default: - panic(fmt.Errorf("unknown type: %d", pb.Results[i].Type)) + panic(fmt.Errorf("unknown type: %T", m.Results[i])) } } @@ -966,6 +974,10 @@ func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) { m.Remote = pb.Remote m.ExcludeRowAttrs = pb.ExcludeRowAttrs m.ExcludeColumns = pb.ExcludeColumns + m.EmbeddedData = make([]*pilosa.Row, len(pb.EmbeddedData)) + for i := range pb.EmbeddedData { + m.EmbeddedData[i] = decodeRow(pb.EmbeddedData[i]) + } } func decodeImportRequest(pb *internal.ImportRequest, m *pilosa.ImportRequest) { @@ -1068,10 +1080,13 @@ const ( queryResultTypeGroupCounts queryResultTypeRowIdentifiers queryResultTypePair + queryResultTypeSignedRow ) func decodeQueryResult(pb *internal.QueryResult) interface{} { switch pb.Type { + case queryResultTypeSignedRow: + return decodeSignedRow(pb.SignedRow) case queryResultTypeRow: return decodeRow(pb.Row) case queryResultTypePairs: @@ -1099,14 +1114,31 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} { // DecodeRow converts r from its internal representation. func decodeRow(pr *internal.Row) *pilosa.Row { if pr == nil { - return nil + return pilosa.NewRow() } - r := pilosa.NewRow() + var r *pilosa.Row + if len(pr.Roaring) > 0 { + r = pilosa.NewRowFromRoaring(pr.Roaring) + } else { + r = pilosa.NewRow() + for _, v := range pr.Columns { + r.SetBit(v) + } + } r.Attrs = decodeAttrs(pr.Attrs) r.Keys = pr.Keys - for _, v := range pr.Columns { - r.SetBit(v) + + return r +} + +func decodeSignedRow(pr *internal.SignedRow) pilosa.SignedRow { + if pr == nil { + return pilosa.SignedRow{} + } + r := pilosa.SignedRow{ + Pos: decodeRow(pr.Pos), + Neg: decodeRow(pr.Neg), } return r } @@ -1213,16 +1245,29 @@ func encodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet { } } +func encodeSignedRow(r pilosa.SignedRow) *internal.SignedRow { + ir := &internal.SignedRow{ + Pos: encodeRow(r.Pos), + Neg: encodeRow(r.Neg), + } + return ir +} + func encodeRow(r *pilosa.Row) *internal.Row { if r == nil { return nil } - return &internal.Row{ - Columns: r.Columns(), - Keys: r.Keys, - Attrs: encodeAttrs(r.Attrs), + ir := &internal.Row{ + Keys: r.Keys, + Attrs: encodeAttrs(r.Attrs), } + if false { + ir.Columns = r.Columns() + } else { + ir.Roaring = r.Roaring() + } + return ir } func encodeRowIdentifiers(r pilosa.RowIdentifiers) *internal.RowIdentifiers { diff --git a/executor.go b/executor.go index 712eafd33..1eb738174 100644 --- a/executor.go +++ b/executor.go @@ -22,7 +22,9 @@ import ( "sync" "time" + "github.com/pilosa/pilosa/v2/ext" "github.com/pilosa/pilosa/v2/pql" + "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/shardwidth" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" @@ -57,6 +59,12 @@ type executor struct { workersWG sync.WaitGroup workerPoolSize int work chan job + // global registry to check for name clashes + additionalOps map[string]*ext.BitmapOp + // typed registries we can use in lookups + additionalBitmapOps map[string]ext.BitmapOpBitmap + additionalCountOps map[string]ext.BitmapOpUnaryCount + additionalFieldOps map[string]ext.BitmapOpBSIBitmap } // executorOption is a functional option type for pilosa.Executor @@ -109,6 +117,38 @@ func (e *executor) Close() error { return nil } +func (e *executor) registerOps(ops []ext.BitmapOp) error { + if e.additionalOps == nil { + e.additionalOps = make(map[string]*ext.BitmapOp) + e.additionalBitmapOps = make(map[string]ext.BitmapOpBitmap) + e.additionalCountOps = make(map[string]ext.BitmapOpUnaryCount) + e.additionalFieldOps = make(map[string]ext.BitmapOpBSIBitmap) + } + for i, op := range ops { + name := op.Name + if _, exists := e.additionalOps[name]; exists { + return fmt.Errorf("op name '%s' already defined", name) + } + e.additionalOps[name] = &ops[i] + typ := ops[i].Func.BitmapOpType() + switch { + case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount: + e.additionalCountOps[name] = ops[i].Func.(ext.BitmapOpUnaryCount) + case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap: + e.additionalBitmapOps[name] = ops[i].Func.(ext.BitmapOpBitmap) + case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap: + if fn, ok := ops[i].Func.(ext.BitmapOpBSIBitmapPrecall); ok { + e.additionalFieldOps[name] = ext.BitmapOpBSIBitmap(fn) + } else { + e.additionalFieldOps[name] = ops[i].Func.(ext.BitmapOpBSIBitmap) + } + default: + return fmt.Errorf("unsupported types for '%s': input type %d, output type %d", name, typ.Input, typ.Output) + } + } + return nil +} + // Execute executes a PQL query. func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute") @@ -237,6 +277,97 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr return ax, nil } +// handlePreCalls traverses the call tree looking for calls that need +// precomputed values. Right now, that's just Distinct. +func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error { + if c.Name == "Precomputed" { + idx := c.Args["valueidx"].(int64) + if idx >= 0 && idx < int64(len(opt.EmbeddedData)) { + row := opt.EmbeddedData[idx] + c.Precomputed = make(map[uint64]interface{}, len(row.segments)) + for _, segment := range row.segments { + c.Precomputed[segment.shard] = &Row{segments: []rowSegment{segment}} + } + } else { + return fmt.Errorf("no precomputed data! index %d, len %d", idx, len(opt.EmbeddedData)) + } + return nil + } + newIndex := c.CallIndex() + // A cross-index query is handled by precall. This is inefficient, + // but we have to do it for now because shards might be different and + // we haven't implemented the local precalls that would be enough + // in some cases. + // + // This makes simple cross-index queries noticably inefficient. + // + // If you're here because of that: We should be using PrecallLocal + // in cases where the call isn't already PrecallGlobal, and + // PrecallLocal should wait until we're running on a specific node + // to do the farming-out of just the sub-queries it has to run + // for its local shards. + // + // As is, we have one node querying every node, then sending out + // all the data to every node, including the data that node already + // has. We could reduce the actual copying around dramatically, + // but only in the cases where local is good enough -- not something + // like Distinct, where you can't predict output shard for a result + // from the shard being queried. + if newIndex != "" && newIndex != index { + c.Type = pql.PrecallGlobal + index = newIndex + } + if c.Type == pql.PrecallNone { + // otherwise, handle the children + return e.handlePreCallChildren(ctx, index, c, shards, opt) + } + // We don't try to handle sub-calls from here. I'm not 100% + // sure that's right, but I think the fact that they're happening + // inside a precomputed call may mean they need different + // handling. In any event, the sub-calls will get handled by + // the executeCall when it gets to them... + + // We set c to look like a normal call, and actually execute it: + c.Type = pql.PrecallNone + // possibly override call index. + v, err := e.executeCall(ctx, index, c, shards, opt) + if err != nil { + return err + } + var row *Row + switch r := v.(type) { + case *Row: + row = r + case SignedRow: + row = r.Pos + default: + return fmt.Errorf("precomputed call %s returned unexpected non-Row data: %T", c.Name, v) + + } + c.Children = []*pql.Call{} + c.Name = "Precomputed" + c.Args = map[string]interface{}{"valueidx": len(opt.EmbeddedData)} + // stash a copy of the full results, which can be forwarded to other + // shards if the query has to go to them + opt.EmbeddedData = append(opt.EmbeddedData, row) + // and stash a copy locally, so local calls can use it + c.Precomputed = make(map[uint64]interface{}, len(row.segments)) + for _, segment := range row.segments { + c.Precomputed[segment.shard] = &Row{segments: []rowSegment{segment}} + } + return nil +} + +// handlePreCallChildren handles any pre-calls in the children of a given call. +func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error { + for i := range c.Children { + if err := e.handlePreCalls(ctx, index, c.Children[i], shards, opt); err != nil { + return err + } + } + return nil +} + func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.execute") defer span.Finish() @@ -270,7 +401,27 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar return nil, err } - v, err := e.executeCall(ctx, index, call, shards, opt) + // If you actually make a top-level Distinct call, you + // want a SignedRow back. Otherwise, it's something else + // that will be using it as a row, and we only care + // about the positive values, because only positive values + // are valid column IDs. So we don't actually eat top-level + // pre calls. + err := e.handlePreCallChildren(ctx, index, call, shards, opt) + if err != nil { + return nil, err + } + var v interface{} + // Top-level calls don't need to precompute cross-index things, + // because we can just pick whatever index we want, but we + // still need to handle them. Since everything else was + // already precomputed by handlePreCallChildren, though, + // we don't need this logic in executeCall. + if newIndex := call.CallIndex(); newIndex != "" { + v, err = e.executeCall(ctx, newIndex, call, shards, opt) + } else { + v, err = e.executeCall(ctx, index, call, shards, opt) + } if err != nil { return nil, err } @@ -299,6 +450,14 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s } // Special handling for mutation and top-n calls. + if op, ok := e.additionalCountOps[c.Name]; ok { + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeGenericCount(ctx, index, c, op, shards, opt) + } + if op, ok := e.additionalFieldOps[c.Name]; ok { + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeGenericField(ctx, index, c, op, shards, opt) + } switch c.Name { case "Sum": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) @@ -343,6 +502,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeOptionsCall(ctx, index, c, shards, opt) case "IncludesColumn": return e.executeIncludesColumnCall(ctx, index, c, shards, opt) + case "Precomputed": + return e.executePrecomputedCall(ctx, index, c, shards, opt) default: e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeBitmapCall(ctx, index, c, shards, opt) @@ -501,6 +662,37 @@ func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, sh return other, nil } +// executeGenericField executes a generic call on a field. Note that in this +// implementation, the operation is always a BSI op. +func (e *executor) executeGenericField(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shards []uint64, opt *execOptions) (SignedRow, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericField") + span.LogKV("name", c.Name) + defer span.Finish() + + if field := c.Args["field"]; field == "" { + return SignedRow{}, fmt.Errorf("plugin operation %s(): field required", c.Name) + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(shard uint64) (interface{}, error) { + return e.executeGenericFieldShard(ctx, index, c, op, shard) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(SignedRow) + return other.union(v.(SignedRow)) + } + + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) + if err != nil { + return SignedRow{}, err + } + other, _ := result.(SignedRow) + + return other, nil +} + // executeMin executes a Min() call. func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMin") @@ -641,6 +833,41 @@ func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call, return e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) } +// executePrecomputedCall pretends to execute a call that we have a precomputed value for. +func (e *executor) executePrecomputedCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executePrecomputedCall") + defer span.Finish() + + // Execute calls in bulk on each remote node and merge. + mapFn := func(shard uint64) (interface{}, error) { + if c.Precomputed != nil { + return c.Precomputed[shard], nil + } + // This might not be an error -- if there were no values, we will not have created + // the corresponding row. + return NewRow(), nil + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(*Row) + if other == nil { + other = NewRow() + } + other.Merge(v.(*Row)) + return other + } + + other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) + if err != nil { + return nil, errors.Wrap(err, "map reduce") + } + + row, _ := other.(*Row) + + return row, nil +} + // executeBitmapCall executes a call that returns a bitmap. func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCall") @@ -719,6 +946,13 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard") defer span.Finish() + if _, ok := e.additionalCountOps[c.Name]; ok { + return nil, fmt.Errorf("count op %s used as bitmap call", c.Name) + } + if op, ok := e.additionalBitmapOps[c.Name]; ok { + return e.executeGenericBitmapShard(ctx, index, c, op, shard) + } + switch c.Name { case "Row", "Range": return e.executeRowShard(ctx, index, c, shard) @@ -734,11 +968,63 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * return e.executeNotShard(ctx, index, c, shard) case "Shift": return e.executeShiftShard(ctx, index, c, shard) + case "Precomputed": + return e.executePrecomputedCallShard(ctx, index, c, shard) default: return nil, fmt.Errorf("unknown call: %s", c.Name) } } +// executeGenericFieldShard executes a generic/extension command on a +// single shard. Note that in this implementation, the op is always +// a BSI op. +func (e *executor) executeGenericFieldShard(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shard uint64) (SignedRow, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericShard") + defer span.Finish() + + var filter *Row + var filterBitmap *roaring.Bitmap + if len(c.Children) == 1 { + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + if err != nil { + return SignedRow{}, errors.Wrap(err, "executing bitmap call") + } + filter = row + if filter != nil && len(filter.segments) > 0 { + filterBitmap = filter.segments[0].data + } + } + + fieldName, _ := c.Args["field"].(string) + + field := e.Holder.Field(index, fieldName) + if field == nil { + return SignedRow{}, nil + } + + bsig := field.bsiGroup(fieldName) + if bsig == nil { + return SignedRow{}, nil + } + + fragment := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) + if fragment == nil { + return SignedRow{}, nil + } + + var out ext.SignedBitmap + if filterBitmap != nil { + out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{WrapBitmap(filterBitmap)}, c.Args) + } else { + out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{}, c.Args) + } + + return SignedRow{ + Neg: NewRowFromBitmap(UnwrapBitmap(out.Neg)), + Pos: NewRowFromBitmap(UnwrapBitmap(out.Pos)), + }, nil +} + // executeSumCountShard calculates the sum and count for bsiGroups on a shard. func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSumCountShard") @@ -1749,6 +2035,44 @@ func (e *executor) executeIntersectShard(ctx context.Context, index string, c *p return other, nil } +// executeGenericBitmapShard executes a generic bitmap call for a local shard. +func (e *executor) executeGenericBitmapShard(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpBitmap, shard uint64) (*Row, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericBitmapShard") + defer span.Finish() + + if op.BitmapOpArity() == ext.OpArityUnary { + if len(c.Children) != 1 { + return nil, fmt.Errorf("%s needs exactly one row parameter", c.Name) + } + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + if err != nil { + return nil, err + } + return row.GenericUnaryOp(op.BitmapOpFunc(), c.Args), nil + } + + var err error + rows := make([]*Row, len(c.Children)) + for i, input := range c.Children { + rows[i], err = e.executeBitmapCallShard(ctx, index, input, shard) + if err != nil { + return nil, err + } + } + var other *Row + switch op.BitmapOpArity() { + case ext.OpArityBinary: + other = rows[0] + for _, row := range rows[1:] { + other = other.GenericBinaryOp(op.BitmapOpFunc(), row, c.Args) + } + case ext.OpArityNary: + other = rows[0].GenericNaryOp(op.BitmapOpFunc(), rows[1:], c.Args) + } + other.invalidateCount() + return other, nil +} + // executeUnionShard executes a union() call for a local shard. func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard") @@ -1793,6 +2117,25 @@ func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Cal return other, nil } +// executePrecomputedCallShard pretends to execute a precomputed call for a local shard. +func (e *executor) executePrecomputedCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + if c.Precomputed != nil { + v := c.Precomputed[shard] + if v == nil { + return NewRow(), nil + } + if r, ok := v.(*Row); ok { + if r != nil { + return r, nil + } else { + return NewRow(), nil + } + } + return nil, fmt.Errorf("precomputed value is not a row: %T", v) + } + return nil, fmt.Errorf("per-shard: missing precomputed values for shard %d", shard) +} + // executeNotShard executes a not() call for a local shard. func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeNotShard") @@ -1849,6 +2192,41 @@ func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.C return row.Shift(n) } +// executeGeneric executes a provided count-like call. +func (e *executor) executeGenericCount(ctx context.Context, index string, c *pql.Call, op ext.BitmapOpUnaryCount, shards []uint64, opt *execOptions) (uint64, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericCount") + defer span.Finish() + + if len(c.Children) == 0 { + return 0, fmt.Errorf("%s() requires an input bitmap", c.Name) + } else if len(c.Children) > 1 { + return 0, fmt.Errorf("%s() only accepts a single bitmap input", c.Name) + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(shard uint64) (interface{}, error) { + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + if err != nil { + return 0, err + } + return row.GenericCount(op, c.Args), nil + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(uint64) + return other + v.(uint64) + } + + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) + if err != nil { + return 0, err + } + n, _ := result.(uint64) + + return n, nil +} + // executeCount executes a count() call. func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount") @@ -1946,7 +2324,7 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq } // Forward call to remote node otherwise. - res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil) + res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil) if err != nil { return false, err } @@ -2222,7 +2600,7 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql. } // Forward call to remote node otherwise. - res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil) + res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil) if err != nil { return false, err } @@ -2257,7 +2635,7 @@ func (e *executor) executeSetValueField(ctx context.Context, index string, c *pq } // Forward call to remote node otherwise. - res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil) + res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil) if err != nil { return false, err } @@ -2311,7 +2689,7 @@ func (e *executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil) + _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil) resp <- err }(node) } @@ -2406,7 +2784,7 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil) + _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil, nil) resp <- err }(node) } @@ -2458,7 +2836,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil) + _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil) resp <- err }(node) } @@ -2474,15 +2852,16 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p } // remoteExec executes a PQL query remotely for a set of shards on a node. -func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64) (results []interface{}, err error) { // nolint: interfacer +func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, embed []*Row) (results []interface{}, err error) { // nolint: interfacer span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeExec") defer span.Finish() // Encode request object. pbreq := &QueryRequest{ - Query: q.String(), - Shards: shards, - Remote: true, + Query: q.String(), + Shards: shards, + Remote: true, + EmbeddedData: embed, } pb, err := e.client.QueryNode(ctx, &node.URI, index, pbreq) @@ -2577,6 +2956,43 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, } } +// makeEmbeddedDataForShards produces new rows containing the rowSegments +// that would correspond to a given set of shards. +func makeEmbeddedDataForShards(allRows []*Row, shards []uint64) []*Row { + if len(allRows) == 0 || len(shards) == 0 { + return nil + } + newRows := make([]*Row, len(allRows)) + for i, row := range allRows { + if row == nil || len(row.segments) == 0 { + continue + } + segments := row.segments + segmentIndex := 0 + newRows[i] = &Row{} + for _, shard := range shards { + for segmentIndex < len(segments) && segments[segmentIndex].shard < shard { + segmentIndex++ + } + // no more segments in this row + if segmentIndex >= len(segments) { + break + } + if segments[segmentIndex].shard == shard { + newRows[i].segments = append(newRows[i].segments, segments[segmentIndex]) + segmentIndex++ + if segmentIndex >= len(segments) { + // no more segments, we're done + break + } + } + // if we got here, segments[segmentIndex].shard exists + // but is greater than the current shard, so we continue. + } + } + return newRows +} + func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapper") defer span.Finish() @@ -2596,7 +3012,11 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod if n.ID == e.Node.ID { resp.result, resp.err = e.mapperLocal(ctx, nodeShards, mapFn, reduceFn) } else if !opt.Remote { - results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeShards) + var embeddedRowsForNode []*Row + if opt.EmbeddedData != nil { + embeddedRowsForNode = makeEmbeddedDataForShards(opt.EmbeddedData, nodeShards) + } + results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeShards, embeddedRowsForNode) if len(results) > 0 { resp.result = results[0] } @@ -3026,6 +3446,7 @@ type execOptions struct { ExcludeRowAttrs bool ExcludeColumns bool ColumnAttrs bool + EmbeddedData []*Row } // hasOnlySetRowAttrs returns true if calls only contains SetRowAttrs() calls. @@ -3060,6 +3481,36 @@ func needsShards(calls []*pql.Call) bool { return false } +// SignedRow represents a signed *Row with two (neg/pos) *Rows. +type SignedRow struct { + Neg *Row `json:"neg"` + Pos *Row `json:"pos"` +} + +func (sr *SignedRow) union(other SignedRow) SignedRow { + ret := SignedRow{&Row{}, &Row{}} + + // merge in sr + if sr != nil { + if sr.Neg != nil { + ret.Neg = ret.Neg.Union(sr.Neg) + } + if sr.Pos != nil { + ret.Pos = ret.Pos.Union(sr.Pos) + } + } + + // merge in other + if other.Neg != nil { + ret.Neg = ret.Neg.Union(other.Neg) + } + if other.Pos != nil { + ret.Pos = ret.Pos.Union(other.Pos) + } + + return ret +} + // ValCount represents a grouping of sum & count for Sum() and Average() calls. type ValCount struct { Val int64 `json:"value"` diff --git a/ext/ext.go b/ext/ext.go new file mode 100644 index 000000000..0d01a9970 --- /dev/null +++ b/ext/ext.go @@ -0,0 +1,224 @@ +// Package ext provides an EXPERIMENTAL AND TEMPORARY interface to use for +// plugin extensions to Pilosa. DO NOT DEVELOP NEW PLUGINS WITH THIS. The +// replacement design is already in process, but it needs more refinement +// to address issues. This one has those issues, and more. +// +// In the current design, plugins will be loaded at runtime using the +// go `plugin` package, so they should be built as a main package using +// the plugin build mode. +// +// Plugins should not import other packages from Pilosa. +// +// To advertise their functionality, plugins define one or more of a +// handful of symbols which will be checked for at plugin load and used +// to register their functionality. +// +// The plugin interface will check for the following function(s). If the +// functions exist, they must have the given signatures. If they return +// a non-nil error, no ops are registered, and the error message will +// be reported in the Pilosa server's logs. +// +// BitmapOps() ([]BitmapOp, error) +// +// These functions may be absent, and may return nil slices; in either +// case, no ops are registered. +package ext + +// The Bitmap type represents a Pilosa bitmap, and is used for bitmap +// operations. +type Bitmap interface { + // AddN and RemoveN can be used to add or remove values from a bitmap. + AddN(a ...uint64) (int, error) + RemoveN(a ...uint64) (int, error) + + // Lookups + Max() uint64 + Min() (uint64, bool) + Count() uint64 + Any() bool + Contains(uint64) bool + Slice() []uint64 + SliceRange(uint64, uint64) []uint64 + // ContainerBits stores the next 1<<16 bits, starting at the provided + // bit index. It may use a provided []uint64 to store them, or may + // provide its own. Don't write to those bits. Offset must be a multiple + // of 1<<16. + ContainerBits(uint64, []uint64) []uint64 + + // These operators provide existing implemented binary ops. + Intersect(Bitmap) Bitmap + Union(Bitmap) Bitmap + IntersectionCount(Bitmap) uint64 + Difference(Bitmap) Bitmap + Xor(Bitmap) Bitmap + Shift(int) (Bitmap, error) + Flip(uint64, uint64) Bitmap + + // New() is an atrocity: it creates a new bitmap, unrelated to the + // existing bitmap. This lets you create a new bitmap without having + // imported any of the packages that have bitmap creation tools, because + // the bitmap wrapper type has to give you one. + New() Bitmap +} + +// SignedBitmap represents a bitmap that can contain both positive and negative +// values. +type SignedBitmap struct { + Pos, Neg Bitmap +} + +// A BitmapOp represents a new bitmap operation that should be exposed +// in PQL. + +type BitmapOpInput byte +type BitmapOpOutput byte +type BitmapOpArity byte +type BitmapOpPrecall byte +type BitmapOpType struct { + Input BitmapOpInput + Arity BitmapOpArity + Output BitmapOpOutput + Precall BitmapOpPrecall +} + +const ( + OpArityUnary = BitmapOpArity(iota) + OpArityBinary + OpArityNary +) + +const ( + // Unary: Exactly one bitmap. + OpInputBitmap = BitmapOpInput(iota) + // The really weird special case used for BSI, where we end up + // needing to do BSI computations. Arguments will be a + // single BitmapBSI, and a []Bitmap for other operands if any. + OpInputNaryBSI +) + +const ( + OpOutputCount = BitmapOpOutput(iota) + OpOutputBitmap + OpOutputSignedBitmap +) + +const ( + OpPrecallNone = BitmapOpPrecall(iota) + OpPrecallGlobal + OpPrecallLocal // unimplemented +) + +// Regardless of arity, non-BSI functions should always take []Bitmap. +type BitmapOpFunc interface { + BitmapOpType() BitmapOpType +} + +// BitmapOpBitmap should actually always be func([]Bitmap) Bitmap, but +// might be different kinds. +type BitmapOpBitmap interface { + BitmapOpArity() BitmapOpArity + BitmapOpFunc() GenericBitmapOpBitmap +} + +// the common underlying type of the other BitmapOpBitmap functions +type GenericBitmapOpBitmap func([]Bitmap, map[string]interface{}) Bitmap + +// BitmapBSI represents the way a single BSI field is passed into a function +// which takes a BSI field. +type BitmapBSI struct { + FieldData Bitmap + ShardWidth uint64 + Offset int64 + Depth uint +} + +type BitmapOpBSIBitmap func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap + +func (b BitmapOpBSIBitmap) BitmapOpType() BitmapOpType { + return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Output: OpOutputSignedBitmap} +} + +type BitmapOpBSIBitmapPrecall func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap + +func (b BitmapOpBSIBitmapPrecall) BitmapOpType() BitmapOpType { + return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Precall: OpPrecallGlobal, Output: OpOutputSignedBitmap} +} + +type BitmapOpUnaryCount func([]Bitmap, map[string]interface{}) int64 + +func (b BitmapOpUnaryCount) BitmapOpType() BitmapOpType { + return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputCount} +} + +type BitmapOpUnaryBitmap func([]Bitmap, map[string]interface{}) Bitmap + +func (b BitmapOpUnaryBitmap) BitmapOpType() BitmapOpType { + return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputBitmap} +} + +func (b BitmapOpUnaryBitmap) BitmapOpArity() BitmapOpArity { + return OpArityUnary +} + +func (b BitmapOpUnaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap { + return GenericBitmapOpBitmap(b) +} + +type BitmapOpBinaryBitmap func([]Bitmap, map[string]interface{}) Bitmap + +func (b BitmapOpBinaryBitmap) BitmapOpType() BitmapOpType { + return BitmapOpType{Input: OpInputBitmap, Arity: OpArityBinary, Output: OpOutputBitmap} +} + +func (b BitmapOpBinaryBitmap) BitmapOpArity() BitmapOpArity { + return OpArityBinary +} + +func (b BitmapOpBinaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap { + return GenericBitmapOpBitmap(b) +} + +type BitmapOpNaryBitmap func([]Bitmap, map[string]interface{}) Bitmap + +func (b BitmapOpNaryBitmap) BitmapOpType() BitmapOpType { + return BitmapOpType{Input: OpInputBitmap, Arity: OpArityNary, Output: OpOutputBitmap} +} + +func (b BitmapOpNaryBitmap) BitmapOpArity() BitmapOpArity { + return OpArityNary +} + +func (b BitmapOpNaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap { + return GenericBitmapOpBitmap(b) +} + +// BitmapOp represents an operation to be supported in PQL. Operations +// on bitmaps should always take []Bitmap. Operations on InputNaryBSI should +// take a []Bitmap, plus a Bitmap/shard-width/offset/depth. +// +// Reserved is a list of words to treat as reserved words in a prototype. +// This is not currently used but might be later, and I want to have the +// concept handy now. +type BitmapOp struct { + Name string + Func BitmapOpFunc + Reserved []string +} + +// ExtensionInfo tells us about the extension. The ExtensionAPI string +// should be "v0". The version is a human-readable version, use something +// that seems meaningful. Name and Description are reasonably self-explanatory, +// I hope. +// +// Extensions should define a function: +// func ExtensionInfo(extensionAPI string) (*ExtensionInfo, error) +// which reports their extension info if they think they can coexist with that +// API string. +type ExtensionInfo struct { + Name string // Extension name. + Description string // Short description. + Version string // Human-readable version info for extension. + ExtensionAPI string // Extension API version. Should be v0 for now. + License string // License info. + BitmapOps []BitmapOp // List of provided ops. +} diff --git a/ext/samples/.gitignore b/ext/samples/.gitignore new file mode 100644 index 000000000..a63fa2c94 --- /dev/null +++ b/ext/samples/.gitignore @@ -0,0 +1 @@ +*/*.so diff --git a/ext/samples/some/some.go b/ext/samples/some/some.go new file mode 100644 index 000000000..421a51bbb --- /dev/null +++ b/ext/samples/some/some.go @@ -0,0 +1,111 @@ +package main + +import ( + "fmt" + "math/bits" + + "github.com/molecula/apophenia" + "github.com/pilosa/pilosa/v2/ext" +) + +// This could be dynamically generated, but for now it's not. +// nolint:unused,deadcode +var extInfoTemplate = &ext.ExtensionInfo{ + Name: "some", + Description: "some of the bits/all of the bits/none of the bits", + Version: "0.01", + ExtensionAPI: "v0", + License: "unreleased", + BitmapOps: []ext.BitmapOp{ + {Name: "Some", Func: ext.BitmapOpUnaryBitmap(Some), Reserved: []string{"p", "seed"}}, + }, +} + +// ExtensionInfo is the entry point used by the plugin code. +func ExtensionInfo(api string) (*ext.ExtensionInfo, error) { // nolint:unused,deadcode + return extInfoTemplate, nil +} + +const batchSize = 1024 + +// Some returns some of the bits from its first input bitmap. Takes seed (int) +// and p (float) values. Seed defaults to 0. +func Some(inputs []ext.Bitmap, args map[string]interface{}) ext.Bitmap { + if len(inputs) == 0 || inputs[0] == nil { + return nil + } + input := inputs[0] + min, ok := input.Min() + // no bits found? + if !ok { + return nil + } + // start at multiple of 128 not greater than min. + min &^= 127 + max := input.Max() + p, ok := args["p"].(float64) + if !ok { + return nil + } + // no bits or impossible probability range + if p <= 0 || p > 1 { + return nil + } + // every bit + if p == 1 { + return inputs[0] + } + // On failure, we default to 0. + seed, _ := args["seed"].(int64) + densityScale := uint64(256) + density := uint64(p * float64(densityScale)) + for density == 0 { + densityScale <<= 1 + density = uint64(p * float64(densityScale)) + // too small + if densityScale > (1 << 32) { + return nil + } + } + w, err := apophenia.NewWeighted(apophenia.NewSequence(seed)) + if err != nil { + return nil + } + someBits := input.New() + toAdd := make([]uint64, batchSize) + toAddN := 0 + offset := apophenia.OffsetFor(apophenia.SequenceWeighted, 0, 0, 0) + for i := min; i < max; i += 128 { + offset.Lo = i + randomBits := w.Bits(offset, density, densityScale) + bit := uint64(0) + for randomBits.Lo != 0 { + next := uint64(bits.TrailingZeros64(randomBits.Lo) + 1) + randomBits.Lo >>= next + toAdd[toAddN] = next + bit + i + toAddN++ + bit += next + } + bit = 64 + for randomBits.Hi != 0 { + next := uint64(bits.TrailingZeros64(randomBits.Hi) + 1) + randomBits.Hi >>= next + toAdd[toAddN] = next + bit + i + toAddN++ + bit += next + } + if toAddN > (batchSize - 128) { + // ignore error + _, _ = someBits.AddN(toAdd[:toAddN]...) + toAddN = 0 + } + } + if toAddN > 0 { + _, _ = someBits.AddN(toAdd[:toAddN]...) + } + return input.Intersect(someBits) +} + +func main() { + fmt.Printf("this is a plugin module only.\n") +} diff --git a/extension.go b/extension.go new file mode 100644 index 000000000..f9758a451 --- /dev/null +++ b/extension.go @@ -0,0 +1,82 @@ +package pilosa + +import ( + "fmt" + + "github.com/pilosa/pilosa/v2/ext" + "github.com/pilosa/pilosa/v2/roaring" +) + +// WrapBitmap yields an extension-Bitmap from a roaring Bitmap. +func WrapBitmap(bm *roaring.Bitmap) ext.Bitmap { + return wrappedBitmap{bm} +} + +// wrappedBitmap is a very shallow glue shim to convert a roaring Bitmap to +// an extension Bitmap. +type wrappedBitmap struct{ *roaring.Bitmap } + +// UnwrapBitmap converts an extension-bitmap to its underlying roaring Bitmap. +func UnwrapBitmap(bm ext.Bitmap) *roaring.Bitmap { + if inner, ok := bm.(wrappedBitmap); ok { + if inner.Bitmap != nil { + return inner.Bitmap + } + return roaring.NewFileBitmap() + } + return roaring.NewFileBitmap() +} + +func (b wrappedBitmap) Intersect(other ext.Bitmap) ext.Bitmap { + return wrappedBitmap{b.Bitmap.Intersect(other.(wrappedBitmap).Bitmap)} +} + +func (b wrappedBitmap) Union(other ext.Bitmap) ext.Bitmap { + return wrappedBitmap{b.Bitmap.Union(other.(wrappedBitmap).Bitmap)} +} + +func (b wrappedBitmap) IntersectionCount(other ext.Bitmap) uint64 { + return b.Bitmap.IntersectionCount(other.(wrappedBitmap).Bitmap) +} + +func (b wrappedBitmap) Difference(other ext.Bitmap) ext.Bitmap { + return wrappedBitmap{b.Bitmap.Difference(other.(wrappedBitmap).Bitmap)} +} + +func (b wrappedBitmap) Xor(other ext.Bitmap) ext.Bitmap { + return wrappedBitmap{b.Bitmap.Xor(other.(wrappedBitmap).Bitmap)} +} + +func (b wrappedBitmap) Shift(n int) (ext.Bitmap, error) { + shifted, err := b.Bitmap.Shift(n) + return wrappedBitmap{shifted}, err +} + +func (b wrappedBitmap) Flip(start, last uint64) ext.Bitmap { + return wrappedBitmap{b.Bitmap.Flip(start, last)} +} + +func (b wrappedBitmap) New() ext.Bitmap { + return WrapBitmap(roaring.NewFileBitmap()) +} + +// ContainerBits tries to get one container's worth of bits. +func (b wrappedBitmap) ContainerBits(offset uint64, target []uint64) (out []uint64) { + // it's an error to call this with a non-container-aligned offset + if offset&0xFFFF != 0 { + return nil + } + if b.Bitmap == nil { + fmt.Printf("ContainerBits on bitmap with no contents\n") + return nil + } + if b.Bitmap.Containers == nil { + fmt.Printf("ContainerBits on bitmap with nil Containers\n") + return nil + } + c := b.Bitmap.Containers.Get(offset >> 16) + if c == nil { + return nil + } + return c.AsBitmap(target) +} diff --git a/go.mod b/go.mod index e546f530e..ef21e563a 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/gorilla/mux v1.7.0 github.com/hashicorp/memberlist v0.1.3 github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b github.com/opentracing/opentracing-go v1.1.0 github.com/pelletier/go-toml v1.2.0 github.com/pkg/errors v0.8.1 @@ -44,4 +45,4 @@ require ( modernc.org/strutil v1.0.0 ) -go 1.11 +go 1.13 diff --git a/go.sum b/go.sum index 68186bbec..dd930d309 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y= +github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -96,6 +98,7 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/handler.go b/handler.go index 810043171..e99dbed69 100644 --- a/handler.go +++ b/handler.go @@ -48,6 +48,10 @@ type QueryRequest struct { // Should we profile this query? Profile bool + + // Additional data associated with the query, in cases where there's + // row-style inputs for precomputed values. + EmbeddedData []*Row } // QueryResponse represent a response from a processed query. diff --git a/internal/private.pb.go b/internal/private.pb.go index c941e4dc1..755370e74 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -3,13 +3,11 @@ package internal -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - io "io" - math "math" - math_bits "math/bits" -) +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +import io "io" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -20,7 +18,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` @@ -34,7 +32,7 @@ func (m *IndexMeta) Reset() { *m = IndexMeta{} } func (m *IndexMeta) String() string { return proto.CompactTextString(m) } func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{0} + return fileDescriptor_private_b229d027a4642df7, []int{0} } func (m *IndexMeta) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -44,15 +42,15 @@ func (m *IndexMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_IndexMeta.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *IndexMeta) XXX_Merge(src proto.Message) { - xxx_messageInfo_IndexMeta.Merge(m, src) +func (dst *IndexMeta) XXX_Merge(src proto.Message) { + xxx_messageInfo_IndexMeta.Merge(dst, src) } func (m *IndexMeta) XXX_Size() int { return m.Size() @@ -98,7 +96,7 @@ func (m *FieldOptions) Reset() { *m = FieldOptions{} } func (m *FieldOptions) String() string { return proto.CompactTextString(m) } func (*FieldOptions) ProtoMessage() {} func (*FieldOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{1} + return fileDescriptor_private_b229d027a4642df7, []int{1} } func (m *FieldOptions) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -108,15 +106,15 @@ func (m *FieldOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_FieldOptions.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *FieldOptions) XXX_Merge(src proto.Message) { - xxx_messageInfo_FieldOptions.Merge(m, src) +func (dst *FieldOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_FieldOptions.Merge(dst, src) } func (m *FieldOptions) XXX_Size() int { return m.Size() @@ -215,7 +213,7 @@ func (m *ImportResponse) Reset() { *m = ImportResponse{} } func (m *ImportResponse) String() string { return proto.CompactTextString(m) } func (*ImportResponse) ProtoMessage() {} func (*ImportResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{2} + return fileDescriptor_private_b229d027a4642df7, []int{2} } func (m *ImportResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -225,15 +223,15 @@ func (m *ImportResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, erro return xxx_messageInfo_ImportResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *ImportResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportResponse.Merge(m, src) +func (dst *ImportResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportResponse.Merge(dst, src) } func (m *ImportResponse) XXX_Size() int { return m.Size() @@ -266,7 +264,7 @@ func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } func (*BlockDataRequest) ProtoMessage() {} func (*BlockDataRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{3} + return fileDescriptor_private_b229d027a4642df7, []int{3} } func (m *BlockDataRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -276,15 +274,15 @@ func (m *BlockDataRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, er return xxx_messageInfo_BlockDataRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *BlockDataRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_BlockDataRequest.Merge(m, src) +func (dst *BlockDataRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_BlockDataRequest.Merge(dst, src) } func (m *BlockDataRequest) XXX_Size() int { return m.Size() @@ -331,8 +329,8 @@ func (m *BlockDataRequest) GetBlock() uint64 { } type BlockDataResponse struct { - RowIDs []uint64 `protobuf:"varint,1,rep,packed,name=RowIDs,proto3" json:"RowIDs,omitempty"` - ColumnIDs []uint64 `protobuf:"varint,2,rep,packed,name=ColumnIDs,proto3" json:"ColumnIDs,omitempty"` + RowIDs []uint64 `protobuf:"varint,1,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,2,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -342,7 +340,7 @@ func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } func (*BlockDataResponse) ProtoMessage() {} func (*BlockDataResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{4} + return fileDescriptor_private_b229d027a4642df7, []int{4} } func (m *BlockDataResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -352,15 +350,15 @@ func (m *BlockDataResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, e return xxx_messageInfo_BlockDataResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *BlockDataResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_BlockDataResponse.Merge(m, src) +func (dst *BlockDataResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_BlockDataResponse.Merge(dst, src) } func (m *BlockDataResponse) XXX_Size() int { return m.Size() @@ -386,7 +384,7 @@ func (m *BlockDataResponse) GetColumnIDs() []uint64 { } type Cache struct { - IDs []uint64 `protobuf:"varint,1,rep,packed,name=IDs,proto3" json:"IDs,omitempty"` + IDs []uint64 `protobuf:"varint,1,rep,packed,name=IDs" json:"IDs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -396,7 +394,7 @@ func (m *Cache) Reset() { *m = Cache{} } func (m *Cache) String() string { return proto.CompactTextString(m) } func (*Cache) ProtoMessage() {} func (*Cache) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{5} + return fileDescriptor_private_b229d027a4642df7, []int{5} } func (m *Cache) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -406,15 +404,15 @@ func (m *Cache) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Cache.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *Cache) XXX_Merge(src proto.Message) { - xxx_messageInfo_Cache.Merge(m, src) +func (dst *Cache) XXX_Merge(src proto.Message) { + xxx_messageInfo_Cache.Merge(dst, src) } func (m *Cache) XXX_Size() int { return m.Size() @@ -433,7 +431,7 @@ func (m *Cache) GetIDs() []uint64 { } type MaxShards struct { - Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard,proto3" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -443,7 +441,7 @@ func (m *MaxShards) Reset() { *m = MaxShards{} } func (m *MaxShards) String() string { return proto.CompactTextString(m) } func (*MaxShards) ProtoMessage() {} func (*MaxShards) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{6} + return fileDescriptor_private_b229d027a4642df7, []int{6} } func (m *MaxShards) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -453,15 +451,15 @@ func (m *MaxShards) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_MaxShards.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *MaxShards) XXX_Merge(src proto.Message) { - xxx_messageInfo_MaxShards.Merge(m, src) +func (dst *MaxShards) XXX_Merge(src proto.Message) { + xxx_messageInfo_MaxShards.Merge(dst, src) } func (m *MaxShards) XXX_Size() int { return m.Size() @@ -492,7 +490,7 @@ func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } func (*CreateShardMessage) ProtoMessage() {} func (*CreateShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{7} + return fileDescriptor_private_b229d027a4642df7, []int{7} } func (m *CreateShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -502,15 +500,15 @@ func (m *CreateShardMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, return xxx_messageInfo_CreateShardMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *CreateShardMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateShardMessage.Merge(m, src) +func (dst *CreateShardMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateShardMessage.Merge(dst, src) } func (m *CreateShardMessage) XXX_Size() int { return m.Size() @@ -553,7 +551,7 @@ func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} } func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) } func (*DeleteIndexMessage) ProtoMessage() {} func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{8} + return fileDescriptor_private_b229d027a4642df7, []int{8} } func (m *DeleteIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -563,15 +561,15 @@ func (m *DeleteIndexMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, return xxx_messageInfo_DeleteIndexMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *DeleteIndexMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteIndexMessage.Merge(m, src) +func (dst *DeleteIndexMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteIndexMessage.Merge(dst, src) } func (m *DeleteIndexMessage) XXX_Size() int { return m.Size() @@ -591,7 +589,7 @@ func (m *DeleteIndexMessage) GetIndex() string { type CreateIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` + Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -601,7 +599,7 @@ func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} } func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) } func (*CreateIndexMessage) ProtoMessage() {} func (*CreateIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{9} + return fileDescriptor_private_b229d027a4642df7, []int{9} } func (m *CreateIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -611,15 +609,15 @@ func (m *CreateIndexMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, return xxx_messageInfo_CreateIndexMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *CreateIndexMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateIndexMessage.Merge(m, src) +func (dst *CreateIndexMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateIndexMessage.Merge(dst, src) } func (m *CreateIndexMessage) XXX_Size() int { return m.Size() @@ -647,7 +645,7 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { type CreateFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta,proto3" json:"Meta,omitempty"` + Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -657,7 +655,7 @@ func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } func (*CreateFieldMessage) ProtoMessage() {} func (*CreateFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{10} + return fileDescriptor_private_b229d027a4642df7, []int{10} } func (m *CreateFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -667,15 +665,15 @@ func (m *CreateFieldMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, return xxx_messageInfo_CreateFieldMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *CreateFieldMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateFieldMessage.Merge(m, src) +func (dst *CreateFieldMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateFieldMessage.Merge(dst, src) } func (m *CreateFieldMessage) XXX_Size() int { return m.Size() @@ -719,7 +717,7 @@ func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } func (*DeleteFieldMessage) ProtoMessage() {} func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{11} + return fileDescriptor_private_b229d027a4642df7, []int{11} } func (m *DeleteFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -729,15 +727,15 @@ func (m *DeleteFieldMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, return xxx_messageInfo_DeleteFieldMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *DeleteFieldMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteFieldMessage.Merge(m, src) +func (dst *DeleteFieldMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteFieldMessage.Merge(dst, src) } func (m *DeleteFieldMessage) XXX_Size() int { return m.Size() @@ -775,7 +773,7 @@ func (m *DeleteAvailableShardMessage) Reset() { *m = DeleteAvailableShar func (m *DeleteAvailableShardMessage) String() string { return proto.CompactTextString(m) } func (*DeleteAvailableShardMessage) ProtoMessage() {} func (*DeleteAvailableShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{12} + return fileDescriptor_private_b229d027a4642df7, []int{12} } func (m *DeleteAvailableShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -785,15 +783,15 @@ func (m *DeleteAvailableShardMessage) XXX_Marshal(b []byte, deterministic bool) return xxx_messageInfo_DeleteAvailableShardMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *DeleteAvailableShardMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteAvailableShardMessage.Merge(m, src) +func (dst *DeleteAvailableShardMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteAvailableShardMessage.Merge(dst, src) } func (m *DeleteAvailableShardMessage) XXX_Size() int { return m.Size() @@ -827,8 +825,8 @@ func (m *DeleteAvailableShardMessage) GetShardID() uint64 { type Field struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` - Views []string `protobuf:"bytes,3,rep,name=Views,proto3" json:"Views,omitempty"` + Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -838,7 +836,7 @@ func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{13} + return fileDescriptor_private_b229d027a4642df7, []int{13} } func (m *Field) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -848,15 +846,15 @@ func (m *Field) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Field.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *Field) XXX_Merge(src proto.Message) { - xxx_messageInfo_Field.Merge(m, src) +func (dst *Field) XXX_Merge(src proto.Message) { + xxx_messageInfo_Field.Merge(dst, src) } func (m *Field) XXX_Size() int { return m.Size() @@ -889,7 +887,7 @@ func (m *Field) GetViews() []string { } type Schema struct { - Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes,proto3" json:"Indexes,omitempty"` + Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes" json:"Indexes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -899,7 +897,7 @@ func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{14} + return fileDescriptor_private_b229d027a4642df7, []int{14} } func (m *Schema) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -909,15 +907,15 @@ func (m *Schema) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Schema.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *Schema) XXX_Merge(src proto.Message) { - xxx_messageInfo_Schema.Merge(m, src) +func (dst *Schema) XXX_Merge(src proto.Message) { + xxx_messageInfo_Schema.Merge(dst, src) } func (m *Schema) XXX_Size() int { return m.Size() @@ -937,7 +935,7 @@ func (m *Schema) GetIndexes() []*Index { type Index struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Fields []*Field `protobuf:"bytes,4,rep,name=Fields,proto3" json:"Fields,omitempty"` + Fields []*Field `protobuf:"bytes,4,rep,name=Fields" json:"Fields,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -947,7 +945,7 @@ func (m *Index) Reset() { *m = Index{} } func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} func (*Index) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{15} + return fileDescriptor_private_b229d027a4642df7, []int{15} } func (m *Index) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -957,15 +955,15 @@ func (m *Index) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Index.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *Index) XXX_Merge(src proto.Message) { - xxx_messageInfo_Index.Merge(m, src) +func (dst *Index) XXX_Merge(src proto.Message) { + xxx_messageInfo_Index.Merge(dst, src) } func (m *Index) XXX_Size() int { return m.Size() @@ -1003,7 +1001,7 @@ func (m *URI) Reset() { *m = URI{} } func (m *URI) String() string { return proto.CompactTextString(m) } func (*URI) ProtoMessage() {} func (*URI) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{16} + return fileDescriptor_private_b229d027a4642df7, []int{16} } func (m *URI) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1013,15 +1011,15 @@ func (m *URI) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_URI.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *URI) XXX_Merge(src proto.Message) { - xxx_messageInfo_URI.Merge(m, src) +func (dst *URI) XXX_Merge(src proto.Message) { + xxx_messageInfo_URI.Merge(dst, src) } func (m *URI) XXX_Size() int { return m.Size() @@ -1055,7 +1053,7 @@ func (m *URI) GetPort() uint32 { type Node struct { ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` - URI *URI `protobuf:"bytes,2,opt,name=URI,proto3" json:"URI,omitempty"` + URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` IsCoordinator bool `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"` State string `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1067,7 +1065,7 @@ func (m *Node) Reset() { *m = Node{} } func (m *Node) String() string { return proto.CompactTextString(m) } func (*Node) ProtoMessage() {} func (*Node) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{17} + return fileDescriptor_private_b229d027a4642df7, []int{17} } func (m *Node) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1077,15 +1075,15 @@ func (m *Node) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Node.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *Node) XXX_Merge(src proto.Message) { - xxx_messageInfo_Node.Merge(m, src) +func (dst *Node) XXX_Merge(src proto.Message) { + xxx_messageInfo_Node.Merge(dst, src) } func (m *Node) XXX_Size() int { return m.Size() @@ -1136,7 +1134,7 @@ func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } func (*NodeStateMessage) ProtoMessage() {} func (*NodeStateMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{18} + return fileDescriptor_private_b229d027a4642df7, []int{18} } func (m *NodeStateMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1146,15 +1144,15 @@ func (m *NodeStateMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, er return xxx_messageInfo_NodeStateMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *NodeStateMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeStateMessage.Merge(m, src) +func (dst *NodeStateMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeStateMessage.Merge(dst, src) } func (m *NodeStateMessage) XXX_Size() int { return m.Size() @@ -1181,7 +1179,7 @@ func (m *NodeStateMessage) GetState() string { type NodeEventMessage struct { Event uint32 `protobuf:"varint,1,opt,name=Event,proto3" json:"Event,omitempty"` - Node *Node `protobuf:"bytes,2,opt,name=Node,proto3" json:"Node,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1191,7 +1189,7 @@ func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } func (*NodeEventMessage) ProtoMessage() {} func (*NodeEventMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{19} + return fileDescriptor_private_b229d027a4642df7, []int{19} } func (m *NodeEventMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1201,15 +1199,15 @@ func (m *NodeEventMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, er return xxx_messageInfo_NodeEventMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *NodeEventMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeEventMessage.Merge(m, src) +func (dst *NodeEventMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeEventMessage.Merge(dst, src) } func (m *NodeEventMessage) XXX_Size() int { return m.Size() @@ -1235,9 +1233,9 @@ func (m *NodeEventMessage) GetNode() *Node { } type NodeStatus struct { - Node *Node `protobuf:"bytes,1,opt,name=Node,proto3" json:"Node,omitempty"` - Schema *Schema `protobuf:"bytes,3,opt,name=Schema,proto3" json:"Schema,omitempty"` - Indexes []*IndexStatus `protobuf:"bytes,4,rep,name=Indexes,proto3" json:"Indexes,omitempty"` + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` + Schema *Schema `protobuf:"bytes,3,opt,name=Schema" json:"Schema,omitempty"` + Indexes []*IndexStatus `protobuf:"bytes,4,rep,name=Indexes" json:"Indexes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1247,7 +1245,7 @@ func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} func (*NodeStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{20} + return fileDescriptor_private_b229d027a4642df7, []int{20} } func (m *NodeStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1257,15 +1255,15 @@ func (m *NodeStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeStatus.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *NodeStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeStatus.Merge(m, src) +func (dst *NodeStatus) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeStatus.Merge(dst, src) } func (m *NodeStatus) XXX_Size() int { return m.Size() @@ -1299,7 +1297,7 @@ func (m *NodeStatus) GetIndexes() []*IndexStatus { type IndexStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields,proto3" json:"Fields,omitempty"` + Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields" json:"Fields,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1309,7 +1307,7 @@ func (m *IndexStatus) Reset() { *m = IndexStatus{} } func (m *IndexStatus) String() string { return proto.CompactTextString(m) } func (*IndexStatus) ProtoMessage() {} func (*IndexStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{21} + return fileDescriptor_private_b229d027a4642df7, []int{21} } func (m *IndexStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1319,15 +1317,15 @@ func (m *IndexStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_IndexStatus.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *IndexStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_IndexStatus.Merge(m, src) +func (dst *IndexStatus) XXX_Merge(src proto.Message) { + xxx_messageInfo_IndexStatus.Merge(dst, src) } func (m *IndexStatus) XXX_Size() int { return m.Size() @@ -1354,7 +1352,7 @@ func (m *IndexStatus) GetFields() []*FieldStatus { type FieldStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards,proto3" json:"AvailableShards,omitempty"` + AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards" json:"AvailableShards,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1364,7 +1362,7 @@ func (m *FieldStatus) Reset() { *m = FieldStatus{} } func (m *FieldStatus) String() string { return proto.CompactTextString(m) } func (*FieldStatus) ProtoMessage() {} func (*FieldStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{22} + return fileDescriptor_private_b229d027a4642df7, []int{22} } func (m *FieldStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1374,15 +1372,15 @@ func (m *FieldStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_FieldStatus.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *FieldStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_FieldStatus.Merge(m, src) +func (dst *FieldStatus) XXX_Merge(src proto.Message) { + xxx_messageInfo_FieldStatus.Merge(dst, src) } func (m *FieldStatus) XXX_Size() int { return m.Size() @@ -1410,7 +1408,7 @@ func (m *FieldStatus) GetAvailableShards() []uint64 { type ClusterStatus struct { ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` - Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes,proto3" json:"Nodes,omitempty"` + Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes" json:"Nodes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1420,7 +1418,7 @@ func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} func (*ClusterStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{23} + return fileDescriptor_private_b229d027a4642df7, []int{23} } func (m *ClusterStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1430,15 +1428,15 @@ func (m *ClusterStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error return xxx_messageInfo_ClusterStatus.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *ClusterStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_ClusterStatus.Merge(m, src) +func (dst *ClusterStatus) XXX_Merge(src proto.Message) { + xxx_messageInfo_ClusterStatus.Merge(dst, src) } func (m *ClusterStatus) XXX_Size() int { return m.Size() @@ -1484,7 +1482,7 @@ func (m *BSIGroup) Reset() { *m = BSIGroup{} } func (m *BSIGroup) String() string { return proto.CompactTextString(m) } func (*BSIGroup) ProtoMessage() {} func (*BSIGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{24} + return fileDescriptor_private_b229d027a4642df7, []int{24} } func (m *BSIGroup) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1494,15 +1492,15 @@ func (m *BSIGroup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BSIGroup.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *BSIGroup) XXX_Merge(src proto.Message) { - xxx_messageInfo_BSIGroup.Merge(m, src) +func (dst *BSIGroup) XXX_Merge(src proto.Message) { + xxx_messageInfo_BSIGroup.Merge(dst, src) } func (m *BSIGroup) XXX_Size() int { return m.Size() @@ -1554,7 +1552,7 @@ func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } func (*CreateViewMessage) ProtoMessage() {} func (*CreateViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{25} + return fileDescriptor_private_b229d027a4642df7, []int{25} } func (m *CreateViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1564,15 +1562,15 @@ func (m *CreateViewMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, e return xxx_messageInfo_CreateViewMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *CreateViewMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateViewMessage.Merge(m, src) +func (dst *CreateViewMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateViewMessage.Merge(dst, src) } func (m *CreateViewMessage) XXX_Size() int { return m.Size() @@ -1617,7 +1615,7 @@ func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} func (*DeleteViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{26} + return fileDescriptor_private_b229d027a4642df7, []int{26} } func (m *DeleteViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1627,15 +1625,15 @@ func (m *DeleteViewMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, e return xxx_messageInfo_DeleteViewMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *DeleteViewMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteViewMessage.Merge(m, src) +func (dst *DeleteViewMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteViewMessage.Merge(dst, src) } func (m *DeleteViewMessage) XXX_Size() int { return m.Size() @@ -1669,11 +1667,11 @@ func (m *DeleteViewMessage) GetView() string { type ResizeInstruction struct { JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - Node *Node `protobuf:"bytes,2,opt,name=Node,proto3" json:"Node,omitempty"` - Coordinator *Node `protobuf:"bytes,3,opt,name=Coordinator,proto3" json:"Coordinator,omitempty"` - Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources,proto3" json:"Sources,omitempty"` - NodeStatus *NodeStatus `protobuf:"bytes,7,opt,name=NodeStatus,proto3" json:"NodeStatus,omitempty"` - ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus,proto3" json:"ClusterStatus,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` + Coordinator *Node `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` + Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` + NodeStatus *NodeStatus `protobuf:"bytes,7,opt,name=NodeStatus" json:"NodeStatus,omitempty"` + ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus" json:"ClusterStatus,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1683,7 +1681,7 @@ func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} func (*ResizeInstruction) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{27} + return fileDescriptor_private_b229d027a4642df7, []int{27} } func (m *ResizeInstruction) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1693,15 +1691,15 @@ func (m *ResizeInstruction) XXX_Marshal(b []byte, deterministic bool) ([]byte, e return xxx_messageInfo_ResizeInstruction.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *ResizeInstruction) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResizeInstruction.Merge(m, src) +func (dst *ResizeInstruction) XXX_Merge(src proto.Message) { + xxx_messageInfo_ResizeInstruction.Merge(dst, src) } func (m *ResizeInstruction) XXX_Size() int { return m.Size() @@ -1755,7 +1753,7 @@ func (m *ResizeInstruction) GetClusterStatus() *ClusterStatus { } type ResizeSource struct { - Node *Node `protobuf:"bytes,1,opt,name=Node,proto3" json:"Node,omitempty"` + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` @@ -1769,7 +1767,7 @@ func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} func (*ResizeSource) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{28} + return fileDescriptor_private_b229d027a4642df7, []int{28} } func (m *ResizeSource) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1779,15 +1777,15 @@ func (m *ResizeSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_ResizeSource.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *ResizeSource) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResizeSource.Merge(m, src) +func (dst *ResizeSource) XXX_Merge(src proto.Message) { + xxx_messageInfo_ResizeSource.Merge(dst, src) } func (m *ResizeSource) XXX_Size() int { return m.Size() @@ -1835,7 +1833,7 @@ func (m *ResizeSource) GetShard() uint64 { type ResizeInstructionComplete struct { JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - Node *Node `protobuf:"bytes,2,opt,name=Node,proto3" json:"Node,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` Error string `protobuf:"bytes,3,opt,name=Error,proto3" json:"Error,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -1846,7 +1844,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{29} + return fileDescriptor_private_b229d027a4642df7, []int{29} } func (m *ResizeInstructionComplete) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1856,15 +1854,15 @@ func (m *ResizeInstructionComplete) XXX_Marshal(b []byte, deterministic bool) ([ return xxx_messageInfo_ResizeInstructionComplete.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *ResizeInstructionComplete) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResizeInstructionComplete.Merge(m, src) +func (dst *ResizeInstructionComplete) XXX_Merge(src proto.Message) { + xxx_messageInfo_ResizeInstructionComplete.Merge(dst, src) } func (m *ResizeInstructionComplete) XXX_Size() int { return m.Size() @@ -1897,7 +1895,7 @@ func (m *ResizeInstructionComplete) GetError() string { } type SetCoordinatorMessage struct { - New *Node `protobuf:"bytes,1,opt,name=New,proto3" json:"New,omitempty"` + New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1907,7 +1905,7 @@ func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{30} + return fileDescriptor_private_b229d027a4642df7, []int{30} } func (m *SetCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1917,15 +1915,15 @@ func (m *SetCoordinatorMessage) XXX_Marshal(b []byte, deterministic bool) ([]byt return xxx_messageInfo_SetCoordinatorMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *SetCoordinatorMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_SetCoordinatorMessage.Merge(m, src) +func (dst *SetCoordinatorMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_SetCoordinatorMessage.Merge(dst, src) } func (m *SetCoordinatorMessage) XXX_Size() int { return m.Size() @@ -1944,7 +1942,7 @@ func (m *SetCoordinatorMessage) GetNew() *Node { } type UpdateCoordinatorMessage struct { - New *Node `protobuf:"bytes,1,opt,name=New,proto3" json:"New,omitempty"` + New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1954,7 +1952,7 @@ func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessa func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*UpdateCoordinatorMessage) ProtoMessage() {} func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{31} + return fileDescriptor_private_b229d027a4642df7, []int{31} } func (m *UpdateCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1964,15 +1962,15 @@ func (m *UpdateCoordinatorMessage) XXX_Marshal(b []byte, deterministic bool) ([] return xxx_messageInfo_UpdateCoordinatorMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *UpdateCoordinatorMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_UpdateCoordinatorMessage.Merge(m, src) +func (dst *UpdateCoordinatorMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_UpdateCoordinatorMessage.Merge(dst, src) } func (m *UpdateCoordinatorMessage) XXX_Size() int { return m.Size() @@ -1992,7 +1990,7 @@ func (m *UpdateCoordinatorMessage) GetNew() *Node { type Topology struct { ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` - NodeIDs []string `protobuf:"bytes,2,rep,name=NodeIDs,proto3" json:"NodeIDs,omitempty"` + NodeIDs []string `protobuf:"bytes,2,rep,name=NodeIDs" json:"NodeIDs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2002,7 +2000,7 @@ func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} func (*Topology) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{32} + return fileDescriptor_private_b229d027a4642df7, []int{32} } func (m *Topology) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2012,15 +2010,15 @@ func (m *Topology) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Topology.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *Topology) XXX_Merge(src proto.Message) { - xxx_messageInfo_Topology.Merge(m, src) +func (dst *Topology) XXX_Merge(src proto.Message) { + xxx_messageInfo_Topology.Merge(dst, src) } func (m *Topology) XXX_Size() int { return m.Size() @@ -2055,7 +2053,7 @@ func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } func (*RecalculateCaches) ProtoMessage() {} func (*RecalculateCaches) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{33} + return fileDescriptor_private_b229d027a4642df7, []int{33} } func (m *RecalculateCaches) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2065,15 +2063,15 @@ func (m *RecalculateCaches) XXX_Marshal(b []byte, deterministic bool) ([]byte, e return xxx_messageInfo_RecalculateCaches.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *RecalculateCaches) XXX_Merge(src proto.Message) { - xxx_messageInfo_RecalculateCaches.Merge(m, src) +func (dst *RecalculateCaches) XXX_Merge(src proto.Message) { + xxx_messageInfo_RecalculateCaches.Merge(dst, src) } func (m *RecalculateCaches) XXX_Size() int { return m.Size() @@ -2121,91 +2119,10 @@ func init() { proto.RegisterType((*Topology)(nil), "internal.Topology") proto.RegisterType((*RecalculateCaches)(nil), "internal.RecalculateCaches") } - -func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } - -var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1174 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x72, 0xdb, 0x44, - 0x18, 0x46, 0x87, 0x38, 0xf6, 0xef, 0x38, 0x07, 0xb5, 0x0d, 0x6a, 0x61, 0x82, 0xd9, 0xe9, 0x50, - 0xd3, 0x19, 0x42, 0xa7, 0xe5, 0x82, 0x53, 0x67, 0x8a, 0xe3, 0x50, 0x44, 0x49, 0x28, 0xeb, 0x24, - 0x77, 0x5c, 0x6c, 0xec, 0x9d, 0x46, 0x13, 0x59, 0x32, 0xd2, 0x2a, 0x89, 0x7b, 0xc1, 0x2d, 0xcc, - 0xf0, 0x02, 0x3c, 0x41, 0x9f, 0x85, 0x4b, 0x1e, 0x81, 0x09, 0x2f, 0xc2, 0xec, 0xbf, 0xbb, 0x92, - 0xec, 0xb8, 0x24, 0x84, 0xde, 0xed, 0xff, 0xfd, 0xfb, 0x9f, 0x0f, 0x5a, 0x41, 0x6b, 0x9c, 0x86, - 0x27, 0x4c, 0xf0, 0xcd, 0x71, 0x9a, 0x88, 0xc4, 0xab, 0x87, 0xb1, 0xe0, 0x69, 0xcc, 0x22, 0xf2, - 0x14, 0x1a, 0x41, 0x3c, 0xe4, 0x67, 0x3b, 0x5c, 0x30, 0xcf, 0x03, 0xf7, 0x19, 0x9f, 0x64, 0xbe, - 0xd3, 0xb6, 0x3a, 0x75, 0x8a, 0x67, 0xef, 0x03, 0x58, 0xde, 0x4b, 0xd9, 0xe0, 0x78, 0xfb, 0x2c, - 0xcc, 0x04, 0x8f, 0x07, 0xdc, 0x77, 0x91, 0x3b, 0x83, 0x92, 0x57, 0x36, 0x2c, 0x7d, 0x1d, 0xf2, - 0x68, 0xf8, 0xfd, 0x58, 0x84, 0x49, 0x9c, 0x49, 0x65, 0x7b, 0x93, 0x31, 0xf7, 0xeb, 0x6d, 0xab, - 0xd3, 0xa0, 0x78, 0xf6, 0xde, 0x85, 0xc6, 0x16, 0x1b, 0x1c, 0x71, 0x64, 0x38, 0xc8, 0x28, 0x81, - 0x82, 0xdb, 0x0f, 0x5f, 0x2a, 0x2b, 0x2d, 0x5a, 0x02, 0x5e, 0x1b, 0x9a, 0x7b, 0xe1, 0x88, 0xff, - 0x90, 0xb3, 0x58, 0xe4, 0x23, 0x7f, 0x01, 0xa5, 0xab, 0x90, 0xb7, 0x0a, 0xce, 0x4e, 0x18, 0xfb, - 0x8d, 0xb6, 0xd5, 0x71, 0xa8, 0x3c, 0x22, 0xc2, 0xce, 0x7c, 0xd0, 0x08, 0x3b, 0x2b, 0x42, 0x6c, - 0x4e, 0x87, 0xb8, 0x9b, 0xf4, 0x05, 0x8b, 0x87, 0x2c, 0x1d, 0x1e, 0x84, 0xfc, 0xd4, 0x5f, 0x52, - 0x21, 0x4e, 0xa3, 0x52, 0xb6, 0xcb, 0x32, 0xee, 0xb7, 0x50, 0x1d, 0x9e, 0xbd, 0x3b, 0x50, 0xef, - 0x86, 0xa2, 0xc7, 0xc7, 0xe2, 0xc8, 0x5f, 0x6e, 0x5b, 0x1d, 0x97, 0x16, 0xb4, 0x77, 0x13, 0x16, - 0xfa, 0x03, 0x16, 0x71, 0x7f, 0x05, 0x05, 0x14, 0x41, 0x08, 0x2c, 0x07, 0xa3, 0x71, 0x92, 0x0a, - 0xca, 0xb3, 0x71, 0x12, 0x67, 0x5c, 0x7a, 0xb9, 0x9d, 0xa6, 0xbe, 0x85, 0x11, 0xc9, 0x23, 0xf9, - 0x19, 0x56, 0xbb, 0x51, 0x32, 0x38, 0xee, 0x31, 0xc1, 0x28, 0xff, 0x29, 0xe7, 0x99, 0x90, 0xda, - 0xb0, 0x52, 0xfa, 0x9e, 0x22, 0x24, 0x8a, 0x59, 0xf7, 0x6d, 0x85, 0x22, 0x21, 0x3d, 0xc5, 0x38, - 0x54, 0x92, 0xf0, 0x8c, 0xde, 0x1c, 0xb1, 0x74, 0x88, 0x99, 0x75, 0xa9, 0x22, 0x24, 0x8a, 0x96, - 0xb0, 0x1a, 0x2e, 0x55, 0x04, 0x09, 0x60, 0xad, 0x62, 0x5f, 0xbb, 0xb9, 0x0e, 0x35, 0x9a, 0x9c, - 0x06, 0xbd, 0xcc, 0xb7, 0xda, 0x4e, 0xc7, 0xa5, 0x9a, 0xc2, 0xb2, 0x25, 0x51, 0x3e, 0x8a, 0x25, - 0xcb, 0x46, 0x56, 0x09, 0x90, 0xdb, 0xb0, 0x80, 0x35, 0x94, 0x51, 0x96, 0xb2, 0xf2, 0x48, 0x7e, - 0xb1, 0xa0, 0xb1, 0xc3, 0xce, 0xd0, 0x91, 0xcc, 0x7b, 0x0c, 0x75, 0x93, 0x6d, 0xbc, 0xd4, 0x7c, - 0xf8, 0xfe, 0xa6, 0x69, 0xd3, 0xcd, 0xe2, 0xda, 0xa6, 0xb9, 0xb3, 0x1d, 0x8b, 0x74, 0x42, 0x0b, - 0x91, 0x3b, 0x5f, 0x40, 0x6b, 0x8a, 0x25, 0xed, 0x1d, 0xf3, 0x89, 0xc9, 0xea, 0x31, 0x9f, 0xc8, - 0x58, 0x4f, 0x58, 0x94, 0x73, 0xcc, 0x95, 0x4b, 0x15, 0xf1, 0xb9, 0xfd, 0xa9, 0x45, 0x0e, 0xc0, - 0xdb, 0x4a, 0x39, 0x13, 0x1c, 0x8d, 0xec, 0xf0, 0x2c, 0x63, 0x2f, 0xf8, 0x65, 0x19, 0x77, 0xaa, - 0x19, 0x2f, 0xb2, 0x6b, 0x57, 0xb2, 0x4b, 0xee, 0x83, 0xd7, 0xe3, 0x11, 0x17, 0x5c, 0xcf, 0xd8, - 0xbf, 0xe8, 0x25, 0x7d, 0xe3, 0xc3, 0xe5, 0x77, 0xbd, 0x7b, 0xe0, 0xca, 0x81, 0x45, 0x63, 0xcd, - 0x87, 0x37, 0xca, 0x3c, 0x15, 0xb3, 0x4c, 0xf1, 0x02, 0x89, 0x8c, 0x52, 0xf4, 0xf2, 0x8a, 0x81, - 0x4d, 0xb5, 0xd2, 0x7d, 0x6d, 0xca, 0x41, 0x53, 0xeb, 0xa5, 0xa9, 0xea, 0xb0, 0x6b, 0x6b, 0x4f, - 0x4c, 0xb8, 0xd7, 0xb5, 0x46, 0x06, 0xf0, 0x8e, 0xd2, 0xf0, 0xd5, 0x09, 0x0b, 0x23, 0x76, 0x18, - 0xfd, 0xa7, 0x8a, 0x4c, 0x39, 0xee, 0xc3, 0x22, 0xca, 0x06, 0x3d, 0xdd, 0xdb, 0x86, 0x24, 0x3f, - 0x42, 0x39, 0x26, 0xbb, 0x6c, 0xc4, 0xb5, 0x36, 0x3c, 0x17, 0xf1, 0xda, 0x97, 0xc7, 0x2b, 0x0d, - 0xcb, 0xd1, 0x92, 0x0b, 0xd3, 0x91, 0x86, 0x91, 0x20, 0x8f, 0xa0, 0xd6, 0x1f, 0x1c, 0xf1, 0x11, - 0xf3, 0x3e, 0x84, 0x45, 0xf4, 0x90, 0x67, 0xba, 0xa3, 0x57, 0x66, 0x2a, 0x45, 0x0d, 0x9f, 0xf4, - 0x74, 0x64, 0x73, 0x7d, 0xba, 0x07, 0x35, 0xb4, 0x9e, 0xf9, 0xee, 0xac, 0x1a, 0xc4, 0xa9, 0x66, - 0x93, 0x6d, 0x70, 0xf6, 0x69, 0x20, 0x27, 0x15, 0x3d, 0x30, 0x5a, 0x34, 0x25, 0x75, 0x7f, 0x93, - 0x64, 0x42, 0xe7, 0x09, 0xcf, 0x12, 0x7b, 0x9e, 0xa4, 0x02, 0x73, 0xd4, 0xa2, 0x78, 0x26, 0x19, - 0xb8, 0xbb, 0xc9, 0x90, 0x7b, 0xcb, 0x60, 0x07, 0x3d, 0xad, 0xc3, 0x0e, 0x7a, 0xde, 0x7b, 0xa8, - 0x5e, 0xa7, 0xa6, 0x55, 0x3a, 0xb1, 0x4f, 0x03, 0x8a, 0x86, 0xef, 0x42, 0x2b, 0xc8, 0xb6, 0x92, - 0x24, 0x1d, 0x86, 0x31, 0x13, 0x49, 0xaa, 0xbf, 0x24, 0xd3, 0x20, 0xce, 0x8a, 0x60, 0x42, 0xed, - 0xf8, 0x06, 0x55, 0x04, 0x79, 0x02, 0xab, 0xd2, 0x28, 0x12, 0xa6, 0xde, 0xeb, 0x50, 0x93, 0x58, - 0xe1, 0x84, 0xa6, 0x4a, 0x0d, 0x76, 0x55, 0xc3, 0x77, 0x4a, 0xc3, 0xf6, 0x09, 0x8f, 0x45, 0xa5, - 0x63, 0x90, 0x46, 0x05, 0x2d, 0xaa, 0x08, 0x8f, 0xa8, 0x00, 0x75, 0x24, 0xcb, 0x65, 0x24, 0x12, - 0xa5, 0xc8, 0x23, 0xbf, 0x59, 0x00, 0xc6, 0xa1, 0x3c, 0x2b, 0x44, 0xac, 0xd7, 0x8b, 0x78, 0x1d, - 0x53, 0x79, 0x3d, 0x2d, 0xab, 0xe5, 0x2d, 0x85, 0x53, 0xd3, 0x19, 0x1f, 0x97, 0x9d, 0xa1, 0x4a, - 0x7a, 0x6b, 0xa6, 0x33, 0x94, 0xd5, 0xb2, 0x3f, 0x9e, 0x43, 0xb3, 0x82, 0xcf, 0xed, 0x92, 0x8f, - 0x8a, 0x2e, 0xb1, 0x67, 0x55, 0x22, 0xae, 0x55, 0x9a, 0x5e, 0x79, 0x06, 0xcd, 0x0a, 0x3c, 0x57, - 0x63, 0x07, 0x56, 0xa6, 0xe7, 0xd0, 0xec, 0xf7, 0x59, 0x98, 0x84, 0xd0, 0xda, 0x8a, 0xf2, 0x4c, - 0xf0, 0x54, 0xab, 0x93, 0x1f, 0x05, 0x05, 0x14, 0xc5, 0x2b, 0x81, 0xf9, 0xf5, 0xf3, 0xee, 0xc2, - 0x82, 0x4c, 0xa3, 0x1a, 0xa7, 0x8b, 0x39, 0x56, 0x4c, 0x72, 0x00, 0xf5, 0x6e, 0x3f, 0x78, 0x9a, - 0x26, 0xf9, 0x78, 0xae, 0xd3, 0xe6, 0xdd, 0x61, 0x57, 0xde, 0x1d, 0xfa, 0x65, 0xe0, 0x5c, 0x78, - 0x19, 0xb8, 0xc5, 0xcb, 0x80, 0xf4, 0x61, 0x4d, 0xad, 0x4a, 0x39, 0xc5, 0xd7, 0x59, 0x38, 0xe6, - 0xa3, 0xeb, 0x94, 0x1f, 0x5d, 0xa9, 0x54, 0xed, 0xb3, 0x37, 0xa9, 0xf4, 0x95, 0x0d, 0x6b, 0x94, - 0x67, 0xe1, 0x4b, 0x1e, 0xc4, 0x99, 0x48, 0xf3, 0x81, 0xdc, 0x49, 0x52, 0xfe, 0xdb, 0xe4, 0x50, - 0x67, 0xdb, 0xa1, 0x8a, 0xb8, 0x4a, 0xa7, 0x7b, 0x0f, 0xa0, 0x39, 0x3b, 0xb3, 0x17, 0xaf, 0x56, - 0xaf, 0x78, 0x0f, 0x60, 0xb1, 0x9f, 0xe4, 0xe9, 0xa0, 0x68, 0xdf, 0xca, 0x9e, 0x54, 0x9e, 0x29, - 0x36, 0x35, 0xd7, 0xbc, 0x4f, 0xaa, 0xc3, 0xe4, 0x2f, 0xa2, 0x89, 0x9b, 0xd3, 0x26, 0x74, 0x7f, - 0x56, 0x87, 0xee, 0xf1, 0x4c, 0x5b, 0xf9, 0x35, 0x14, 0x7c, 0xbb, 0x14, 0x9c, 0x62, 0xd3, 0xe9, - 0xdb, 0xe4, 0x57, 0x0b, 0x96, 0xaa, 0xee, 0x5c, 0x69, 0x88, 0x8b, 0xea, 0xd8, 0x97, 0x7f, 0xf5, - 0x4d, 0x75, 0xdc, 0x79, 0xef, 0xac, 0x85, 0xea, 0x4b, 0xe0, 0x18, 0x6e, 0x5f, 0x28, 0xd9, 0x56, - 0x32, 0x1a, 0xcb, 0xde, 0xf8, 0x1f, 0xa5, 0x93, 0xeb, 0x2d, 0x4d, 0x75, 0xd1, 0x1a, 0x54, 0x11, - 0xe4, 0x33, 0xb8, 0xd5, 0xe7, 0xa2, 0x52, 0x30, 0xd3, 0x79, 0x6d, 0x70, 0x76, 0xf9, 0xe9, 0x6b, - 0xc2, 0x97, 0x2c, 0xf2, 0x25, 0xf8, 0xfb, 0xe3, 0x21, 0x13, 0xfc, 0x5a, 0xd2, 0x5d, 0xa8, 0xef, - 0x25, 0xe3, 0x24, 0x4a, 0x5e, 0x4c, 0x2e, 0xd9, 0x00, 0x3e, 0x2c, 0xaa, 0x5d, 0xae, 0x56, 0x4a, - 0x83, 0x1a, 0x92, 0xdc, 0x90, 0xcd, 0x3d, 0x60, 0xd1, 0x20, 0x8f, 0xa4, 0x1b, 0xf2, 0xed, 0x98, - 0x75, 0x57, 0xff, 0x38, 0xdf, 0xb0, 0xfe, 0x3c, 0xdf, 0xb0, 0xfe, 0x3a, 0xdf, 0xb0, 0x7e, 0xff, - 0x7b, 0xe3, 0xad, 0xc3, 0x1a, 0xfe, 0xc9, 0x3c, 0xfa, 0x27, 0x00, 0x00, 0xff, 0xff, 0xe1, 0xbf, - 0xc3, 0x49, 0xda, 0x0c, 0x00, 0x00, -} - func (m *IndexMeta) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2213,46 +2130,40 @@ func (m *IndexMeta) Marshal() (dAtA []byte, err error) { } func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *IndexMeta) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if m.TrackExistence { - i-- - if m.TrackExistence { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x20 - } if m.Keys { - i-- + dAtA[i] = 0x18 + i++ if m.Keys { dAtA[i] = 1 } else { dAtA[i] = 0 } - i-- - dAtA[i] = 0x18 + i++ } - return len(dAtA) - i, nil + if m.TrackExistence { + dAtA[i] = 0x20 + i++ + if m.TrackExistence { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *FieldOptions) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2260,97 +2171,88 @@ func (m *FieldOptions) Marshal() (dAtA []byte, err error) { } func (m *FieldOptions) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *FieldOptions) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) + if len(m.CacheType) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.CacheType))) + i += copy(dAtA[i:], m.CacheType) } - if m.Scale != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.Scale)) - i-- - dAtA[i] = 0x78 + if m.CacheSize != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.CacheSize)) } - if m.BitDepth != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.BitDepth)) - i-- - dAtA[i] = 0x70 + if len(m.TimeQuantum) > 0 { + dAtA[i] = 0x2a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) + i += copy(dAtA[i:], m.TimeQuantum) } - if m.Base != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.Base)) - i-- - dAtA[i] = 0x68 + if len(m.Type) > 0 { + dAtA[i] = 0x42 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) + i += copy(dAtA[i:], m.Type) } - if m.NoStandardView { - i-- - if m.NoStandardView { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x60 + if m.Min != 0 { + dAtA[i] = 0x48 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) + } + if m.Max != 0 { + dAtA[i] = 0x50 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) } if m.Keys { - i-- + dAtA[i] = 0x58 + i++ if m.Keys { dAtA[i] = 1 } else { dAtA[i] = 0 } - i-- - dAtA[i] = 0x58 + i++ } - if m.Max != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) - i-- - dAtA[i] = 0x50 + if m.NoStandardView { + dAtA[i] = 0x60 + i++ + if m.NoStandardView { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ } - if m.Min != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) - i-- - dAtA[i] = 0x48 + if m.Base != 0 { + dAtA[i] = 0x68 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Base)) } - if len(m.Type) > 0 { - i -= len(m.Type) - copy(dAtA[i:], m.Type) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) - i-- - dAtA[i] = 0x42 + if m.BitDepth != 0 { + dAtA[i] = 0x70 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.BitDepth)) } - if len(m.TimeQuantum) > 0 { - i -= len(m.TimeQuantum) - copy(dAtA[i:], m.TimeQuantum) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) - i-- - dAtA[i] = 0x2a + if m.Scale != 0 { + dAtA[i] = 0x78 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Scale)) } - if m.CacheSize != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.CacheSize)) - i-- - dAtA[i] = 0x20 + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) } - if len(m.CacheType) > 0 { - i -= len(m.CacheType) - copy(dAtA[i:], m.CacheType) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.CacheType))) - i-- - dAtA[i] = 0x1a - } - return len(dAtA) - i, nil + return i, nil } func (m *ImportResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2358,33 +2260,26 @@ func (m *ImportResponse) Marshal() (dAtA []byte, err error) { } func (m *ImportResponse) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ImportResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.Err) > 0 { - i -= len(m.Err) - copy(dAtA[i:], m.Err) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Err))) - i-- dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Err))) + i += copy(dAtA[i:], m.Err) } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *BlockDataRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2392,57 +2287,48 @@ func (m *BlockDataRequest) Marshal() (dAtA []byte, err error) { } func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *BlockDataRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.View) > 0 { - i -= len(m.View) - copy(dAtA[i:], m.View) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) - i-- - dAtA[i] = 0x2a - } - if m.Shard != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) - i-- - dAtA[i] = 0x20 - } - if m.Block != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.Block)) - i-- - dAtA[i] = 0x18 + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if len(m.Field) > 0 { - i -= len(m.Field) - copy(dAtA[i:], m.Field) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } - if len(m.Index) > 0 { - i -= len(m.Index) - copy(dAtA[i:], m.Index) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i-- - dAtA[i] = 0xa + if m.Block != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Block)) } - return len(dAtA) - i, nil + if m.Shard != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) + } + if len(m.View) > 0 { + dAtA[i] = 0x2a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) + i += copy(dAtA[i:], m.View) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *BlockDataResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2450,23 +2336,14 @@ func (m *BlockDataResponse) Marshal() (dAtA []byte, err error) { } func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *BlockDataResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.ColumnIDs) > 0 { - dAtA2 := make([]byte, len(m.ColumnIDs)*10) + if len(m.RowIDs) > 0 { + dAtA2 := make([]byte, len(m.RowIDs)*10) var j1 int - for _, num := range m.ColumnIDs { + for _, num := range m.RowIDs { for num >= 1<<7 { dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2475,16 +2352,15 @@ func (m *BlockDataResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA2[j1] = uint8(num) j1++ } - i -= j1 - copy(dAtA[i:], dAtA2[:j1]) + dAtA[i] = 0xa + i++ i = encodeVarintPrivate(dAtA, i, uint64(j1)) - i-- - dAtA[i] = 0x12 + i += copy(dAtA[i:], dAtA2[:j1]) } - if len(m.RowIDs) > 0 { - dAtA4 := make([]byte, len(m.RowIDs)*10) + if len(m.ColumnIDs) > 0 { + dAtA4 := make([]byte, len(m.ColumnIDs)*10) var j3 int - for _, num := range m.RowIDs { + for _, num := range m.ColumnIDs { for num >= 1<<7 { dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2493,19 +2369,21 @@ func (m *BlockDataResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA4[j3] = uint8(num) j3++ } - i -= j3 - copy(dAtA[i:], dAtA4[:j3]) + dAtA[i] = 0x12 + i++ i = encodeVarintPrivate(dAtA, i, uint64(j3)) - i-- - dAtA[i] = 0xa + i += copy(dAtA[i:], dAtA4[:j3]) } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *Cache) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2513,19 +2391,10 @@ func (m *Cache) Marshal() (dAtA []byte, err error) { } func (m *Cache) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Cache) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.IDs) > 0 { dAtA6 := make([]byte, len(m.IDs)*10) var j5 int @@ -2538,19 +2407,21 @@ func (m *Cache) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA6[j5] = uint8(num) j5++ } - i -= j5 - copy(dAtA[i:], dAtA6[:j5]) - i = encodeVarintPrivate(dAtA, i, uint64(j5)) - i-- dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(j5)) + i += copy(dAtA[i:], dAtA6[:j5]) } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *MaxShards) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2558,43 +2429,36 @@ func (m *MaxShards) Marshal() (dAtA []byte, err error) { } func (m *MaxShards) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *MaxShards) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.Standard) > 0 { - for k := range m.Standard { + for k, _ := range m.Standard { + dAtA[i] = 0xa + i++ v := m.Standard[k] - baseI := i - i = encodeVarintPrivate(dAtA, i, uint64(v)) - i-- - dAtA[i] = 0x10 - i -= len(k) - copy(dAtA[i:], k) + mapSize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) + i = encodeVarintPrivate(dAtA, i, uint64(mapSize)) + dAtA[i] = 0xa + i++ i = encodeVarintPrivate(dAtA, i, uint64(len(k))) - i-- - dAtA[i] = 0xa - i = encodeVarintPrivate(dAtA, i, uint64(baseI-i)) - i-- - dAtA[i] = 0xa + i += copy(dAtA[i:], k) + dAtA[i] = 0x10 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(v)) } } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *CreateShardMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2602,45 +2466,37 @@ func (m *CreateShardMessage) Marshal() (dAtA []byte, err error) { } func (m *CreateShardMessage) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *CreateShardMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.Field) > 0 { - i -= len(m.Field) - copy(dAtA[i:], m.Field) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i-- - dAtA[i] = 0x1a + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if m.Shard != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) - i-- dAtA[i] = 0x10 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) } - if len(m.Index) > 0 { - i -= len(m.Index) - copy(dAtA[i:], m.Index) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i-- - dAtA[i] = 0xa + if len(m.Field) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *DeleteIndexMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2648,33 +2504,26 @@ func (m *DeleteIndexMessage) Marshal() (dAtA []byte, err error) { } func (m *DeleteIndexMessage) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *DeleteIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.Index) > 0 { - i -= len(m.Index) - copy(dAtA[i:], m.Index) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i-- dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *CreateIndexMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2682,45 +2531,36 @@ func (m *CreateIndexMessage) Marshal() (dAtA []byte, err error) { } func (m *CreateIndexMessage) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *CreateIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if m.Meta != nil { - { - size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n7, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n7 } - if len(m.Index) > 0 { - i -= len(m.Index) - copy(dAtA[i:], m.Index) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i-- - dAtA[i] = 0xa + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) } - return len(dAtA) - i, nil + return i, nil } func (m *CreateFieldMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2728,52 +2568,42 @@ func (m *CreateFieldMessage) Marshal() (dAtA []byte, err error) { } func (m *CreateFieldMessage) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *CreateFieldMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if m.Meta != nil { - { - size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x1a + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if len(m.Field) > 0 { - i -= len(m.Field) - copy(dAtA[i:], m.Field) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } - if len(m.Index) > 0 { - i -= len(m.Index) - copy(dAtA[i:], m.Index) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i-- - dAtA[i] = 0xa + if m.Meta != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n8, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n8 } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *DeleteFieldMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2781,40 +2611,32 @@ func (m *DeleteFieldMessage) Marshal() (dAtA []byte, err error) { } func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *DeleteFieldMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if len(m.Field) > 0 { - i -= len(m.Field) - copy(dAtA[i:], m.Field) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } - if len(m.Index) > 0 { - i -= len(m.Index) - copy(dAtA[i:], m.Index) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i-- - dAtA[i] = 0xa + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) } - return len(dAtA) - i, nil + return i, nil } func (m *DeleteAvailableShardMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2822,45 +2644,37 @@ func (m *DeleteAvailableShardMessage) Marshal() (dAtA []byte, err error) { } func (m *DeleteAvailableShardMessage) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *DeleteAvailableShardMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if m.ShardID != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.ShardID)) - i-- - dAtA[i] = 0x18 + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if len(m.Field) > 0 { - i -= len(m.Field) - copy(dAtA[i:], m.Field) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } - if len(m.Index) > 0 { - i -= len(m.Index) - copy(dAtA[i:], m.Index) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i-- - dAtA[i] = 0xa + if m.ShardID != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.ShardID)) } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *Field) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2868,54 +2682,51 @@ func (m *Field) Marshal() (dAtA []byte, err error) { } func (m *Field) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Field) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.Views) > 0 { - for iNdEx := len(m.Views) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.Views[iNdEx]) - copy(dAtA[i:], m.Views[iNdEx]) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Views[iNdEx]))) - i-- - dAtA[i] = 0x1a - } + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) } if m.Meta != nil { - { - size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n9, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n9 } - if len(m.Name) > 0 { - i -= len(m.Name) - copy(dAtA[i:], m.Name) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) - i-- - dAtA[i] = 0xa + if len(m.Views) > 0 { + for _, s := range m.Views { + dAtA[i] = 0x1a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *Schema) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2923,40 +2734,32 @@ func (m *Schema) Marshal() (dAtA []byte, err error) { } func (m *Schema) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Schema) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.Indexes) > 0 { - for iNdEx := len(m.Indexes) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Indexes[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- + for _, msg := range m.Indexes { dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n } } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *Index) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2964,47 +2767,38 @@ func (m *Index) Marshal() (dAtA []byte, err error) { } func (m *Index) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) } if len(m.Fields) > 0 { - for iNdEx := len(m.Fields) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Fields[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- + for _, msg := range m.Fields { dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n } } - if len(m.Name) > 0 { - i -= len(m.Name) - copy(dAtA[i:], m.Name) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) - i-- - dAtA[i] = 0xa + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) } - return len(dAtA) - i, nil + return i, nil } func (m *URI) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3012,45 +2806,37 @@ func (m *URI) Marshal() (dAtA []byte, err error) { } func (m *URI) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *URI) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if m.Port != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.Port)) - i-- - dAtA[i] = 0x18 + if len(m.Scheme) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Scheme))) + i += copy(dAtA[i:], m.Scheme) } if len(m.Host) > 0 { - i -= len(m.Host) - copy(dAtA[i:], m.Host) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) + i += copy(dAtA[i:], m.Host) } - if len(m.Scheme) > 0 { - i -= len(m.Scheme) - copy(dAtA[i:], m.Scheme) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Scheme))) - i-- - dAtA[i] = 0xa + if m.Port != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Port)) } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *Node) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3058,62 +2844,52 @@ func (m *Node) Marshal() (dAtA []byte, err error) { } func (m *Node) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Node) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) + if len(m.ID) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ID))) + i += copy(dAtA[i:], m.ID) } - if len(m.State) > 0 { - i -= len(m.State) - copy(dAtA[i:], m.State) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) - i-- - dAtA[i] = 0x22 + if m.URI != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) + n10, err := m.URI.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n10 } if m.IsCoordinator { - i-- + dAtA[i] = 0x18 + i++ if m.IsCoordinator { dAtA[i] = 1 } else { dAtA[i] = 0 } - i-- - dAtA[i] = 0x18 + i++ } - if m.URI != nil { - { - size, err := m.URI.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 + if len(m.State) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) + i += copy(dAtA[i:], m.State) } - if len(m.ID) > 0 { - i -= len(m.ID) - copy(dAtA[i:], m.ID) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.ID))) - i-- - dAtA[i] = 0xa + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) } - return len(dAtA) - i, nil + return i, nil } func (m *NodeStateMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3121,40 +2897,32 @@ func (m *NodeStateMessage) Marshal() (dAtA []byte, err error) { } func (m *NodeStateMessage) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *NodeStateMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) + if len(m.NodeID) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.NodeID))) + i += copy(dAtA[i:], m.NodeID) } if len(m.State) > 0 { - i -= len(m.State) - copy(dAtA[i:], m.State) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) + i += copy(dAtA[i:], m.State) } - if len(m.NodeID) > 0 { - i -= len(m.NodeID) - copy(dAtA[i:], m.NodeID) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.NodeID))) - i-- - dAtA[i] = 0xa + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) } - return len(dAtA) - i, nil + return i, nil } func (m *NodeEventMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3162,43 +2930,35 @@ func (m *NodeEventMessage) Marshal() (dAtA []byte, err error) { } func (m *NodeEventMessage) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *NodeEventMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) + if m.Event != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Event)) } if m.Node != nil { - { - size, err := m.Node.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n11, err := m.Node.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n11 } - if m.Event != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.Event)) - i-- - dAtA[i] = 0x8 + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) } - return len(dAtA) - i, nil + return i, nil } func (m *NodeStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3206,64 +2966,52 @@ func (m *NodeStatus) Marshal() (dAtA []byte, err error) { } func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *NodeStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.Indexes) > 0 { - for iNdEx := len(m.Indexes) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Indexes[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x22 + if m.Node != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n12, err := m.Node.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } + i += n12 } if m.Schema != nil { - { - size, err := m.Schema.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) + n13, err := m.Schema.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n13 } - if m.Node != nil { - { - size, err := m.Node.MarshalToSizedBuffer(dAtA[:i]) + if len(m.Indexes) > 0 { + for _, msg := range m.Indexes { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) + i += n } - i-- - dAtA[i] = 0xa } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *IndexStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3271,47 +3019,38 @@ func (m *IndexStatus) Marshal() (dAtA []byte, err error) { } func (m *IndexStatus) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *IndexStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) } if len(m.Fields) > 0 { - for iNdEx := len(m.Fields) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Fields[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- + for _, msg := range m.Fields { dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n } } - if len(m.Name) > 0 { - i -= len(m.Name) - copy(dAtA[i:], m.Name) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) - i-- - dAtA[i] = 0xa + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) } - return len(dAtA) - i, nil + return i, nil } func (m *FieldStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3319,18 +3058,15 @@ func (m *FieldStatus) Marshal() (dAtA []byte, err error) { } func (m *FieldStatus) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *FieldStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) } if len(m.AvailableShards) > 0 { dAtA15 := make([]byte, len(m.AvailableShards)*10) @@ -3344,26 +3080,21 @@ func (m *FieldStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA15[j14] = uint8(num) j14++ } - i -= j14 - copy(dAtA[i:], dAtA15[:j14]) - i = encodeVarintPrivate(dAtA, i, uint64(j14)) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(j14)) + i += copy(dAtA[i:], dAtA15[:j14]) } - if len(m.Name) > 0 { - i -= len(m.Name) - copy(dAtA[i:], m.Name) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) - i-- - dAtA[i] = 0xa + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) } - return len(dAtA) - i, nil + return i, nil } func (m *ClusterStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3371,54 +3102,44 @@ func (m *ClusterStatus) Marshal() (dAtA []byte, err error) { } func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ClusterStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.Nodes) > 0 { - for iNdEx := len(m.Nodes) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Nodes[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x1a - } + if len(m.ClusterID) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClusterID))) + i += copy(dAtA[i:], m.ClusterID) } if len(m.State) > 0 { - i -= len(m.State) - copy(dAtA[i:], m.State) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) + i += copy(dAtA[i:], m.State) } - if len(m.ClusterID) > 0 { - i -= len(m.ClusterID) - copy(dAtA[i:], m.ClusterID) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClusterID))) - i-- - dAtA[i] = 0xa + if len(m.Nodes) > 0 { + for _, msg := range m.Nodes { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *BSIGroup) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3426,50 +3147,42 @@ func (m *BSIGroup) Marshal() (dAtA []byte, err error) { } func (m *BSIGroup) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *BSIGroup) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if m.Max != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) - i-- - dAtA[i] = 0x20 - } - if m.Min != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) - i-- - dAtA[i] = 0x18 + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) } if len(m.Type) > 0 { - i -= len(m.Type) - copy(dAtA[i:], m.Type) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) + i += copy(dAtA[i:], m.Type) } - if len(m.Name) > 0 { - i -= len(m.Name) - copy(dAtA[i:], m.Name) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) - i-- - dAtA[i] = 0xa + if m.Min != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) } - return len(dAtA) - i, nil + if m.Max != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *CreateViewMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3477,47 +3190,38 @@ func (m *CreateViewMessage) Marshal() (dAtA []byte, err error) { } func (m *CreateViewMessage) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *CreateViewMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.View) > 0 { - i -= len(m.View) - copy(dAtA[i:], m.View) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) - i-- - dAtA[i] = 0x1a + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if len(m.Field) > 0 { - i -= len(m.Field) - copy(dAtA[i:], m.Field) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } - if len(m.Index) > 0 { - i -= len(m.Index) - copy(dAtA[i:], m.Index) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i-- - dAtA[i] = 0xa + if len(m.View) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) + i += copy(dAtA[i:], m.View) } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *DeleteViewMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3525,47 +3229,38 @@ func (m *DeleteViewMessage) Marshal() (dAtA []byte, err error) { } func (m *DeleteViewMessage) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *DeleteViewMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.View) > 0 { - i -= len(m.View) - copy(dAtA[i:], m.View) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) - i-- - dAtA[i] = 0x1a + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if len(m.Field) > 0 { - i -= len(m.Field) - copy(dAtA[i:], m.Field) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } - if len(m.Index) > 0 { - i -= len(m.Index) - copy(dAtA[i:], m.Index) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i-- - dAtA[i] = 0xa + if len(m.View) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) + i += copy(dAtA[i:], m.View) } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *ResizeInstruction) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3573,93 +3268,77 @@ func (m *ResizeInstruction) Marshal() (dAtA []byte, err error) { } func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ResizeInstruction) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if m.NodeStatus != nil { - { - size, err := m.NodeStatus.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x3a - } - if m.ClusterStatus != nil { - { - size, err := m.ClusterStatus.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x32 - } - if len(m.Sources) > 0 { - for iNdEx := len(m.Sources) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Sources[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x22 - } - } - if m.Coordinator != nil { - { - size, err := m.Coordinator.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x1a + if m.JobID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) } if m.Node != nil { - { - size, err := m.Node.MarshalToSizedBuffer(dAtA[:i]) + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n16, err := m.Node.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n16 + } + if m.Coordinator != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Coordinator.Size())) + n17, err := m.Coordinator.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n17 + } + if len(m.Sources) > 0 { + for _, msg := range m.Sources { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) + i += n } - i-- - dAtA[i] = 0x12 } - if m.JobID != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) - i-- - dAtA[i] = 0x8 + if m.ClusterStatus != nil { + dAtA[i] = 0x32 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size())) + n18, err := m.ClusterStatus.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n18 } - return len(dAtA) - i, nil + if m.NodeStatus != nil { + dAtA[i] = 0x3a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.NodeStatus.Size())) + n19, err := m.NodeStatus.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n19 + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *ResizeSource) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3667,64 +3346,53 @@ func (m *ResizeSource) Marshal() (dAtA []byte, err error) { } func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ResizeSource) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if m.Shard != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) - i-- - dAtA[i] = 0x28 - } - if len(m.View) > 0 { - i -= len(m.View) - copy(dAtA[i:], m.View) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) - i-- - dAtA[i] = 0x22 - } - if len(m.Field) > 0 { - i -= len(m.Field) - copy(dAtA[i:], m.Field) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i-- - dAtA[i] = 0x1a + if m.Node != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n20, err := m.Node.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n20 } if len(m.Index) > 0 { - i -= len(m.Index) - copy(dAtA[i:], m.Index) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } - if m.Node != nil { - { - size, err := m.Node.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0xa + if len(m.Field) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } - return len(dAtA) - i, nil + if len(m.View) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) + i += copy(dAtA[i:], m.View) + } + if m.Shard != 0 { + dAtA[i] = 0x28 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *ResizeInstructionComplete) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3732,50 +3400,41 @@ func (m *ResizeInstructionComplete) Marshal() (dAtA []byte, err error) { } func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ResizeInstructionComplete) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.Error) > 0 { - i -= len(m.Error) - copy(dAtA[i:], m.Error) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Error))) - i-- - dAtA[i] = 0x1a + if m.JobID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) } if m.Node != nil { - { - size, err := m.Node.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n21, err := m.Node.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n21 } - if m.JobID != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) - i-- - dAtA[i] = 0x8 + if len(m.Error) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Error))) + i += copy(dAtA[i:], m.Error) } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *SetCoordinatorMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3783,38 +3442,30 @@ func (m *SetCoordinatorMessage) Marshal() (dAtA []byte, err error) { } func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *SetCoordinatorMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if m.New != nil { - { - size, err := m.New.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) + n22, err := m.New.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n22 } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *UpdateCoordinatorMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3822,38 +3473,30 @@ func (m *UpdateCoordinatorMessage) Marshal() (dAtA []byte, err error) { } func (m *UpdateCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *UpdateCoordinatorMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if m.New != nil { - { - size, err := m.New.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPrivate(dAtA, i, uint64(size)) - } - i-- dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) + n23, err := m.New.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n23 } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *Topology) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3861,42 +3504,41 @@ func (m *Topology) Marshal() (dAtA []byte, err error) { } func (m *Topology) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Topology) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) + if len(m.ClusterID) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClusterID))) + i += copy(dAtA[i:], m.ClusterID) } if len(m.NodeIDs) > 0 { - for iNdEx := len(m.NodeIDs) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.NodeIDs[iNdEx]) - copy(dAtA[i:], m.NodeIDs[iNdEx]) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.NodeIDs[iNdEx]))) - i-- + for _, s := range m.NodeIDs { dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) } } - if len(m.ClusterID) > 0 { - i -= len(m.ClusterID) - copy(dAtA[i:], m.ClusterID) - i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClusterID))) - i-- - dAtA[i] = 0xa + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) } - return len(dAtA) - i, nil + return i, nil } func (m *RecalculateCaches) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -3904,32 +3546,24 @@ func (m *RecalculateCaches) Marshal() (dAtA []byte, err error) { } func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *RecalculateCaches) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) + i += copy(dAtA[i:], m.XXX_unrecognized) } - return len(dAtA) - i, nil + return i, nil } func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { - offset -= sovPrivate(v) - base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) - return base + return offset + 1 } func (m *IndexMeta) Size() (n int) { if m == nil { @@ -4718,7 +4352,14 @@ func (m *RecalculateCaches) Size() (n int) { } func sovPrivate(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n } func sozPrivate(x uint64) (n int) { return sovPrivate(uint64((x << 1) ^ uint64((int64(x) >> 63)))) @@ -4738,7 +4379,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4766,7 +4407,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + v |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4786,7 +4427,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + v |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4801,9 +4442,6 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4832,7 +4470,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4860,7 +4498,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4870,9 +4508,6 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4892,7 +4527,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.CacheSize |= uint32(b&0x7F) << shift + m.CacheSize |= (uint32(b) & 0x7F) << shift if b < 0x80 { break } @@ -4911,7 +4546,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4921,9 +4556,6 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4943,7 +4575,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4953,9 +4585,6 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4975,7 +4604,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Min |= int64(b&0x7F) << shift + m.Min |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4994,7 +4623,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Max |= int64(b&0x7F) << shift + m.Max |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5013,7 +4642,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + v |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -5033,7 +4662,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + v |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -5053,7 +4682,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Base |= int64(b&0x7F) << shift + m.Base |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5072,7 +4701,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.BitDepth |= uint64(b&0x7F) << shift + m.BitDepth |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5091,7 +4720,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Scale |= int64(b&0x7F) << shift + m.Scale |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5105,9 +4734,6 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5136,7 +4762,7 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5164,7 +4790,7 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5174,9 +4800,6 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5191,9 +4814,6 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5222,7 +4842,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5250,7 +4870,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5260,9 +4880,6 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5282,7 +4899,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5292,9 +4909,6 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5314,7 +4928,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Block |= uint64(b&0x7F) << shift + m.Block |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5333,7 +4947,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Shard |= uint64(b&0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5352,7 +4966,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5362,9 +4976,6 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5379,9 +4990,6 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5410,7 +5018,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5436,7 +5044,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5453,7 +5061,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -5462,15 +5070,12 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -5490,7 +5095,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5512,7 +5117,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5529,7 +5134,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -5538,15 +5143,12 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -5566,7 +5168,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5585,9 +5187,6 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5616,7 +5215,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5642,7 +5241,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5659,7 +5258,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -5668,15 +5267,12 @@ func (m *Cache) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -5696,7 +5292,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5715,9 +5311,6 @@ func (m *Cache) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5746,7 +5339,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5774,7 +5367,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -5783,9 +5376,6 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5806,7 +5396,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5823,7 +5413,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLenmapkey |= uint64(b&0x7F) << shift + stringLenmapkey |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5833,9 +5423,6 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey < 0 { - return ErrInvalidLengthPrivate - } if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } @@ -5851,7 +5438,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - mapvalue |= uint64(b&0x7F) << shift + mapvalue |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5882,9 +5469,6 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5913,7 +5497,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5941,7 +5525,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5951,9 +5535,6 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5973,7 +5554,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Shard |= uint64(b&0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5992,7 +5573,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6002,9 +5583,6 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6019,9 +5597,6 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6050,7 +5625,7 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6078,7 +5653,7 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6088,9 +5663,6 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6105,9 +5677,6 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6136,7 +5705,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6164,7 +5733,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6174,9 +5743,6 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6196,7 +5762,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -6205,9 +5771,6 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6227,9 +5790,6 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6258,7 +5818,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6286,7 +5846,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6296,9 +5856,6 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6318,7 +5875,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6328,9 +5885,6 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6350,7 +5904,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -6359,9 +5913,6 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6381,9 +5932,6 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6412,7 +5960,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6440,7 +5988,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6450,9 +5998,6 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6472,7 +6017,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6482,9 +6027,6 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6499,9 +6041,6 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6530,7 +6069,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6558,7 +6097,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6568,9 +6107,6 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6590,7 +6126,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6600,9 +6136,6 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6622,7 +6155,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ShardID |= uint64(b&0x7F) << shift + m.ShardID |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6636,9 +6169,6 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6667,7 +6197,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6695,7 +6225,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6705,9 +6235,6 @@ func (m *Field) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6727,7 +6254,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -6736,9 +6263,6 @@ func (m *Field) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6763,7 +6287,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6773,9 +6297,6 @@ func (m *Field) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6790,9 +6311,6 @@ func (m *Field) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6821,7 +6339,7 @@ func (m *Schema) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6849,7 +6367,7 @@ func (m *Schema) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -6858,9 +6376,6 @@ func (m *Schema) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6878,9 +6393,6 @@ func (m *Schema) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6909,7 +6421,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6937,7 +6449,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6947,9 +6459,6 @@ func (m *Index) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6969,7 +6478,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -6978,9 +6487,6 @@ func (m *Index) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6998,9 +6504,6 @@ func (m *Index) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7029,7 +6532,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7057,7 +6560,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7067,9 +6570,6 @@ func (m *URI) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7089,7 +6589,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7099,9 +6599,6 @@ func (m *URI) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7121,7 +6618,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Port |= uint32(b&0x7F) << shift + m.Port |= (uint32(b) & 0x7F) << shift if b < 0x80 { break } @@ -7135,9 +6632,6 @@ func (m *URI) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7166,7 +6660,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7194,7 +6688,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7204,9 +6698,6 @@ func (m *Node) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7226,7 +6717,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -7235,9 +6726,6 @@ func (m *Node) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7262,7 +6750,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + v |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -7282,7 +6770,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7292,9 +6780,6 @@ func (m *Node) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7309,9 +6794,6 @@ func (m *Node) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7340,7 +6822,7 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7368,7 +6850,7 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7378,9 +6860,6 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7400,7 +6879,7 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7410,9 +6889,6 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7427,9 +6903,6 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7458,7 +6931,7 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7486,7 +6959,7 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Event |= uint32(b&0x7F) << shift + m.Event |= (uint32(b) & 0x7F) << shift if b < 0x80 { break } @@ -7505,7 +6978,7 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -7514,9 +6987,6 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7536,9 +7006,6 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7567,7 +7034,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7595,7 +7062,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -7604,9 +7071,6 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7631,7 +7095,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -7640,9 +7104,6 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7667,7 +7128,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -7676,9 +7137,6 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7696,9 +7154,6 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7727,7 +7182,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7755,7 +7210,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7765,9 +7220,6 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7787,7 +7239,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -7796,9 +7248,6 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7816,9 +7265,6 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7847,7 +7293,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7875,7 +7321,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7885,9 +7331,6 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7905,7 +7348,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7922,7 +7365,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -7931,15 +7374,12 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -7959,7 +7399,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -7978,9 +7418,6 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8009,7 +7446,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8037,7 +7474,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8047,9 +7484,6 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8069,7 +7503,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8079,9 +7513,6 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8101,7 +7532,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -8110,9 +7541,6 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8130,9 +7558,6 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8161,7 +7586,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8189,7 +7614,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8199,9 +7624,6 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8221,7 +7643,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8231,9 +7653,6 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8253,7 +7672,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Min |= int64(b&0x7F) << shift + m.Min |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8272,7 +7691,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Max |= int64(b&0x7F) << shift + m.Max |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8286,9 +7705,6 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8317,7 +7733,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8345,7 +7761,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8355,9 +7771,6 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8377,7 +7790,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8387,9 +7800,6 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8409,7 +7819,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8419,9 +7829,6 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8436,9 +7843,6 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8467,7 +7871,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8495,7 +7899,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8505,9 +7909,6 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8527,7 +7928,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8537,9 +7938,6 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8559,7 +7957,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8569,9 +7967,6 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8586,9 +7981,6 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8617,7 +8009,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8645,7 +8037,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.JobID |= int64(b&0x7F) << shift + m.JobID |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8664,7 +8056,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -8673,9 +8065,6 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8700,7 +8089,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -8709,9 +8098,6 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8736,7 +8122,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -8745,9 +8131,6 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8770,7 +8153,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -8779,9 +8162,6 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8806,7 +8186,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -8815,9 +8195,6 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8837,9 +8214,6 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8868,7 +8242,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8896,7 +8270,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -8905,9 +8279,6 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8932,7 +8303,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8942,9 +8313,6 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8964,7 +8332,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -8974,9 +8342,6 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8996,7 +8361,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -9006,9 +8371,6 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -9028,7 +8390,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Shard |= uint64(b&0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -9042,9 +8404,6 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -9073,7 +8432,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -9101,7 +8460,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.JobID |= int64(b&0x7F) << shift + m.JobID |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -9120,7 +8479,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -9129,9 +8488,6 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -9156,7 +8512,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -9166,9 +8522,6 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -9183,9 +8536,6 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -9214,7 +8564,7 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -9242,7 +8592,7 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -9251,9 +8601,6 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -9273,9 +8620,6 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -9304,7 +8648,7 @@ func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -9332,7 +8676,7 @@ func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -9341,9 +8685,6 @@ func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -9363,9 +8704,6 @@ func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -9394,7 +8732,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -9422,7 +8760,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -9432,9 +8770,6 @@ func (m *Topology) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -9454,7 +8789,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -9464,9 +8799,6 @@ func (m *Topology) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPrivate - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -9481,9 +8813,6 @@ func (m *Topology) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -9512,7 +8841,7 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -9535,9 +8864,6 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPrivate - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -9554,7 +8880,6 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { func skipPrivate(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 - depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { @@ -9586,8 +8911,10 @@ func skipPrivate(dAtA []byte) (n int, err error) { break } } + return iNdEx, nil case 1: iNdEx += 8 + return iNdEx, nil case 2: var length int for shift := uint(0); ; shift += 7 { @@ -9604,34 +8931,133 @@ func skipPrivate(dAtA []byte) (n int, err error) { break } } + iNdEx += length if length < 0 { return 0, ErrInvalidLengthPrivate } - iNdEx += length + return iNdEx, nil case 3: - depth++ - case 4: - if depth == 0 { - return 0, ErrUnexpectedEndOfGroupPrivate + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowPrivate + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipPrivate(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next } - depth-- + return iNdEx, nil + case 4: + return iNdEx, nil case 5: iNdEx += 4 + return iNdEx, nil default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } - if iNdEx < 0 { - return 0, ErrInvalidLengthPrivate - } - if depth == 0 { - return iNdEx, nil - } } - return 0, io.ErrUnexpectedEOF + panic("unreachable") } var ( - ErrInvalidLengthPrivate = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow") - ErrUnexpectedEndOfGroupPrivate = fmt.Errorf("proto: unexpected end of group") + ErrInvalidLengthPrivate = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow") ) + +func init() { proto.RegisterFile("private.proto", fileDescriptor_private_b229d027a4642df7) } + +var fileDescriptor_private_b229d027a4642df7 = []byte{ + // 1174 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0x45, + 0x18, 0x66, 0x0f, 0x71, 0xec, 0xdf, 0x71, 0x0e, 0xdb, 0x36, 0x6c, 0x0b, 0x0a, 0x66, 0x54, 0x51, + 0x53, 0x89, 0x50, 0xb5, 0x5c, 0x70, 0xaa, 0x54, 0x1c, 0x87, 0xb2, 0x94, 0x84, 0x32, 0x4e, 0x72, + 0xc7, 0xc5, 0xc4, 0x1e, 0x35, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x12, 0xf7, 0x82, 0x5b, 0x90, + 0x78, 0x01, 0x9e, 0xa0, 0xcf, 0xc2, 0x25, 0x8f, 0x80, 0xc2, 0x8b, 0xa0, 0xf9, 0x67, 0xf6, 0x60, + 0xc7, 0x21, 0x51, 0xe0, 0x6e, 0xfe, 0xd3, 0xf7, 0x9f, 0x7f, 0xaf, 0xa1, 0x35, 0x4e, 0xc2, 0x13, + 0x26, 0xf9, 0xe6, 0x38, 0x11, 0x52, 0x78, 0xf5, 0x30, 0x96, 0x3c, 0x89, 0x59, 0x44, 0x9e, 0x43, + 0x23, 0x88, 0x87, 0xfc, 0x6c, 0x87, 0x4b, 0xe6, 0x79, 0xe0, 0xbe, 0xe0, 0x93, 0xd4, 0x77, 0xda, + 0x56, 0xa7, 0x4e, 0xf1, 0xed, 0x7d, 0x00, 0xcb, 0x7b, 0x09, 0x1b, 0x1c, 0x6f, 0x9f, 0x85, 0xa9, + 0xe4, 0xf1, 0x80, 0xfb, 0x2e, 0x4a, 0x67, 0xb8, 0xe4, 0x8d, 0x0d, 0x4b, 0x5f, 0x87, 0x3c, 0x1a, + 0x7e, 0x3f, 0x96, 0xa1, 0x88, 0x53, 0xef, 0x5d, 0x68, 0x6c, 0xb1, 0xc1, 0x11, 0xdf, 0x9b, 0x8c, + 0x39, 0x22, 0x36, 0x68, 0xc9, 0x28, 0xa4, 0xfd, 0xf0, 0xb5, 0x46, 0x6c, 0xd1, 0x92, 0xe1, 0xb5, + 0xa1, 0xb9, 0x17, 0x8e, 0xf8, 0x0f, 0x19, 0x8b, 0x65, 0x36, 0xf2, 0x17, 0xd0, 0xba, 0xca, 0x52, + 0xa1, 0x22, 0x70, 0x1d, 0x45, 0xf8, 0xf6, 0x56, 0xc1, 0xd9, 0x09, 0x63, 0xbf, 0xd1, 0xb6, 0x3a, + 0x0e, 0x55, 0x4f, 0xe4, 0xb0, 0x33, 0x1f, 0x0c, 0x87, 0x9d, 0x15, 0x29, 0x36, 0xa7, 0x53, 0xdc, + 0x15, 0x7d, 0xc9, 0xe2, 0x21, 0x4b, 0x86, 0x07, 0x21, 0x3f, 0xf5, 0x97, 0x74, 0x8a, 0xd3, 0x5c, + 0x65, 0xdb, 0x65, 0x29, 0xf7, 0x5b, 0x08, 0x87, 0x6f, 0xef, 0x1e, 0xd4, 0xbb, 0xa1, 0xec, 0xf1, + 0xb1, 0x3c, 0xf2, 0x97, 0xdb, 0x56, 0xc7, 0xa5, 0x05, 0xed, 0xdd, 0x86, 0x85, 0xfe, 0x80, 0x45, + 0xdc, 0x5f, 0x41, 0x03, 0x4d, 0x10, 0x02, 0xcb, 0xc1, 0x68, 0x2c, 0x12, 0x49, 0x79, 0x3a, 0x16, + 0x71, 0x8a, 0x71, 0x6f, 0x27, 0x89, 0x6f, 0x61, 0x2a, 0xea, 0x49, 0x7e, 0x86, 0xd5, 0x6e, 0x24, + 0x06, 0xc7, 0x3d, 0x26, 0x19, 0xe5, 0x3f, 0x65, 0x3c, 0x95, 0x0a, 0x0d, 0x3b, 0x65, 0xf4, 0x34, + 0xa1, 0xb8, 0x58, 0x75, 0xdf, 0xd6, 0x5c, 0x24, 0x14, 0x17, 0xed, 0xb1, 0xee, 0x2e, 0xd5, 0x04, + 0xc6, 0x73, 0xc4, 0x92, 0x21, 0xd6, 0xdb, 0xa5, 0x9a, 0x50, 0x59, 0x61, 0xce, 0xba, 0xc8, 0xf8, + 0x26, 0x01, 0xac, 0x55, 0xfc, 0x9b, 0x30, 0xd7, 0xa1, 0x46, 0xc5, 0x69, 0xd0, 0x4b, 0x7d, 0xab, + 0xed, 0x74, 0x5c, 0x6a, 0x28, 0x6c, 0xa5, 0x88, 0xb2, 0x51, 0xac, 0x44, 0x36, 0x8a, 0x4a, 0x06, + 0xb9, 0x0b, 0x0b, 0xd8, 0x57, 0x95, 0x65, 0x69, 0xab, 0x9e, 0xe4, 0x17, 0x0b, 0x1a, 0x3b, 0xec, + 0x0c, 0xc3, 0x48, 0xbd, 0xa7, 0x50, 0xcf, 0xab, 0x8d, 0x4a, 0xcd, 0xc7, 0xef, 0x6f, 0xe6, 0x63, + 0xba, 0x59, 0xa8, 0x6d, 0xe6, 0x3a, 0xdb, 0xb1, 0x4c, 0x26, 0xb4, 0x30, 0xb9, 0xf7, 0x05, 0xb4, + 0xa6, 0x44, 0xca, 0xdf, 0x31, 0x9f, 0xe4, 0x55, 0x3d, 0xe6, 0x13, 0x95, 0xff, 0x09, 0x8b, 0x32, + 0x8e, 0xb5, 0x72, 0xa9, 0x26, 0x3e, 0xb7, 0x3f, 0xb5, 0xc8, 0x01, 0x78, 0x5b, 0x09, 0x67, 0x92, + 0xa3, 0x93, 0x1d, 0x9e, 0xa6, 0xec, 0x15, 0xbf, 0xbc, 0xe2, 0xba, 0x8a, 0x76, 0xb5, 0x8a, 0x45, + 0x1f, 0x9c, 0x4a, 0x1f, 0xc8, 0x43, 0xf0, 0x7a, 0x3c, 0xe2, 0x92, 0x9b, 0x1d, 0xfb, 0x17, 0x5c, + 0xd2, 0xcf, 0x63, 0xb8, 0x5a, 0xd7, 0x7b, 0x00, 0xae, 0x5a, 0x58, 0x0c, 0xa1, 0xf9, 0xf8, 0x56, + 0x59, 0xa7, 0x62, 0x97, 0x29, 0x2a, 0x90, 0x28, 0x07, 0xc5, 0x78, 0xae, 0x4c, 0x6c, 0xce, 0x28, + 0x3d, 0x34, 0xae, 0x1c, 0x74, 0xb5, 0x5e, 0xba, 0xaa, 0x2e, 0xbb, 0xf1, 0xf6, 0x2c, 0x4f, 0xf7, + 0xa6, 0xde, 0xc8, 0x00, 0xde, 0xd1, 0x08, 0x5f, 0x9d, 0xb0, 0x30, 0x62, 0x87, 0xd1, 0x35, 0x3b, + 0x32, 0x27, 0x70, 0x1f, 0x16, 0xd1, 0x36, 0xe8, 0x99, 0x2d, 0xc8, 0x49, 0xf2, 0xa3, 0xd1, 0x57, + 0xa3, 0xbf, 0xcb, 0x46, 0xdc, 0xa0, 0xe1, 0xbb, 0xc8, 0xd7, 0xbe, 0x3a, 0x5f, 0xe5, 0x58, 0xad, + 0x8b, 0x3a, 0x98, 0x8e, 0x72, 0x8c, 0x04, 0x79, 0x02, 0xb5, 0xfe, 0xe0, 0x88, 0x8f, 0x98, 0xf7, + 0x21, 0x2c, 0x62, 0x84, 0x3c, 0x35, 0x13, 0xbd, 0x32, 0xd3, 0x29, 0x9a, 0xcb, 0x49, 0xcf, 0x64, + 0x36, 0x37, 0xa6, 0x07, 0x50, 0x43, 0xef, 0xa9, 0xef, 0xce, 0xc2, 0x20, 0x9f, 0x1a, 0x31, 0xd9, + 0x06, 0x67, 0x9f, 0x06, 0x6a, 0x53, 0x31, 0x82, 0x1c, 0xc5, 0x50, 0x0a, 0xfb, 0x1b, 0x91, 0x4a, + 0x53, 0x27, 0x7c, 0x2b, 0xde, 0x4b, 0x91, 0x48, 0xac, 0x51, 0x8b, 0xe2, 0x9b, 0xa4, 0xe0, 0xee, + 0x8a, 0x21, 0xf7, 0x96, 0xc1, 0x0e, 0x7a, 0x06, 0xc3, 0x0e, 0x7a, 0xde, 0x7b, 0x08, 0x6f, 0x4a, + 0xd3, 0x2a, 0x83, 0xd8, 0xa7, 0x01, 0x45, 0xc7, 0xf7, 0xa1, 0x15, 0xa4, 0x5b, 0x42, 0x24, 0xc3, + 0x30, 0x66, 0x52, 0x24, 0xe6, 0x97, 0x64, 0x9a, 0x89, 0x1b, 0x24, 0x99, 0xd4, 0x77, 0xbf, 0x41, + 0x35, 0x41, 0x9e, 0xc1, 0xaa, 0x72, 0x8a, 0x44, 0xde, 0xef, 0x75, 0xa8, 0x29, 0x5e, 0x11, 0x84, + 0xa1, 0x4a, 0x04, 0xbb, 0x8a, 0xf0, 0x9d, 0x46, 0xd8, 0x3e, 0xe1, 0xb1, 0xac, 0x4c, 0x0c, 0xd2, + 0x08, 0xd0, 0xa2, 0x9a, 0xf0, 0x88, 0x4e, 0xd0, 0x64, 0xb2, 0x5c, 0x66, 0xa2, 0xb8, 0x14, 0x65, + 0xe4, 0x37, 0x0b, 0x20, 0x0f, 0x28, 0x4b, 0x0b, 0x13, 0xeb, 0x72, 0x13, 0xaf, 0x93, 0x77, 0xde, + 0x6c, 0xcb, 0x6a, 0xa9, 0xa5, 0xf9, 0x34, 0x9f, 0x8c, 0x8f, 0xcb, 0xc9, 0xd0, 0x2d, 0xbd, 0x33, + 0x33, 0x19, 0xda, 0x6b, 0x39, 0x1f, 0x2f, 0xa1, 0x59, 0xe1, 0xcf, 0x9d, 0x92, 0x8f, 0x8a, 0x29, + 0xb1, 0x67, 0x21, 0x91, 0x6f, 0x20, 0xf3, 0x59, 0x79, 0x01, 0xcd, 0x0a, 0x7b, 0x2e, 0x62, 0x07, + 0x56, 0xa6, 0xf7, 0x30, 0xbf, 0xef, 0xb3, 0x6c, 0x12, 0x42, 0x6b, 0x2b, 0xca, 0x52, 0xc9, 0x13, + 0x03, 0xa7, 0x7e, 0x14, 0x34, 0xa3, 0x68, 0x5e, 0xc9, 0x98, 0xdf, 0x3f, 0xef, 0x3e, 0x2c, 0xa8, + 0x32, 0xea, 0x75, 0xba, 0x58, 0x63, 0x2d, 0x24, 0x07, 0x50, 0xef, 0xf6, 0x83, 0xe7, 0x89, 0xc8, + 0xc6, 0x73, 0x83, 0xce, 0xbf, 0x0c, 0xec, 0x8b, 0x5f, 0x06, 0xce, 0x85, 0x2f, 0x03, 0xb7, 0xf8, + 0x32, 0x20, 0x7d, 0x58, 0xd3, 0xa7, 0x52, 0x6d, 0xf1, 0x4d, 0x0e, 0x4e, 0xfe, 0x43, 0xea, 0x54, + 0x7e, 0x48, 0xfb, 0xb0, 0xa6, 0xef, 0xd9, 0xff, 0x09, 0xfa, 0xc6, 0x86, 0x35, 0xca, 0xd3, 0xf0, + 0x35, 0x0f, 0xe2, 0x54, 0x26, 0xd9, 0x40, 0xdd, 0x24, 0x65, 0xff, 0xad, 0x38, 0x34, 0xd5, 0x76, + 0xa8, 0x26, 0xae, 0x33, 0xe9, 0xde, 0x23, 0x68, 0xce, 0xee, 0xec, 0x45, 0xd5, 0xaa, 0x8a, 0xf7, + 0x08, 0x16, 0xfb, 0x22, 0x4b, 0x06, 0xc5, 0xf8, 0x56, 0xee, 0xa4, 0x8e, 0x4c, 0x8b, 0x69, 0xae, + 0xe6, 0x3d, 0x9d, 0x19, 0x10, 0xbf, 0x86, 0x5e, 0xde, 0x2e, 0xed, 0xa6, 0xc4, 0x74, 0x66, 0x9c, + 0x3e, 0xa9, 0xee, 0xa2, 0xbf, 0x88, 0xb6, 0xb7, 0xa7, 0x23, 0x34, 0x86, 0x15, 0x3d, 0xf2, 0xab, + 0x05, 0x4b, 0xd5, 0x70, 0xae, 0xb5, 0xc4, 0x45, 0x77, 0xec, 0xb9, 0xdd, 0x71, 0xe6, 0x75, 0xc7, + 0x2d, 0xbb, 0x53, 0x7e, 0x1f, 0x2c, 0x54, 0xbe, 0x0f, 0xc8, 0x31, 0xdc, 0xbd, 0xd0, 0xb2, 0x2d, + 0x31, 0x1a, 0xab, 0xd9, 0xf8, 0x0f, 0xad, 0x53, 0xe7, 0x2d, 0x49, 0x4c, 0xd3, 0x1a, 0x54, 0x13, + 0xe4, 0x33, 0xb8, 0xd3, 0xe7, 0xb2, 0xd2, 0xb0, 0x7c, 0xf2, 0xda, 0xe0, 0xec, 0xf2, 0xd3, 0x4b, + 0xd2, 0x57, 0x22, 0xf2, 0x25, 0xf8, 0xfb, 0xe3, 0x21, 0x93, 0xfc, 0x46, 0xd6, 0x5d, 0xa8, 0xef, + 0x89, 0xb1, 0x88, 0xc4, 0xab, 0xc9, 0x15, 0x17, 0xc0, 0x87, 0x45, 0x7d, 0xcb, 0xf5, 0x49, 0x69, + 0xd0, 0x9c, 0x24, 0xb7, 0xd4, 0x70, 0x0f, 0x58, 0x34, 0xc8, 0x22, 0x15, 0x86, 0xfa, 0x76, 0x4c, + 0xbb, 0xab, 0x7f, 0x9c, 0x6f, 0x58, 0x7f, 0x9e, 0x6f, 0x58, 0x7f, 0x9d, 0x6f, 0x58, 0xbf, 0xff, + 0xbd, 0xf1, 0xd6, 0x61, 0x0d, 0xff, 0xc9, 0x3c, 0xf9, 0x27, 0x00, 0x00, 0xff, 0xff, 0xb4, 0x70, + 0x9a, 0xfe, 0xda, 0x0c, 0x00, 0x00, +} diff --git a/internal/public.pb.go b/internal/public.pb.go index 61506c081..6722e33dd 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -3,14 +3,13 @@ package internal -import ( - encoding_binary "encoding/binary" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - io "io" - math "math" - math_bits "math/bits" -) +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +import encoding_binary "encoding/binary" + +import io "io" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -21,12 +20,13 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type Row struct { - Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns,proto3" json:"Columns,omitempty"` - Keys []string `protobuf:"bytes,3,rep,name=Keys,proto3" json:"Keys,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,proto3" json:"Attrs,omitempty"` + Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns" json:"Columns,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` + Roaring []byte `protobuf:"bytes,4,opt,name=Roaring,proto3" json:"Roaring,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -36,7 +36,7 @@ func (m *Row) Reset() { *m = Row{} } func (m *Row) String() string { return proto.CompactTextString(m) } func (*Row) ProtoMessage() {} func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{0} + return fileDescriptor_public_17d2a22edfa80498, []int{0} } func (m *Row) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -46,15 +46,15 @@ func (m *Row) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Row.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *Row) XXX_Merge(src proto.Message) { - xxx_messageInfo_Row.Merge(m, src) +func (dst *Row) XXX_Merge(src proto.Message) { + xxx_messageInfo_Row.Merge(dst, src) } func (m *Row) XXX_Size() int { return m.Size() @@ -86,9 +86,71 @@ func (m *Row) GetAttrs() []*Attr { return nil } +func (m *Row) GetRoaring() []byte { + if m != nil { + return m.Roaring + } + return nil +} + +type SignedRow struct { + Pos *Row `protobuf:"bytes,1,opt,name=Pos" json:"Pos,omitempty"` + Neg *Row `protobuf:"bytes,2,opt,name=Neg" json:"Neg,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SignedRow) Reset() { *m = SignedRow{} } +func (m *SignedRow) String() string { return proto.CompactTextString(m) } +func (*SignedRow) ProtoMessage() {} +func (*SignedRow) Descriptor() ([]byte, []int) { + return fileDescriptor_public_17d2a22edfa80498, []int{1} +} +func (m *SignedRow) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SignedRow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SignedRow.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *SignedRow) XXX_Merge(src proto.Message) { + xxx_messageInfo_SignedRow.Merge(dst, src) +} +func (m *SignedRow) XXX_Size() int { + return m.Size() +} +func (m *SignedRow) XXX_DiscardUnknown() { + xxx_messageInfo_SignedRow.DiscardUnknown(m) +} + +var xxx_messageInfo_SignedRow proto.InternalMessageInfo + +func (m *SignedRow) GetPos() *Row { + if m != nil { + return m.Pos + } + return nil +} + +func (m *SignedRow) GetNeg() *Row { + if m != nil { + return m.Neg + } + return nil +} + type RowIdentifiers struct { - Rows []uint64 `protobuf:"varint,1,rep,packed,name=Rows,proto3" json:"Rows,omitempty"` - Keys []string `protobuf:"bytes,2,rep,name=Keys,proto3" json:"Keys,omitempty"` + Rows []uint64 `protobuf:"varint,1,rep,packed,name=Rows" json:"Rows,omitempty"` + Keys []string `protobuf:"bytes,2,rep,name=Keys" json:"Keys,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -98,7 +160,7 @@ func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } func (*RowIdentifiers) ProtoMessage() {} func (*RowIdentifiers) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{1} + return fileDescriptor_public_17d2a22edfa80498, []int{2} } func (m *RowIdentifiers) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -108,15 +170,15 @@ func (m *RowIdentifiers) XXX_Marshal(b []byte, deterministic bool) ([]byte, erro return xxx_messageInfo_RowIdentifiers.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *RowIdentifiers) XXX_Merge(src proto.Message) { - xxx_messageInfo_RowIdentifiers.Merge(m, src) +func (dst *RowIdentifiers) XXX_Merge(src proto.Message) { + xxx_messageInfo_RowIdentifiers.Merge(dst, src) } func (m *RowIdentifiers) XXX_Size() int { return m.Size() @@ -154,7 +216,7 @@ func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{2} + return fileDescriptor_public_17d2a22edfa80498, []int{3} } func (m *Pair) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -164,15 +226,15 @@ func (m *Pair) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Pair.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *Pair) XXX_Merge(src proto.Message) { - xxx_messageInfo_Pair.Merge(m, src) +func (dst *Pair) XXX_Merge(src proto.Message) { + xxx_messageInfo_Pair.Merge(dst, src) } func (m *Pair) XXX_Size() int { return m.Size() @@ -217,7 +279,7 @@ func (m *FieldRow) Reset() { *m = FieldRow{} } func (m *FieldRow) String() string { return proto.CompactTextString(m) } func (*FieldRow) ProtoMessage() {} func (*FieldRow) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{3} + return fileDescriptor_public_17d2a22edfa80498, []int{4} } func (m *FieldRow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -227,15 +289,15 @@ func (m *FieldRow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FieldRow.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *FieldRow) XXX_Merge(src proto.Message) { - xxx_messageInfo_FieldRow.Merge(m, src) +func (dst *FieldRow) XXX_Merge(src proto.Message) { + xxx_messageInfo_FieldRow.Merge(dst, src) } func (m *FieldRow) XXX_Size() int { return m.Size() @@ -268,7 +330,7 @@ func (m *FieldRow) GetRowKey() string { } type GroupCount struct { - Group []*FieldRow `protobuf:"bytes,1,rep,name=Group,proto3" json:"Group,omitempty"` + Group []*FieldRow `protobuf:"bytes,1,rep,name=Group" json:"Group,omitempty"` Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -279,7 +341,7 @@ func (m *GroupCount) Reset() { *m = GroupCount{} } func (m *GroupCount) String() string { return proto.CompactTextString(m) } func (*GroupCount) ProtoMessage() {} func (*GroupCount) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{4} + return fileDescriptor_public_17d2a22edfa80498, []int{5} } func (m *GroupCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -289,15 +351,15 @@ func (m *GroupCount) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_GroupCount.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *GroupCount) XXX_Merge(src proto.Message) { - xxx_messageInfo_GroupCount.Merge(m, src) +func (dst *GroupCount) XXX_Merge(src proto.Message) { + xxx_messageInfo_GroupCount.Merge(dst, src) } func (m *GroupCount) XXX_Size() int { return m.Size() @@ -334,7 +396,7 @@ func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} func (*ValCount) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{5} + return fileDescriptor_public_17d2a22edfa80498, []int{6} } func (m *ValCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -344,15 +406,15 @@ func (m *ValCount) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ValCount.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *ValCount) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValCount.Merge(m, src) +func (dst *ValCount) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValCount.Merge(dst, src) } func (m *ValCount) XXX_Size() int { return m.Size() @@ -380,7 +442,7 @@ func (m *ValCount) GetCount() int64 { type ColumnAttrSet struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,proto3" json:"Attrs,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -390,7 +452,7 @@ func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} func (*ColumnAttrSet) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{6} + return fileDescriptor_public_17d2a22edfa80498, []int{7} } func (m *ColumnAttrSet) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -400,15 +462,15 @@ func (m *ColumnAttrSet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error return xxx_messageInfo_ColumnAttrSet.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *ColumnAttrSet) XXX_Merge(src proto.Message) { - xxx_messageInfo_ColumnAttrSet.Merge(m, src) +func (dst *ColumnAttrSet) XXX_Merge(src proto.Message) { + xxx_messageInfo_ColumnAttrSet.Merge(dst, src) } func (m *ColumnAttrSet) XXX_Size() int { return m.Size() @@ -456,7 +518,7 @@ func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{7} + return fileDescriptor_public_17d2a22edfa80498, []int{8} } func (m *Attr) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -466,15 +528,15 @@ func (m *Attr) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Attr.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *Attr) XXX_Merge(src proto.Message) { - xxx_messageInfo_Attr.Merge(m, src) +func (dst *Attr) XXX_Merge(src proto.Message) { + xxx_messageInfo_Attr.Merge(dst, src) } func (m *Attr) XXX_Size() int { return m.Size() @@ -528,7 +590,7 @@ func (m *Attr) GetFloatValue() float64 { } type AttrMap struct { - Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs,proto3" json:"Attrs,omitempty"` + Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -538,7 +600,7 @@ func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} func (*AttrMap) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{8} + return fileDescriptor_public_17d2a22edfa80498, []int{9} } func (m *AttrMap) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -548,15 +610,15 @@ func (m *AttrMap) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_AttrMap.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *AttrMap) XXX_Merge(src proto.Message) { - xxx_messageInfo_AttrMap.Merge(m, src) +func (dst *AttrMap) XXX_Merge(src proto.Message) { + xxx_messageInfo_AttrMap.Merge(dst, src) } func (m *AttrMap) XXX_Size() int { return m.Size() @@ -576,11 +638,12 @@ func (m *AttrMap) GetAttrs() []*Attr { type QueryRequest struct { Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` - Shards []uint64 `protobuf:"varint,2,rep,packed,name=Shards,proto3" json:"Shards,omitempty"` + Shards []uint64 `protobuf:"varint,2,rep,packed,name=Shards" json:"Shards,omitempty"` ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"` ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"` + EmbeddedData []*Row `protobuf:"bytes,8,rep,name=EmbeddedData" json:"EmbeddedData,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -590,7 +653,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{9} + return fileDescriptor_public_17d2a22edfa80498, []int{10} } func (m *QueryRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -600,15 +663,15 @@ func (m *QueryRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_QueryRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *QueryRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryRequest.Merge(m, src) +func (dst *QueryRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryRequest.Merge(dst, src) } func (m *QueryRequest) XXX_Size() int { return m.Size() @@ -661,10 +724,17 @@ func (m *QueryRequest) GetExcludeColumns() bool { return false } +func (m *QueryRequest) GetEmbeddedData() []*Row { + if m != nil { + return m.EmbeddedData + } + return nil +} + type QueryResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` - Results []*QueryResult `protobuf:"bytes,2,rep,name=Results,proto3" json:"Results,omitempty"` - ColumnAttrSets []*ColumnAttrSet `protobuf:"bytes,3,rep,name=ColumnAttrSets,proto3" json:"ColumnAttrSets,omitempty"` + Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` + ColumnAttrSets []*ColumnAttrSet `protobuf:"bytes,3,rep,name=ColumnAttrSets" json:"ColumnAttrSets,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -674,7 +744,7 @@ func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} func (*QueryResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{10} + return fileDescriptor_public_17d2a22edfa80498, []int{11} } func (m *QueryResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -684,15 +754,15 @@ func (m *QueryResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error return xxx_messageInfo_QueryResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *QueryResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryResponse.Merge(m, src) +func (dst *QueryResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryResponse.Merge(dst, src) } func (m *QueryResponse) XXX_Size() int { return m.Size() @@ -726,14 +796,15 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { type QueryResult struct { Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` - Row *Row `protobuf:"bytes,1,opt,name=Row,proto3" json:"Row,omitempty"` + Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs,proto3" json:"Pairs,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` - ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount,proto3" json:"ValCount,omitempty"` - RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs,proto3" json:"RowIDs,omitempty"` - GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts,proto3" json:"GroupCounts,omitempty"` - RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers,proto3" json:"RowIdentifiers,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` + RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` + RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers" json:"RowIdentifiers,omitempty"` + SignedRow *SignedRow `protobuf:"bytes,10,opt,name=SignedRow" json:"SignedRow,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -743,7 +814,7 @@ func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} func (*QueryResult) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{11} + return fileDescriptor_public_17d2a22edfa80498, []int{12} } func (m *QueryResult) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -753,15 +824,15 @@ func (m *QueryResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_QueryResult.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *QueryResult) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryResult.Merge(m, src) +func (dst *QueryResult) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryResult.Merge(dst, src) } func (m *QueryResult) XXX_Size() int { return m.Size() @@ -835,15 +906,22 @@ func (m *QueryResult) GetRowIdentifiers() *RowIdentifiers { return nil } +func (m *QueryResult) GetSignedRow() *SignedRow { + if m != nil { + return m.SignedRow + } + return nil +} + type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` - RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs,proto3" json:"RowIDs,omitempty"` - ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs,proto3" json:"ColumnIDs,omitempty"` - RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys,proto3" json:"RowKeys,omitempty"` - ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys,proto3" json:"ColumnKeys,omitempty"` - Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps,proto3" json:"Timestamps,omitempty"` + RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` + RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys" json:"RowKeys,omitempty"` + ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` + Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps" json:"Timestamps,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -853,7 +931,7 @@ func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} func (*ImportRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{12} + return fileDescriptor_public_17d2a22edfa80498, []int{13} } func (m *ImportRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -863,15 +941,15 @@ func (m *ImportRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error return xxx_messageInfo_ImportRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *ImportRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportRequest.Merge(m, src) +func (dst *ImportRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportRequest.Merge(dst, src) } func (m *ImportRequest) XXX_Size() int { return m.Size() @@ -942,10 +1020,10 @@ type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` - ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs,proto3" json:"ColumnIDs,omitempty"` - ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys,proto3" json:"ColumnKeys,omitempty"` - Values []int64 `protobuf:"varint,6,rep,packed,name=Values,proto3" json:"Values,omitempty"` - FloatValues []float64 `protobuf:"fixed64,8,rep,packed,name=FloatValues,proto3" json:"FloatValues,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` + ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` + Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` + FloatValues []float64 `protobuf:"fixed64,8,rep,packed,name=FloatValues" json:"FloatValues,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -955,7 +1033,7 @@ func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} func (*ImportValueRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{13} + return fileDescriptor_public_17d2a22edfa80498, []int{14} } func (m *ImportValueRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -965,15 +1043,15 @@ func (m *ImportValueRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, return xxx_messageInfo_ImportValueRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *ImportValueRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportValueRequest.Merge(m, src) +func (dst *ImportValueRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportValueRequest.Merge(dst, src) } func (m *ImportValueRequest) XXX_Size() int { return m.Size() @@ -1036,7 +1114,7 @@ func (m *ImportValueRequest) GetFloatValues() []float64 { type TranslateKeysRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Keys []string `protobuf:"bytes,3,rep,name=Keys,proto3" json:"Keys,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1046,7 +1124,7 @@ func (m *TranslateKeysRequest) Reset() { *m = TranslateKeysRequest{} } func (m *TranslateKeysRequest) String() string { return proto.CompactTextString(m) } func (*TranslateKeysRequest) ProtoMessage() {} func (*TranslateKeysRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{14} + return fileDescriptor_public_17d2a22edfa80498, []int{15} } func (m *TranslateKeysRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1056,15 +1134,15 @@ func (m *TranslateKeysRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte return xxx_messageInfo_TranslateKeysRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *TranslateKeysRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_TranslateKeysRequest.Merge(m, src) +func (dst *TranslateKeysRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_TranslateKeysRequest.Merge(dst, src) } func (m *TranslateKeysRequest) XXX_Size() int { return m.Size() @@ -1097,7 +1175,7 @@ func (m *TranslateKeysRequest) GetKeys() []string { } type TranslateKeysResponse struct { - IDs []uint64 `protobuf:"varint,3,rep,packed,name=IDs,proto3" json:"IDs,omitempty"` + IDs []uint64 `protobuf:"varint,3,rep,packed,name=IDs" json:"IDs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1107,7 +1185,7 @@ func (m *TranslateKeysResponse) Reset() { *m = TranslateKeysResponse{} } func (m *TranslateKeysResponse) String() string { return proto.CompactTextString(m) } func (*TranslateKeysResponse) ProtoMessage() {} func (*TranslateKeysResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{15} + return fileDescriptor_public_17d2a22edfa80498, []int{16} } func (m *TranslateKeysResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1117,15 +1195,15 @@ func (m *TranslateKeysResponse) XXX_Marshal(b []byte, deterministic bool) ([]byt return xxx_messageInfo_TranslateKeysResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *TranslateKeysResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_TranslateKeysResponse.Merge(m, src) +func (dst *TranslateKeysResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_TranslateKeysResponse.Merge(dst, src) } func (m *TranslateKeysResponse) XXX_Size() int { return m.Size() @@ -1155,7 +1233,7 @@ func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestVi func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequestView) ProtoMessage() {} func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{16} + return fileDescriptor_public_17d2a22edfa80498, []int{17} } func (m *ImportRoaringRequestView) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1165,15 +1243,15 @@ func (m *ImportRoaringRequestView) XXX_Marshal(b []byte, deterministic bool) ([] return xxx_messageInfo_ImportRoaringRequestView.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *ImportRoaringRequestView) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportRoaringRequestView.Merge(m, src) +func (dst *ImportRoaringRequestView) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportRoaringRequestView.Merge(dst, src) } func (m *ImportRoaringRequestView) XXX_Size() int { return m.Size() @@ -1200,7 +1278,7 @@ func (m *ImportRoaringRequestView) GetData() []byte { type ImportRoaringRequest struct { Clear bool `protobuf:"varint,1,opt,name=Clear,proto3" json:"Clear,omitempty"` - Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views,proto3" json:"views,omitempty"` + Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views" json:"views,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1210,7 +1288,7 @@ func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} } func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequest) ProtoMessage() {} func (*ImportRoaringRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_413a91106d7bcce8, []int{17} + return fileDescriptor_public_17d2a22edfa80498, []int{18} } func (m *ImportRoaringRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1220,15 +1298,15 @@ func (m *ImportRoaringRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte return xxx_messageInfo_ImportRoaringRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } return b[:n], nil } } -func (m *ImportRoaringRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportRoaringRequest.Merge(m, src) +func (dst *ImportRoaringRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportRoaringRequest.Merge(dst, src) } func (m *ImportRoaringRequest) XXX_Size() int { return m.Size() @@ -1255,6 +1333,7 @@ func (m *ImportRoaringRequest) GetViews() []*ImportRoaringRequestView { func init() { proto.RegisterType((*Row)(nil), "internal.Row") + proto.RegisterType((*SignedRow)(nil), "internal.SignedRow") proto.RegisterType((*RowIdentifiers)(nil), "internal.RowIdentifiers") proto.RegisterType((*Pair)(nil), "internal.Pair") proto.RegisterType((*FieldRow)(nil), "internal.FieldRow") @@ -1273,73 +1352,10 @@ func init() { proto.RegisterType((*ImportRoaringRequestView)(nil), "internal.ImportRoaringRequestView") proto.RegisterType((*ImportRoaringRequest)(nil), "internal.ImportRoaringRequest") } - -func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } - -var fileDescriptor_413a91106d7bcce8 = []byte{ - // 889 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x5f, 0x8f, 0xdb, 0x44, - 0x10, 0x67, 0x63, 0x27, 0x71, 0x26, 0x97, 0x70, 0x5a, 0xa5, 0xc5, 0x42, 0x55, 0x88, 0x2c, 0x84, - 0xcc, 0xcb, 0x55, 0x0a, 0x12, 0xea, 0x13, 0x7f, 0xae, 0xb9, 0xa2, 0xa8, 0x70, 0x82, 0xb9, 0x53, - 0x10, 0x8f, 0xdb, 0x66, 0xdb, 0x5a, 0x72, 0xbc, 0xc1, 0x5e, 0x93, 0xde, 0xe7, 0xe0, 0x85, 0x8f, - 0xc0, 0x47, 0xe9, 0x13, 0x42, 0x7c, 0x02, 0x38, 0xbe, 0x08, 0xda, 0x59, 0xef, 0xad, 0x13, 0xda, - 0xea, 0x84, 0x78, 0x9b, 0xdf, 0xcc, 0xec, 0x78, 0x7e, 0xf3, 0x2f, 0x81, 0xa3, 0x6d, 0xfd, 0x24, - 0xcf, 0x9e, 0x9e, 0x6c, 0x4b, 0xa5, 0x15, 0x8f, 0xb2, 0x42, 0xcb, 0xb2, 0x10, 0x79, 0xf2, 0x03, - 0x04, 0xa8, 0x76, 0x3c, 0x86, 0xfe, 0x43, 0x95, 0xd7, 0x9b, 0xa2, 0x8a, 0xd9, 0x2c, 0x48, 0x43, - 0x74, 0x90, 0x73, 0x08, 0x1f, 0xcb, 0xab, 0x2a, 0x0e, 0x66, 0x41, 0x3a, 0x40, 0x92, 0xf9, 0x87, - 0xd0, 0xfd, 0x52, 0xeb, 0xb2, 0x8a, 0x3b, 0xb3, 0x20, 0x1d, 0xce, 0xc7, 0x27, 0x2e, 0xdc, 0x89, - 0x51, 0xa3, 0x35, 0x26, 0x0f, 0x60, 0x8c, 0x6a, 0xb7, 0x5c, 0xcb, 0x42, 0x67, 0xcf, 0x32, 0x59, - 0x52, 0x2c, 0x54, 0x3b, 0xf7, 0x09, 0x92, 0x6f, 0xe2, 0x77, 0x7c, 0xfc, 0xe4, 0x33, 0x08, 0xbf, - 0x15, 0x59, 0xc9, 0xc7, 0xd0, 0x59, 0x2e, 0x62, 0x36, 0x63, 0x69, 0x88, 0x9d, 0xe5, 0x82, 0x1f, - 0x43, 0xf0, 0x58, 0x5e, 0xc5, 0xc1, 0x8c, 0xa5, 0x03, 0x34, 0x22, 0x9f, 0x40, 0xf7, 0xa1, 0xaa, - 0x0b, 0x1d, 0x77, 0xc8, 0xc9, 0x82, 0xe4, 0x1c, 0xa2, 0x47, 0x99, 0xcc, 0xd7, 0x86, 0xd9, 0x04, - 0xba, 0x24, 0x53, 0x98, 0x01, 0x5a, 0x60, 0xb4, 0x26, 0xb7, 0x85, 0x7b, 0x47, 0x80, 0xdf, 0x85, - 0x1e, 0xaa, 0x9d, 0xff, 0x44, 0x83, 0x92, 0xaf, 0x01, 0xbe, 0x2a, 0x55, 0xbd, 0xa5, 0xe8, 0x3c, - 0x85, 0x2e, 0x21, 0xa2, 0x31, 0x9c, 0x73, 0xcf, 0xde, 0x7d, 0x14, 0xad, 0xc3, 0x1b, 0xb2, 0x9b, - 0x43, 0xb4, 0x12, 0xb9, 0x8d, 0x75, 0x0c, 0xc1, 0x4a, 0xe4, 0x94, 0x5b, 0x80, 0x46, 0xdc, 0x7f, - 0x13, 0xb8, 0x37, 0xdf, 0xc3, 0xc8, 0x36, 0xc4, 0x94, 0xf6, 0x42, 0xea, 0x5b, 0x94, 0xe6, 0x76, - 0x4d, 0xfa, 0x95, 0x41, 0x68, 0x24, 0x17, 0x80, 0xf9, 0x00, 0x1c, 0xc2, 0xcb, 0xab, 0xad, 0x6c, - 0x92, 0x27, 0x99, 0xcf, 0x60, 0x78, 0xa1, 0xcb, 0xac, 0x78, 0xbe, 0x12, 0x79, 0x2d, 0x9b, 0xcf, - 0xb5, 0x55, 0xfc, 0x7d, 0x88, 0x96, 0x85, 0xb6, 0xe6, 0x90, 0x28, 0xdc, 0x60, 0x7e, 0x0f, 0x06, - 0xa7, 0x4a, 0xe5, 0xd6, 0xd8, 0x9d, 0xb1, 0x34, 0x42, 0xaf, 0xe0, 0x53, 0x80, 0x47, 0xb9, 0x12, - 0xcd, 0xdb, 0xde, 0x8c, 0xa5, 0x0c, 0x5b, 0x9a, 0xe4, 0x3e, 0xf4, 0x4d, 0xa6, 0xdf, 0x88, 0xad, - 0xe7, 0xc6, 0xde, 0xc6, 0xed, 0x15, 0x83, 0xa3, 0xef, 0x6a, 0x59, 0x5e, 0xa1, 0xfc, 0xb1, 0x96, - 0x95, 0x36, 0xb5, 0x25, 0xec, 0x66, 0x81, 0x80, 0xe9, 0xfa, 0xc5, 0x0b, 0x51, 0xae, 0x6d, 0xa5, - 0x42, 0x6c, 0x90, 0xe1, 0xea, 0x6b, 0x5e, 0x11, 0xd7, 0x08, 0xdb, 0x2a, 0x9a, 0x17, 0xb9, 0x51, - 0xda, 0x91, 0x69, 0x10, 0x4f, 0xe1, 0xdd, 0xb3, 0x97, 0x4f, 0xf3, 0x7a, 0x2d, 0x51, 0xed, 0xec, - 0xeb, 0x1e, 0x39, 0x1c, 0xaa, 0xf9, 0x47, 0x30, 0x6e, 0x54, 0x6e, 0xfd, 0xfa, 0xe4, 0x78, 0xa0, - 0x4d, 0x7e, 0x66, 0x30, 0x6a, 0xa8, 0x54, 0x5b, 0x55, 0x54, 0xd2, 0xf4, 0xeb, 0xac, 0x2c, 0x5d, - 0xbf, 0xce, 0xca, 0x92, 0xdf, 0x87, 0x3e, 0xca, 0xaa, 0xce, 0xb5, 0x6b, 0xf9, 0x1d, 0x5f, 0x16, - 0xf7, 0xb6, 0xce, 0x35, 0x3a, 0x2f, 0xfe, 0x39, 0x8c, 0xf7, 0x86, 0xca, 0x2e, 0xf9, 0x70, 0xfe, - 0x9e, 0x7f, 0xb7, 0x67, 0xc7, 0x03, 0xf7, 0xe4, 0x8f, 0x0e, 0x0c, 0x5b, 0x91, 0x6f, 0x26, 0xc6, - 0x90, 0x1d, 0x35, 0x13, 0xf3, 0x01, 0x1d, 0x18, 0xca, 0x73, 0x38, 0x1f, 0xf9, 0xc8, 0x66, 0x4d, - 0xe8, 0xf4, 0x1c, 0x01, 0x3b, 0x6f, 0x66, 0x8c, 0x9d, 0x9b, 0xce, 0x9a, 0xd5, 0x77, 0xa9, 0xb4, - 0x3a, 0x6b, 0xd4, 0x68, 0x8d, 0x74, 0xae, 0x5e, 0x88, 0xe2, 0xb9, 0x5c, 0xd3, 0x8c, 0x45, 0xe8, - 0x20, 0x3f, 0xf1, 0xcb, 0x45, 0x4d, 0xd9, 0xdb, 0x4f, 0x67, 0x41, 0xbf, 0x80, 0x76, 0xe5, 0x97, - 0x0b, 0x53, 0x78, 0x6a, 0xbe, 0x45, 0xfc, 0x53, 0x18, 0xfa, 0x95, 0xaf, 0xe2, 0x88, 0xb2, 0x99, - 0xf8, 0x50, 0xde, 0x88, 0x6d, 0x47, 0xfe, 0xc5, 0xe1, 0xd1, 0x8b, 0x07, 0x94, 0x45, 0xbc, 0xc7, - 0xbc, 0x65, 0xc7, 0x03, 0xff, 0xe4, 0x2f, 0x06, 0xa3, 0xe5, 0x66, 0xab, 0x4a, 0xdd, 0x1a, 0xdb, - 0x65, 0xb1, 0x96, 0x2f, 0xdd, 0xd8, 0x12, 0xf0, 0x87, 0xad, 0x73, 0x70, 0xd8, 0x68, 0x7c, 0x69, - 0x5c, 0x43, 0xb4, 0xa0, 0xc5, 0x32, 0xdc, 0x63, 0x79, 0x0f, 0x06, 0xb6, 0xa5, 0xc6, 0xd4, 0x25, - 0x93, 0x57, 0x98, 0x2a, 0xdb, 0x03, 0x68, 0x8b, 0x33, 0x40, 0x07, 0xcd, 0xaa, 0x5a, 0x37, 0x32, - 0x46, 0x64, 0x6c, 0x69, 0x8c, 0xfd, 0x32, 0xdb, 0xc8, 0x4a, 0x8b, 0xcd, 0xd6, 0xcc, 0x7e, 0x90, - 0x06, 0xd8, 0xd2, 0x24, 0xbf, 0x31, 0xe0, 0x96, 0x23, 0xad, 0xf6, 0xff, 0x47, 0xf4, 0xed, 0x84, - 0xf6, 0xd3, 0xee, 0xff, 0x2b, 0xed, 0xbb, 0xd0, 0xa3, 0x7c, 0x5c, 0xca, 0x0d, 0x32, 0x97, 0xc0, - 0xdf, 0x21, 0xcb, 0x97, 0x61, 0x5b, 0x95, 0xac, 0x60, 0x72, 0x59, 0x8a, 0xa2, 0xca, 0x85, 0x96, - 0x26, 0xd4, 0x7f, 0x61, 0xf4, 0x9a, 0x5f, 0xda, 0xe4, 0x63, 0xb8, 0x73, 0x10, 0xd7, 0xaf, 0xbf, - 0xa1, 0x18, 0x10, 0x45, 0x23, 0x26, 0xa7, 0x10, 0x37, 0x63, 0xa3, 0x84, 0x39, 0xc7, 0x4d, 0x0a, - 0xab, 0x4c, 0xee, 0x4c, 0xe8, 0x73, 0xb1, 0x91, 0x4d, 0x16, 0x24, 0x1b, 0xdd, 0x42, 0x68, 0x41, - 0x39, 0x1c, 0x21, 0xc9, 0xc9, 0x33, 0x98, 0xbc, 0x2e, 0x06, 0xfd, 0x28, 0xe5, 0x52, 0xd8, 0x73, - 0x13, 0xa1, 0x05, 0xfc, 0x01, 0x74, 0x7f, 0xca, 0xe4, 0xce, 0x9d, 0x9b, 0xc4, 0x8f, 0xf8, 0x9b, - 0x12, 0x41, 0xfb, 0xe0, 0xf4, 0xf8, 0xd5, 0xf5, 0x94, 0xfd, 0x7e, 0x3d, 0x65, 0x7f, 0x5e, 0x4f, - 0xd9, 0x2f, 0x7f, 0x4f, 0xdf, 0x79, 0xd2, 0xa3, 0x3f, 0x26, 0x9f, 0xfc, 0x13, 0x00, 0x00, 0xff, - 0xff, 0xbf, 0xdd, 0x78, 0x1b, 0xa8, 0x08, 0x00, 0x00, -} - func (m *Row) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -1347,42 +1363,10 @@ func (m *Row) Marshal() (dAtA []byte, err error) { } func (m *Row) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Row) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.Keys) > 0 { - for iNdEx := len(m.Keys) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.Keys[iNdEx]) - copy(dAtA[i:], m.Keys[iNdEx]) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Keys[iNdEx]))) - i-- - dAtA[i] = 0x1a - } - } - if len(m.Attrs) > 0 { - for iNdEx := len(m.Attrs) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Attrs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPublic(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - } if len(m.Columns) > 0 { dAtA2 := make([]byte, len(m.Columns)*10) var j1 int @@ -1395,19 +1379,95 @@ func (m *Row) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA2[j1] = uint8(num) j1++ } - i -= j1 - copy(dAtA[i:], dAtA2[:j1]) - i = encodeVarintPublic(dAtA, i, uint64(j1)) - i-- dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(j1)) + i += copy(dAtA[i:], dAtA2[:j1]) } - return len(dAtA) - i, nil + if len(m.Attrs) > 0 { + for _, msg := range m.Attrs { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if len(m.Keys) > 0 { + for _, s := range m.Keys { + dAtA[i] = 0x1a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if len(m.Roaring) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Roaring))) + i += copy(dAtA[i:], m.Roaring) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *SignedRow) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SignedRow) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Pos != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Pos.Size())) + n3, err := m.Pos.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n3 + } + if m.Neg != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Neg.Size())) + n4, err := m.Neg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n4 + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *RowIdentifiers) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -1415,451 +1475,14 @@ func (m *RowIdentifiers) Marshal() (dAtA []byte, err error) { } func (m *RowIdentifiers) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *RowIdentifiers) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.Keys) > 0 { - for iNdEx := len(m.Keys) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.Keys[iNdEx]) - copy(dAtA[i:], m.Keys[iNdEx]) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Keys[iNdEx]))) - i-- - dAtA[i] = 0x12 - } - } if len(m.Rows) > 0 { - dAtA4 := make([]byte, len(m.Rows)*10) - var j3 int - for _, num := range m.Rows { - for num >= 1<<7 { - dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j3++ - } - dAtA4[j3] = uint8(num) - j3++ - } - i -= j3 - copy(dAtA[i:], dAtA4[:j3]) - i = encodeVarintPublic(dAtA, i, uint64(j3)) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *Pair) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Pair) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Pair) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.Key) > 0 { - i -= len(m.Key) - copy(dAtA[i:], m.Key) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) - i-- - dAtA[i] = 0x1a - } - if m.Count != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.Count)) - i-- - dAtA[i] = 0x10 - } - if m.ID != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.ID)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *FieldRow) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *FieldRow) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *FieldRow) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.RowKey) > 0 { - i -= len(m.RowKey) - copy(dAtA[i:], m.RowKey) - i = encodeVarintPublic(dAtA, i, uint64(len(m.RowKey))) - i-- - dAtA[i] = 0x1a - } - if m.RowID != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.RowID)) - i-- - dAtA[i] = 0x10 - } - if len(m.Field) > 0 { - i -= len(m.Field) - copy(dAtA[i:], m.Field) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *GroupCount) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *GroupCount) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *GroupCount) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if m.Count != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.Count)) - i-- - dAtA[i] = 0x10 - } - if len(m.Group) > 0 { - for iNdEx := len(m.Group) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Group[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPublic(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0xa - } - } - return len(dAtA) - i, nil -} - -func (m *ValCount) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ValCount) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ValCount) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if m.Count != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.Count)) - i-- - dAtA[i] = 0x10 - } - if m.Val != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.Val)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *ColumnAttrSet) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ColumnAttrSet) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ColumnAttrSet) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.Key) > 0 { - i -= len(m.Key) - copy(dAtA[i:], m.Key) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) - i-- - dAtA[i] = 0x1a - } - if len(m.Attrs) > 0 { - for iNdEx := len(m.Attrs) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Attrs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPublic(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - } - if m.ID != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.ID)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *Attr) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Attr) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Attr) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if m.FloatValue != 0 { - i -= 8 - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) - i-- - dAtA[i] = 0x31 - } - if m.BoolValue { - i-- - if m.BoolValue { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x28 - } - if m.IntValue != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.IntValue)) - i-- - dAtA[i] = 0x20 - } - if len(m.StringValue) > 0 { - i -= len(m.StringValue) - copy(dAtA[i:], m.StringValue) - i = encodeVarintPublic(dAtA, i, uint64(len(m.StringValue))) - i-- - dAtA[i] = 0x1a - } - if m.Type != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.Type)) - i-- - dAtA[i] = 0x10 - } - if len(m.Key) > 0 { - i -= len(m.Key) - copy(dAtA[i:], m.Key) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *AttrMap) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *AttrMap) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *AttrMap) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.Attrs) > 0 { - for iNdEx := len(m.Attrs) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Attrs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPublic(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0xa - } - } - return len(dAtA) - i, nil -} - -func (m *QueryRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *QueryRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if m.ExcludeColumns { - i-- - if m.ExcludeColumns { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x38 - } - if m.ExcludeRowAttrs { - i-- - if m.ExcludeRowAttrs { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x30 - } - if m.Remote { - i-- - if m.Remote { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x28 - } - if m.ColumnAttrs { - i-- - if m.ColumnAttrs { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x18 - } - if len(m.Shards) > 0 { - dAtA6 := make([]byte, len(m.Shards)*10) + dAtA6 := make([]byte, len(m.Rows)*10) var j5 int - for _, num := range m.Shards { + for _, num := range m.Rows { for num >= 1<<7 { dAtA6[j5] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -1868,26 +1491,412 @@ func (m *QueryRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA6[j5] = uint8(num) j5++ } - i -= j5 - copy(dAtA[i:], dAtA6[:j5]) - i = encodeVarintPublic(dAtA, i, uint64(j5)) - i-- - dAtA[i] = 0x12 - } - if len(m.Query) > 0 { - i -= len(m.Query) - copy(dAtA[i:], m.Query) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Query))) - i-- dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(j5)) + i += copy(dAtA[i:], dAtA6[:j5]) } - return len(dAtA) - i, nil + if len(m.Keys) > 0 { + for _, s := range m.Keys { + dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *Pair) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Pair) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.ID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.ID)) + } + if m.Count != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Count)) + } + if len(m.Key) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) + i += copy(dAtA[i:], m.Key) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *FieldRow) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FieldRow) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Field) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + if m.RowID != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.RowID)) + } + if len(m.RowKey) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.RowKey))) + i += copy(dAtA[i:], m.RowKey) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *GroupCount) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GroupCount) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Group) > 0 { + for _, msg := range m.Group { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.Count != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Count)) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *ValCount) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ValCount) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Val != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Val)) + } + if m.Count != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Count)) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *ColumnAttrSet) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ColumnAttrSet) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.ID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.ID)) + } + if len(m.Attrs) > 0 { + for _, msg := range m.Attrs { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if len(m.Key) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) + i += copy(dAtA[i:], m.Key) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *Attr) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Attr) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Key) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) + i += copy(dAtA[i:], m.Key) + } + if m.Type != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Type)) + } + if len(m.StringValue) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.StringValue))) + i += copy(dAtA[i:], m.StringValue) + } + if m.IntValue != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.IntValue)) + } + if m.BoolValue { + dAtA[i] = 0x28 + i++ + if m.BoolValue { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.FloatValue != 0 { + dAtA[i] = 0x31 + i++ + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + i += 8 + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *AttrMap) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *AttrMap) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Attrs) > 0 { + for _, msg := range m.Attrs { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *QueryRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Query) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Query))) + i += copy(dAtA[i:], m.Query) + } + if len(m.Shards) > 0 { + dAtA8 := make([]byte, len(m.Shards)*10) + var j7 int + for _, num := range m.Shards { + for num >= 1<<7 { + dAtA8[j7] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j7++ + } + dAtA8[j7] = uint8(num) + j7++ + } + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(j7)) + i += copy(dAtA[i:], dAtA8[:j7]) + } + if m.ColumnAttrs { + dAtA[i] = 0x18 + i++ + if m.ColumnAttrs { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.Remote { + dAtA[i] = 0x28 + i++ + if m.Remote { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.ExcludeRowAttrs { + dAtA[i] = 0x30 + i++ + if m.ExcludeRowAttrs { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.ExcludeColumns { + dAtA[i] = 0x38 + i++ + if m.ExcludeColumns { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if len(m.EmbeddedData) > 0 { + for _, msg := range m.EmbeddedData { + dAtA[i] = 0x42 + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *QueryResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -1895,61 +1904,50 @@ func (m *QueryResponse) Marshal() (dAtA []byte, err error) { } func (m *QueryResponse) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *QueryResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.ColumnAttrSets) > 0 { - for iNdEx := len(m.ColumnAttrSets) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.ColumnAttrSets[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPublic(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x1a - } + if len(m.Err) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Err))) + i += copy(dAtA[i:], m.Err) } if len(m.Results) > 0 { - for iNdEx := len(m.Results) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Results[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPublic(dAtA, i, uint64(size)) - } - i-- + for _, msg := range m.Results { dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n } } - if len(m.Err) > 0 { - i -= len(m.Err) - copy(dAtA[i:], m.Err) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Err))) - i-- - dAtA[i] = 0xa + if len(m.ColumnAttrSets) > 0 { + for _, msg := range m.ColumnAttrSets { + dAtA[i] = 0x1a + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } } - return len(dAtA) - i, nil + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *QueryResult) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -1957,128 +1955,121 @@ func (m *QueryResult) Marshal() (dAtA []byte, err error) { } func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *QueryResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) + if m.Row != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Row.Size())) + n9, err := m.Row.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n9 } - if m.RowIdentifiers != nil { - { - size, err := m.RowIdentifiers.MarshalToSizedBuffer(dAtA[:i]) + if m.N != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.N)) + } + if len(m.Pairs) > 0 { + for _, msg := range m.Pairs { + dAtA[i] = 0x1a + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i -= size - i = encodeVarintPublic(dAtA, i, uint64(size)) + i += n } - i-- - dAtA[i] = 0x4a - } - if len(m.GroupCounts) > 0 { - for iNdEx := len(m.GroupCounts) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.GroupCounts[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPublic(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x42 - } - } - if len(m.RowIDs) > 0 { - dAtA9 := make([]byte, len(m.RowIDs)*10) - var j8 int - for _, num := range m.RowIDs { - for num >= 1<<7 { - dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j8++ - } - dAtA9[j8] = uint8(num) - j8++ - } - i -= j8 - copy(dAtA[i:], dAtA9[:j8]) - i = encodeVarintPublic(dAtA, i, uint64(j8)) - i-- - dAtA[i] = 0x3a - } - if m.Type != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.Type)) - i-- - dAtA[i] = 0x30 - } - if m.ValCount != nil { - { - size, err := m.ValCount.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPublic(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x2a } if m.Changed { - i-- + dAtA[i] = 0x20 + i++ if m.Changed { dAtA[i] = 1 } else { dAtA[i] = 0 } - i-- - dAtA[i] = 0x20 + i++ } - if len(m.Pairs) > 0 { - for iNdEx := len(m.Pairs) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Pairs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPublic(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x1a + if m.ValCount != nil { + dAtA[i] = 0x2a + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.ValCount.Size())) + n10, err := m.ValCount.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } + i += n10 } - if m.N != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.N)) - i-- - dAtA[i] = 0x10 + if m.Type != 0 { + dAtA[i] = 0x30 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Type)) } - if m.Row != nil { - { - size, err := m.Row.MarshalToSizedBuffer(dAtA[:i]) + if len(m.RowIDs) > 0 { + dAtA12 := make([]byte, len(m.RowIDs)*10) + var j11 int + for _, num := range m.RowIDs { + for num >= 1<<7 { + dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j11++ + } + dAtA12[j11] = uint8(num) + j11++ + } + dAtA[i] = 0x3a + i++ + i = encodeVarintPublic(dAtA, i, uint64(j11)) + i += copy(dAtA[i:], dAtA12[:j11]) + } + if len(m.GroupCounts) > 0 { + for _, msg := range m.GroupCounts { + dAtA[i] = 0x42 + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i -= size - i = encodeVarintPublic(dAtA, i, uint64(size)) + i += n } - i-- - dAtA[i] = 0xa } - return len(dAtA) - i, nil + if m.RowIdentifiers != nil { + dAtA[i] = 0x4a + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.RowIdentifiers.Size())) + n13, err := m.RowIdentifiers.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n13 + } + if m.SignedRow != nil { + dAtA[i] = 0x52 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.SignedRow.Size())) + n14, err := m.SignedRow.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n14 + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *ImportRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2086,161 +2077,65 @@ func (m *ImportRequest) Marshal() (dAtA []byte, err error) { } func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ImportRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.ColumnKeys) > 0 { - for iNdEx := len(m.ColumnKeys) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.ColumnKeys[iNdEx]) - copy(dAtA[i:], m.ColumnKeys[iNdEx]) - i = encodeVarintPublic(dAtA, i, uint64(len(m.ColumnKeys[iNdEx]))) - i-- - dAtA[i] = 0x42 - } - } - if len(m.RowKeys) > 0 { - for iNdEx := len(m.RowKeys) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.RowKeys[iNdEx]) - copy(dAtA[i:], m.RowKeys[iNdEx]) - i = encodeVarintPublic(dAtA, i, uint64(len(m.RowKeys[iNdEx]))) - i-- - dAtA[i] = 0x3a - } - } - if len(m.Timestamps) > 0 { - dAtA13 := make([]byte, len(m.Timestamps)*10) - var j12 int - for _, num1 := range m.Timestamps { - num := uint64(num1) - for num >= 1<<7 { - dAtA13[j12] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j12++ - } - dAtA13[j12] = uint8(num) - j12++ - } - i -= j12 - copy(dAtA[i:], dAtA13[:j12]) - i = encodeVarintPublic(dAtA, i, uint64(j12)) - i-- - dAtA[i] = 0x32 - } - if len(m.ColumnIDs) > 0 { - dAtA15 := make([]byte, len(m.ColumnIDs)*10) - var j14 int - for _, num := range m.ColumnIDs { - for num >= 1<<7 { - dAtA15[j14] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j14++ - } - dAtA15[j14] = uint8(num) - j14++ - } - i -= j14 - copy(dAtA[i:], dAtA15[:j14]) - i = encodeVarintPublic(dAtA, i, uint64(j14)) - i-- - dAtA[i] = 0x2a - } - if len(m.RowIDs) > 0 { - dAtA17 := make([]byte, len(m.RowIDs)*10) - var j16 int - for _, num := range m.RowIDs { - for num >= 1<<7 { - dAtA17[j16] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j16++ - } - dAtA17[j16] = uint8(num) - j16++ - } - i -= j16 - copy(dAtA[i:], dAtA17[:j16]) - i = encodeVarintPublic(dAtA, i, uint64(j16)) - i-- - dAtA[i] = 0x22 - } - if m.Shard != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) - i-- - dAtA[i] = 0x18 + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if len(m.Field) > 0 { - i -= len(m.Field) - copy(dAtA[i:], m.Field) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } - if len(m.Index) > 0 { - i -= len(m.Index) - copy(dAtA[i:], m.Index) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) - i-- - dAtA[i] = 0xa + if m.Shard != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } - return len(dAtA) - i, nil -} - -func (m *ImportValueRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.FloatValues) > 0 { - for iNdEx := len(m.FloatValues) - 1; iNdEx >= 0; iNdEx-- { - f18 := math.Float64bits(float64(m.FloatValues[iNdEx])) - i -= 8 - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f18)) + if len(m.RowIDs) > 0 { + dAtA16 := make([]byte, len(m.RowIDs)*10) + var j15 int + for _, num := range m.RowIDs { + for num >= 1<<7 { + dAtA16[j15] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j15++ + } + dAtA16[j15] = uint8(num) + j15++ } - i = encodeVarintPublic(dAtA, i, uint64(len(m.FloatValues)*8)) - i-- - dAtA[i] = 0x42 + dAtA[i] = 0x22 + i++ + i = encodeVarintPublic(dAtA, i, uint64(j15)) + i += copy(dAtA[i:], dAtA16[:j15]) } - if len(m.ColumnKeys) > 0 { - for iNdEx := len(m.ColumnKeys) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.ColumnKeys[iNdEx]) - copy(dAtA[i:], m.ColumnKeys[iNdEx]) - i = encodeVarintPublic(dAtA, i, uint64(len(m.ColumnKeys[iNdEx]))) - i-- - dAtA[i] = 0x3a + if len(m.ColumnIDs) > 0 { + dAtA18 := make([]byte, len(m.ColumnIDs)*10) + var j17 int + for _, num := range m.ColumnIDs { + for num >= 1<<7 { + dAtA18[j17] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j17++ + } + dAtA18[j17] = uint8(num) + j17++ } + dAtA[i] = 0x2a + i++ + i = encodeVarintPublic(dAtA, i, uint64(j17)) + i += copy(dAtA[i:], dAtA18[:j17]) } - if len(m.Values) > 0 { - dAtA20 := make([]byte, len(m.Values)*10) + if len(m.Timestamps) > 0 { + dAtA20 := make([]byte, len(m.Timestamps)*10) var j19 int - for _, num1 := range m.Values { + for _, num1 := range m.Timestamps { num := uint64(num1) for num >= 1<<7 { dAtA20[j19] = uint8(uint64(num)&0x7f | 0x80) @@ -2250,11 +2145,78 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA20[j19] = uint8(num) j19++ } - i -= j19 - copy(dAtA[i:], dAtA20[:j19]) - i = encodeVarintPublic(dAtA, i, uint64(j19)) - i-- dAtA[i] = 0x32 + i++ + i = encodeVarintPublic(dAtA, i, uint64(j19)) + i += copy(dAtA[i:], dAtA20[:j19]) + } + if len(m.RowKeys) > 0 { + for _, s := range m.RowKeys { + dAtA[i] = 0x3a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if len(m.ColumnKeys) > 0 { + for _, s := range m.ColumnKeys { + dAtA[i] = 0x42 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *ImportValueRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Field) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + if m.Shard != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.ColumnIDs) > 0 { dAtA22 := make([]byte, len(m.ColumnIDs)*10) @@ -2268,112 +2230,16 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA22[j21] = uint8(num) j21++ } - i -= j21 - copy(dAtA[i:], dAtA22[:j21]) - i = encodeVarintPublic(dAtA, i, uint64(j21)) - i-- dAtA[i] = 0x2a + i++ + i = encodeVarintPublic(dAtA, i, uint64(j21)) + i += copy(dAtA[i:], dAtA22[:j21]) } - if m.Shard != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) - i-- - dAtA[i] = 0x18 - } - if len(m.Field) > 0 { - i -= len(m.Field) - copy(dAtA[i:], m.Field) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) - i-- - dAtA[i] = 0x12 - } - if len(m.Index) > 0 { - i -= len(m.Index) - copy(dAtA[i:], m.Index) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *TranslateKeysRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *TranslateKeysRequest) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *TranslateKeysRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.Keys) > 0 { - for iNdEx := len(m.Keys) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.Keys[iNdEx]) - copy(dAtA[i:], m.Keys[iNdEx]) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Keys[iNdEx]))) - i-- - dAtA[i] = 0x1a - } - } - if len(m.Field) > 0 { - i -= len(m.Field) - copy(dAtA[i:], m.Field) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) - i-- - dAtA[i] = 0x12 - } - if len(m.Index) > 0 { - i -= len(m.Index) - copy(dAtA[i:], m.Index) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *TranslateKeysResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *TranslateKeysResponse) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *TranslateKeysResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.IDs) > 0 { - dAtA24 := make([]byte, len(m.IDs)*10) + if len(m.Values) > 0 { + dAtA24 := make([]byte, len(m.Values)*10) var j23 int - for _, num := range m.IDs { + for _, num1 := range m.Values { + num := uint64(num1) for num >= 1<<7 { dAtA24[j23] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2382,19 +2248,132 @@ func (m *TranslateKeysResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA24[j23] = uint8(num) j23++ } - i -= j23 - copy(dAtA[i:], dAtA24[:j23]) + dAtA[i] = 0x32 + i++ i = encodeVarintPublic(dAtA, i, uint64(j23)) - i-- - dAtA[i] = 0x1a + i += copy(dAtA[i:], dAtA24[:j23]) } - return len(dAtA) - i, nil + if len(m.ColumnKeys) > 0 { + for _, s := range m.ColumnKeys { + dAtA[i] = 0x3a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if len(m.FloatValues) > 0 { + dAtA[i] = 0x42 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.FloatValues)*8)) + for _, num := range m.FloatValues { + f25 := math.Float64bits(float64(num)) + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f25)) + i += 8 + } + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *TranslateKeysRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TranslateKeysRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Field) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + if len(m.Keys) > 0 { + for _, s := range m.Keys { + dAtA[i] = 0x1a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *TranslateKeysResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TranslateKeysResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.IDs) > 0 { + dAtA27 := make([]byte, len(m.IDs)*10) + var j26 int + for _, num := range m.IDs { + for num >= 1<<7 { + dAtA27[j26] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j26++ + } + dAtA27[j26] = uint8(num) + j26++ + } + dAtA[i] = 0x1a + i++ + i = encodeVarintPublic(dAtA, i, uint64(j26)) + i += copy(dAtA[i:], dAtA27[:j26]) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func (m *ImportRoaringRequestView) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2402,40 +2381,32 @@ func (m *ImportRoaringRequestView) Marshal() (dAtA []byte, err error) { } func (m *ImportRoaringRequestView) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ImportRoaringRequestView) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) } if len(m.Data) > 0 { - i -= len(m.Data) - copy(dAtA[i:], m.Data) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Data))) - i-- dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Data))) + i += copy(dAtA[i:], m.Data) } - if len(m.Name) > 0 { - i -= len(m.Name) - copy(dAtA[i:], m.Name) - i = encodeVarintPublic(dAtA, i, uint64(len(m.Name))) - i-- - dAtA[i] = 0xa + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) } - return len(dAtA) - i, nil + return i, nil } func (m *ImportRoaringRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -2443,56 +2414,46 @@ func (m *ImportRoaringRequest) Marshal() (dAtA []byte, err error) { } func (m *ImportRoaringRequest) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ImportRoaringRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.Views) > 0 { - for iNdEx := len(m.Views) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Views[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPublic(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - } if m.Clear { - i-- + dAtA[i] = 0x8 + i++ if m.Clear { dAtA[i] = 1 } else { dAtA[i] = 0 } - i-- - dAtA[i] = 0x8 + i++ } - return len(dAtA) - i, nil + if len(m.Views) > 0 { + for _, msg := range m.Views { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil } func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { - offset -= sovPublic(v) - base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) - return base + return offset + 1 } func (m *Row) Size() (n int) { if m == nil { @@ -2519,6 +2480,30 @@ func (m *Row) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + l = len(m.Roaring) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *SignedRow) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Pos != nil { + l = m.Pos.Size() + n += 1 + l + sovPublic(uint64(l)) + } + if m.Neg != nil { + l = m.Neg.Size() + n += 1 + l + sovPublic(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -2738,6 +2723,12 @@ func (m *QueryRequest) Size() (n int) { if m.ExcludeColumns { n += 2 } + if len(m.EmbeddedData) > 0 { + for _, e := range m.EmbeddedData { + l = e.Size() + n += 1 + l + sovPublic(uint64(l)) + } + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -2818,6 +2809,10 @@ func (m *QueryResult) Size() (n int) { l = m.RowIdentifiers.Size() n += 1 + l + sovPublic(uint64(l)) } + if m.SignedRow != nil { + l = m.SignedRow.Size() + n += 1 + l + sovPublic(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -3013,7 +3008,14 @@ func (m *ImportRoaringRequest) Size() (n int) { } func sovPublic(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n } func sozPublic(x uint64) (n int) { return sovPublic(uint64((x << 1) ^ uint64((int64(x) >> 63)))) @@ -3033,7 +3035,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3059,7 +3061,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3076,7 +3078,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -3085,15 +3087,12 @@ func (m *Row) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -3113,7 +3112,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3137,7 +3136,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -3146,9 +3145,6 @@ func (m *Row) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3171,7 +3167,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3181,14 +3177,42 @@ func (m *Row) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } m.Keys = append(m.Keys, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Roaring", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Roaring = append(m.Roaring[:0], dAtA[iNdEx:postIndex]...) + if m.Roaring == nil { + m.Roaring = []byte{} + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -3198,7 +3222,121 @@ func (m *Row) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SignedRow) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SignedRow: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SignedRow: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Pos", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Pos == nil { + m.Pos = &Row{} + } + if err := m.Pos.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Neg", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Neg == nil { + m.Neg = &Row{} + } + if err := m.Neg.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -3229,7 +3367,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3255,7 +3393,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3272,7 +3410,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -3281,15 +3419,12 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -3309,7 +3444,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3333,7 +3468,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3343,9 +3478,6 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3360,9 +3492,6 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -3391,7 +3520,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3419,7 +3548,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ID |= uint64(b&0x7F) << shift + m.ID |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3438,7 +3567,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Count |= uint64(b&0x7F) << shift + m.Count |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3457,7 +3586,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3467,9 +3596,6 @@ func (m *Pair) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3484,9 +3610,6 @@ func (m *Pair) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -3515,7 +3638,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3543,7 +3666,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3553,9 +3676,6 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3575,7 +3695,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.RowID |= uint64(b&0x7F) << shift + m.RowID |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3594,7 +3714,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3604,9 +3724,6 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3621,9 +3738,6 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -3652,7 +3766,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3680,7 +3794,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -3689,9 +3803,6 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3714,7 +3825,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Count |= uint64(b&0x7F) << shift + m.Count |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3728,9 +3839,6 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -3759,7 +3867,7 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3787,7 +3895,7 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Val |= int64(b&0x7F) << shift + m.Val |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3806,7 +3914,7 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Count |= int64(b&0x7F) << shift + m.Count |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3820,9 +3928,6 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -3851,7 +3956,7 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3879,7 +3984,7 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ID |= uint64(b&0x7F) << shift + m.ID |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3898,7 +4003,7 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -3907,9 +4012,6 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3932,7 +4034,7 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3942,9 +4044,6 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3959,9 +4058,6 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -3990,7 +4086,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4018,7 +4114,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4028,9 +4124,6 @@ func (m *Attr) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4050,7 +4143,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Type |= uint64(b&0x7F) << shift + m.Type |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4069,7 +4162,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4079,9 +4172,6 @@ func (m *Attr) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4101,7 +4191,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.IntValue |= int64(b&0x7F) << shift + m.IntValue |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4120,7 +4210,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + v |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4146,9 +4236,6 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4177,7 +4264,7 @@ func (m *AttrMap) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4205,7 +4292,7 @@ func (m *AttrMap) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4214,9 +4301,6 @@ func (m *AttrMap) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4234,9 +4318,6 @@ func (m *AttrMap) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4265,7 +4346,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4293,7 +4374,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4303,9 +4384,6 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4323,7 +4401,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4340,7 +4418,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4349,15 +4427,12 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -4377,7 +4452,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4401,7 +4476,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + v |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4421,7 +4496,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + v |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4441,7 +4516,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + v |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4461,12 +4536,43 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + v |= (int(b) & 0x7F) << shift if b < 0x80 { break } } m.ExcludeColumns = bool(v != 0) + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EmbeddedData", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.EmbeddedData = append(m.EmbeddedData, &Row{}) + if err := m.EmbeddedData[len(m.EmbeddedData)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -4476,9 +4582,6 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4507,7 +4610,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4535,7 +4638,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4545,9 +4648,6 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4567,7 +4667,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4576,9 +4676,6 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4601,7 +4698,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4610,9 +4707,6 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4630,9 +4724,6 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4661,7 +4752,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4689,7 +4780,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4698,9 +4789,6 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4725,7 +4813,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.N |= uint64(b&0x7F) << shift + m.N |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4744,7 +4832,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4753,9 +4841,6 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4778,7 +4863,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + v |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4798,7 +4883,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4807,9 +4892,6 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4834,7 +4916,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Type |= uint32(b&0x7F) << shift + m.Type |= (uint32(b) & 0x7F) << shift if b < 0x80 { break } @@ -4851,7 +4933,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4868,7 +4950,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4877,15 +4959,12 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -4905,7 +4984,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4929,7 +5008,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4938,9 +5017,6 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4963,7 +5039,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -4972,9 +5048,6 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4985,6 +5058,39 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 10: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SignedRow", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SignedRow == nil { + m.SignedRow = &SignedRow{} + } + if err := m.SignedRow.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -4994,9 +5100,6 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5025,7 +5128,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5053,7 +5156,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5063,9 +5166,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5085,7 +5185,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5095,9 +5195,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5117,7 +5214,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Shard |= uint64(b&0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5134,7 +5231,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5151,7 +5248,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -5160,15 +5257,12 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -5188,7 +5282,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5210,7 +5304,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5227,7 +5321,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -5236,15 +5330,12 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -5264,7 +5355,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5286,7 +5377,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int64(b&0x7F) << shift + v |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5303,7 +5394,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -5312,15 +5403,12 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -5340,7 +5428,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int64(b&0x7F) << shift + v |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5364,7 +5452,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5374,9 +5462,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5396,7 +5481,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5406,9 +5491,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5423,9 +5505,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5454,7 +5533,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5482,7 +5561,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5492,9 +5571,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5514,7 +5590,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5524,9 +5600,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5546,7 +5619,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Shard |= uint64(b&0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5563,7 +5636,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5580,7 +5653,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -5589,15 +5662,12 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -5617,7 +5687,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5639,7 +5709,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int64(b&0x7F) << shift + v |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5656,7 +5726,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -5665,15 +5735,12 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -5693,7 +5760,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int64(b&0x7F) << shift + v |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5717,7 +5784,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5727,9 +5794,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5756,7 +5820,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -5765,9 +5829,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5798,9 +5859,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5829,7 +5887,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5857,7 +5915,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5867,9 +5925,6 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5889,7 +5944,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5899,9 +5954,6 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5921,7 +5973,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5931,9 +5983,6 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5948,9 +5997,6 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5979,7 +6025,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6005,7 +6051,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6022,7 +6068,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= int(b&0x7F) << shift + packedLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -6031,15 +6077,12 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA[iNdEx:postIndex] { + for _, integer := range dAtA { if integer < 128 { count++ } @@ -6059,7 +6102,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= uint64(b&0x7F) << shift + v |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6078,9 +6121,6 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6109,7 +6149,7 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6137,7 +6177,7 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6147,9 +6187,6 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6169,7 +6206,7 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + byteLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -6178,9 +6215,6 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6198,9 +6232,6 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6229,7 +6260,7 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6257,7 +6288,7 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + v |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -6277,7 +6308,7 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -6286,9 +6317,6 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPublic - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6306,9 +6334,6 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPublic - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6325,7 +6350,6 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { func skipPublic(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 - depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { @@ -6357,8 +6381,10 @@ func skipPublic(dAtA []byte) (n int, err error) { break } } + return iNdEx, nil case 1: iNdEx += 8 + return iNdEx, nil case 2: var length int for shift := uint(0); ; shift += 7 { @@ -6375,34 +6401,120 @@ func skipPublic(dAtA []byte) (n int, err error) { break } } + iNdEx += length if length < 0 { return 0, ErrInvalidLengthPublic } - iNdEx += length + return iNdEx, nil case 3: - depth++ - case 4: - if depth == 0 { - return 0, ErrUnexpectedEndOfGroupPublic + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowPublic + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipPublic(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next } - depth-- + return iNdEx, nil + case 4: + return iNdEx, nil case 5: iNdEx += 4 + return iNdEx, nil default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } - if iNdEx < 0 { - return 0, ErrInvalidLengthPublic - } - if depth == 0 { - return iNdEx, nil - } } - return 0, io.ErrUnexpectedEOF + panic("unreachable") } var ( - ErrInvalidLengthPublic = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow") - ErrUnexpectedEndOfGroupPublic = fmt.Errorf("proto: unexpected end of group") + ErrInvalidLengthPublic = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow") ) + +func init() { proto.RegisterFile("public.proto", fileDescriptor_public_17d2a22edfa80498) } + +var fileDescriptor_public_17d2a22edfa80498 = []byte{ + // 966 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x8e, 0xdb, 0x44, + 0x14, 0x66, 0x62, 0x67, 0xe3, 0x9c, 0xfc, 0x50, 0x0d, 0x69, 0xb1, 0x50, 0x15, 0x22, 0x0b, 0x21, + 0x73, 0xb3, 0x55, 0x83, 0x84, 0x7a, 0xc5, 0x4f, 0x9b, 0x2d, 0x8a, 0x4a, 0xa3, 0x72, 0x76, 0x15, + 0xae, 0xbd, 0xcd, 0x34, 0xb5, 0xe4, 0x78, 0x82, 0x7f, 0x70, 0xf7, 0x01, 0x78, 0x02, 0x6e, 0x78, + 0x04, 0x1e, 0x85, 0x2b, 0xc4, 0x23, 0xc0, 0xf2, 0x18, 0xdc, 0xa0, 0x39, 0xe3, 0xc9, 0x38, 0xee, + 0xb6, 0x42, 0x88, 0xbb, 0x39, 0xbf, 0x73, 0xbe, 0x39, 0xe7, 0x7c, 0x36, 0x0c, 0xf7, 0xe5, 0x65, + 0x12, 0x3f, 0x3f, 0xdd, 0x67, 0xb2, 0x90, 0xdc, 0x8b, 0xd3, 0x42, 0x64, 0x69, 0x94, 0x04, 0x39, + 0x38, 0x28, 0x2b, 0xee, 0x43, 0xef, 0x91, 0x4c, 0xca, 0x5d, 0x9a, 0xfb, 0x6c, 0xe6, 0x84, 0x2e, + 0x1a, 0x91, 0x7f, 0x04, 0xdd, 0xaf, 0x8a, 0x22, 0xcb, 0xfd, 0xce, 0xcc, 0x09, 0x07, 0xf3, 0xf1, + 0xa9, 0x09, 0x3d, 0x55, 0x6a, 0xd4, 0x46, 0xce, 0xc1, 0x7d, 0x22, 0xae, 0x72, 0xdf, 0x99, 0x39, + 0x61, 0x1f, 0xe9, 0xac, 0x72, 0xa2, 0x8c, 0xb2, 0x38, 0xdd, 0xfa, 0xee, 0x8c, 0x85, 0x43, 0x34, + 0x62, 0xf0, 0x14, 0xfa, 0xe7, 0xf1, 0x36, 0x15, 0x1b, 0x75, 0xf5, 0x87, 0xe0, 0x3c, 0x93, 0xea, + 0x5a, 0x16, 0x0e, 0xe6, 0x23, 0x9b, 0x1e, 0x65, 0x85, 0xca, 0xa2, 0x1c, 0x56, 0x62, 0xeb, 0x77, + 0x6e, 0x74, 0x58, 0x89, 0x6d, 0xf0, 0x00, 0xc6, 0x28, 0xab, 0xe5, 0x46, 0xa4, 0x45, 0xfc, 0x22, + 0x16, 0xba, 0x1c, 0x94, 0x95, 0xc1, 0x42, 0xe7, 0x43, 0x89, 0x1d, 0x5b, 0x62, 0xf0, 0x39, 0xb8, + 0xcf, 0xa2, 0x38, 0xe3, 0x63, 0xe8, 0x2c, 0x17, 0x54, 0x82, 0x8b, 0x9d, 0xe5, 0x82, 0x4f, 0xa0, + 0xfb, 0x48, 0x96, 0x69, 0x41, 0x97, 0xba, 0xa8, 0x05, 0x7e, 0x0b, 0x9c, 0x27, 0xe2, 0xca, 0x77, + 0x66, 0x2c, 0xec, 0xa3, 0x3a, 0x06, 0x2b, 0xf0, 0x1e, 0xc7, 0x22, 0x21, 0x1c, 0x13, 0xe8, 0xd2, + 0x99, 0xd2, 0xf4, 0x51, 0x0b, 0x4a, 0xab, 0x6a, 0x5b, 0x98, 0x4c, 0x24, 0xf0, 0x3b, 0x70, 0x82, + 0xb2, 0xb2, 0xc9, 0x6a, 0x29, 0xf8, 0x06, 0xe0, 0xeb, 0x4c, 0x96, 0x7b, 0x7d, 0x5f, 0x08, 0x5d, + 0x92, 0x08, 0xc6, 0x60, 0xce, 0x2d, 0x74, 0x73, 0x29, 0x6a, 0x87, 0x9b, 0xeb, 0x0d, 0xe6, 0xe0, + 0xad, 0xa3, 0xe4, 0x50, 0xfb, 0x3a, 0x4a, 0xa8, 0x36, 0x07, 0xd5, 0xf1, 0x38, 0xc6, 0x31, 0x31, + 0xdf, 0xc1, 0x48, 0x77, 0x5e, 0xf5, 0xf5, 0x5c, 0x14, 0xaf, 0x3d, 0xcd, 0xbf, 0x9b, 0x87, 0xd7, + 0x9f, 0xea, 0x17, 0x06, 0xae, 0xb2, 0x19, 0x13, 0x3b, 0x98, 0x54, 0x67, 0x2e, 0xae, 0xf6, 0xa2, + 0x2e, 0x9e, 0xce, 0x7c, 0x06, 0x83, 0xf3, 0x42, 0x0d, 0xcb, 0x3a, 0x4a, 0x4a, 0x51, 0x27, 0x6a, + 0xaa, 0xf8, 0x07, 0xe0, 0x2d, 0xd3, 0x42, 0x9b, 0x5d, 0x82, 0x70, 0x90, 0xf9, 0x5d, 0xe8, 0x3f, + 0x94, 0x32, 0xd1, 0xc6, 0xee, 0x8c, 0x85, 0x1e, 0x5a, 0x05, 0x9f, 0x02, 0x3c, 0x4e, 0x64, 0x54, + 0xc7, 0x9e, 0xcc, 0x58, 0xc8, 0xb0, 0xa1, 0x09, 0xee, 0x41, 0x4f, 0x55, 0xfa, 0x34, 0xda, 0x5b, + 0xb4, 0xec, 0x2d, 0x68, 0x83, 0xbf, 0x19, 0x0c, 0xbf, 0x2d, 0x45, 0x76, 0x85, 0xe2, 0xfb, 0x52, + 0xe4, 0x85, 0x7a, 0x5b, 0x92, 0xcd, 0x2c, 0x90, 0xa0, 0xba, 0x7e, 0xfe, 0x32, 0xca, 0x36, 0xfa, + 0xed, 0x5c, 0xac, 0x25, 0x85, 0xd5, 0xbe, 0x79, 0x4e, 0x58, 0x3d, 0x6c, 0xaa, 0x68, 0x5e, 0xc4, + 0x4e, 0x16, 0x06, 0x4c, 0x2d, 0xf1, 0x10, 0xde, 0x3d, 0x7b, 0xf5, 0x3c, 0x29, 0x37, 0x02, 0x65, + 0xa5, 0xa3, 0x4f, 0xc8, 0xa1, 0xad, 0xe6, 0x1f, 0xc3, 0xb8, 0x56, 0x99, 0x3d, 0xef, 0x91, 0x63, + 0x4b, 0xcb, 0xef, 0xc3, 0xf0, 0x6c, 0x77, 0x29, 0x36, 0x1b, 0xb1, 0x59, 0x44, 0x45, 0xe4, 0x7b, + 0x84, 0xbb, 0xb5, 0x75, 0x47, 0x2e, 0xc1, 0x4f, 0x0c, 0x46, 0x35, 0xfa, 0x7c, 0x2f, 0xd3, 0x5c, + 0xa8, 0x16, 0x9f, 0x65, 0x99, 0x69, 0xf1, 0x59, 0x96, 0xf1, 0x7b, 0xd0, 0x43, 0x91, 0x97, 0x49, + 0x61, 0xe6, 0xe6, 0xb6, 0xcd, 0x68, 0x62, 0xcb, 0xa4, 0x40, 0xe3, 0xc5, 0xbf, 0x80, 0xf1, 0xd1, + 0x1c, 0x6a, 0x6a, 0x19, 0xcc, 0xdf, 0xb7, 0x71, 0x47, 0x76, 0x6c, 0xb9, 0x07, 0x3f, 0x3a, 0x30, + 0x68, 0x64, 0x56, 0x2c, 0x82, 0xb2, 0x7a, 0x03, 0xcd, 0xa8, 0xfd, 0x1d, 0x02, 0x5b, 0xd5, 0x23, + 0xc8, 0x56, 0xaa, 0xf1, 0x8a, 0x19, 0xcc, 0xb5, 0x8d, 0xc6, 0x2b, 0x35, 0x6a, 0x23, 0xd1, 0xe6, + 0xcb, 0x28, 0xdd, 0x8a, 0x0d, 0x8d, 0xa0, 0x87, 0x46, 0xe4, 0xa7, 0x76, 0xf7, 0xa8, 0x67, 0x47, + 0xeb, 0x6b, 0x2c, 0x68, 0xf7, 0xd3, 0xec, 0x80, 0x6a, 0xdf, 0xa8, 0xde, 0x01, 0xcd, 0x12, 0xcb, + 0x85, 0xea, 0x15, 0xcd, 0x8b, 0x96, 0xf8, 0x67, 0x30, 0xb0, 0x2c, 0x91, 0xd7, 0x2d, 0x9a, 0xd8, + 0xf4, 0xd6, 0x88, 0x4d, 0x47, 0xfe, 0x65, 0x9b, 0x27, 0xfd, 0x3e, 0x55, 0xe6, 0x1f, 0xbd, 0x46, + 0xc3, 0x8e, 0x6d, 0x5e, 0xbd, 0xdf, 0x20, 0x6e, 0x1f, 0x28, 0xf8, 0x3d, 0x1b, 0x7c, 0x30, 0xa1, + 0xf5, 0x0a, 0xfe, 0x64, 0x30, 0x5a, 0xee, 0xf6, 0x32, 0x2b, 0x1a, 0xcb, 0xb1, 0x4c, 0x37, 0xe2, + 0x95, 0x59, 0x0e, 0x12, 0x2c, 0x7d, 0x76, 0x5a, 0xf4, 0x49, 0x4b, 0x42, 0x4b, 0xe1, 0xa2, 0x16, + 0x1a, 0x0f, 0xe3, 0x1e, 0x3d, 0xcc, 0x5d, 0xe8, 0xeb, 0x29, 0x50, 0xa6, 0x2e, 0x99, 0xac, 0x42, + 0xad, 0xfd, 0x45, 0xbc, 0x13, 0x79, 0x11, 0xed, 0xf6, 0x6a, 0x4f, 0x9c, 0xd0, 0xc1, 0x86, 0x46, + 0x7f, 0xaf, 0x2a, 0xfa, 0x46, 0xf4, 0xe8, 0x1b, 0x61, 0x44, 0x15, 0xa9, 0xd3, 0x90, 0xd1, 0x23, + 0x63, 0x43, 0x13, 0xfc, 0xc6, 0x80, 0x6b, 0x8c, 0x44, 0x20, 0xff, 0x1f, 0xd0, 0xb7, 0x03, 0xba, + 0x03, 0x27, 0x74, 0x9f, 0x01, 0x53, 0x4b, 0xad, 0x72, 0x7b, 0xed, 0x72, 0x15, 0xdf, 0x58, 0xb6, + 0xd3, 0x78, 0x18, 0x36, 0x55, 0xc1, 0x1a, 0x26, 0x17, 0x59, 0x94, 0xe6, 0x49, 0x54, 0x08, 0x15, + 0xf2, 0x5f, 0x10, 0xdd, 0xf0, 0x4b, 0x10, 0x7c, 0x02, 0xb7, 0x5b, 0x79, 0x2d, 0x63, 0x28, 0x88, + 0x0e, 0x41, 0x54, 0xc7, 0xe0, 0x21, 0xf8, 0xf5, 0xd8, 0xe8, 0x9f, 0x86, 0xba, 0x84, 0x75, 0x2c, + 0x2a, 0x95, 0x7a, 0x15, 0xed, 0x44, 0x5d, 0x05, 0x9d, 0x95, 0x8e, 0x08, 0xab, 0x43, 0xbf, 0x1a, + 0x74, 0x0e, 0x5e, 0xc0, 0xe4, 0xa6, 0x1c, 0xf4, 0xe9, 0x4b, 0x44, 0xa4, 0x19, 0xca, 0x43, 0x2d, + 0xf0, 0x07, 0xd0, 0xfd, 0x21, 0x16, 0x95, 0x61, 0xa8, 0xc0, 0x0e, 0xf6, 0x9b, 0x0a, 0x41, 0x1d, + 0xf0, 0xf0, 0xd6, 0xaf, 0xd7, 0x53, 0xf6, 0xfb, 0xf5, 0x94, 0xfd, 0x71, 0x3d, 0x65, 0x3f, 0xff, + 0x35, 0x7d, 0xe7, 0xf2, 0x84, 0xfe, 0xb3, 0x3e, 0xfd, 0x27, 0x00, 0x00, 0xff, 0xff, 0xe6, 0xfc, + 0xfe, 0x74, 0x77, 0x09, 0x00, 0x00, +} diff --git a/internal/public.proto b/internal/public.proto index 5c48cf016..35d1a7258 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -6,6 +6,12 @@ message Row { repeated uint64 Columns = 1; repeated string Keys = 3; repeated Attr Attrs = 2; + bytes Roaring = 4; +} + +message SignedRow { + Row Pos = 1; + Row Neg = 2; } message RowIdentifiers { @@ -61,6 +67,7 @@ message QueryRequest { bool Remote = 5; bool ExcludeRowAttrs = 6; bool ExcludeColumns = 7; + repeated Row EmbeddedData = 8; } message QueryResponse { @@ -79,6 +86,7 @@ message QueryResult { repeated uint64 RowIDs = 7; repeated GroupCount GroupCounts = 8; RowIdentifiers RowIdentifiers = 9; + SignedRow SignedRow = 10; } message ImportRequest { @@ -120,4 +128,4 @@ message ImportRoaringRequestView { message ImportRoaringRequest { bool Clear = 1; repeated ImportRoaringRequestView views = 2; -} \ No newline at end of file +} diff --git a/pql/ast.go b/pql/ast.go index 49503474c..0162293d7 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -17,10 +17,13 @@ package pql import ( "bytes" "fmt" + "reflect" "sort" "strconv" "strings" "time" + + "github.com/pilosa/pilosa/v2/ext" ) // Query represents a PQL query. @@ -259,11 +262,250 @@ type callStackElem struct { inList bool } -// Call represents a function call in the AST. +// Some call types may require special handling, which needs to occur +// before distributing processing to individual shards. +type CallType byte + +const ( + // Normal calls can be executed per shard. + PrecallNone = CallType(iota) + // PreCallGlobal indicates a call which must be run globally *before* + // distributing the call to other shards. Example: A Distinct query, + // where every shard could potentially produce results for any shard, + // so you have to produce the results up front. + PrecallGlobal + // PreCallPerNode indicates a call which needs to be run per-shard + // in a way that lets it be done on each shard, but where it should + // be done prior to spawning per-shard goroutines. Example: + // A cross-index query, where each local shard may or may not need + // to get data from a remote node, but batches of shards can + // probably be gotten from the same remote node. + PrecallPerNode +) + +// Call represents a function call in the AST. The Precomputed field +// is used by the executor to handle non-standard call types; it does +// these by actually executing them separately, then replacing them +// in the call tree with a new call using the special precomputed +// type, with the Precomputed field set to a map from shards to results. type Call struct { - Name string - Args map[string]interface{} - Children []*Call + Name string + Args map[string]interface{} + Children []*Call + Type CallType + Precomputed map[uint64]interface{} +} + +// callInfo defines the arguments allowed for a particular PQL call, and +// possibly things about its semantics. If allowUnknown is true, unfamiliar +// non-reserved names are allowed on the assumption that they're field names. +// Otherwise, only those names explicitly listed are allowed. Reserved args +// (those with a leading underscore) are never allowed unless explicitly +// present. +// +// The prototypes map maps from argument names to a value. If the value is +// non-nil, the argument will be checked for type-matching. So, for instance, +// `x: 10` would indicate that x must be an int. +type callInfo struct { + allowUnknown bool + prototypes map[string]interface{} + callType CallType +} + +// We want to be able to accept either a string or int64 for +// field names. Special-case type: +type stringOrInt64Type struct{} + +var stringOrInt64 stringOrInt64Type + +var allowUnderField = callInfo{ + allowUnknown: true, + prototypes: map[string]interface{}{ + "_field": "", + }, +} + +var allowField = callInfo{ + allowUnknown: false, + prototypes: map[string]interface{}{ + "field": "", + }, +} + +var callInfoByFunc = map[string]callInfo{ + // the easy cases: things that take arbitrary inputs, because they're + // taking field=value cases + "Bitmap": {allowUnknown: true}, + "Count": {allowUnknown: true}, + "Row": {allowUnknown: true}, + "Range": {allowUnknown: true}, + + // allow only "field=X" cases with string field names + "Max": allowField, + "Min": allowField, + "Sum": allowField, + + // only take other calls, should never have "args" + "Difference": {allowUnknown: false}, + "Intersect": {allowUnknown: false}, + "Not": {allowUnknown: false}, + "ClearRow": {allowUnknown: true}, + "Store": {allowUnknown: true}, + "MinRow": allowField, + "MaxRow": allowField, + "Rows": { + allowUnknown: false, + prototypes: map[string]interface{}{ + "_field": "", + "field": "", + "limit": int64(0), + "column": nil, + "previous": nil, + "from": nil, + "to": nil, + }, + }, + "Shift": {allowUnknown: false, + prototypes: map[string]interface{}{ + "n": int64(0), + }, + }, + "Union": {allowUnknown: false}, + "Xor": {allowUnknown: false}, + + // things that take _field + "TopN": allowUnderField, + // special cases: + "Clear": { + allowUnknown: true, + prototypes: map[string]interface{}{ + "_col": stringOrInt64, + }, + }, + "GroupBy": { + allowUnknown: false, + prototypes: map[string]interface{}{ + "filter": nil, + "limit": int64(0), + "previous": nil, + }, + }, + "Options": { + allowUnknown: false, + prototypes: map[string]interface{}{ + "excludeRowAttrs": true, + "excludeColumns": true, + "columnAttrs": true, + "shards": nil, + }, + }, + "Set": { + allowUnknown: true, + prototypes: map[string]interface{}{ + "_col": stringOrInt64, + "_timestamp": "", + }, + }, + "Precomputed": { + allowUnknown: true, + }, + "SetBit": { + allowUnknown: true, + prototypes: map[string]interface{}{ + "_col": stringOrInt64, + }, + }, + "SetRowAttrs": { + allowUnknown: true, + prototypes: map[string]interface{}{ + "_field": "", + "_row": stringOrInt64, + }, + }, + "SetColumnAttrs": { + allowUnknown: true, + prototypes: map[string]interface{}{ + "_field": "", + "_col": stringOrInt64, + }, + }, + "IncludesColumn": { + allowUnknown: false, + prototypes: map[string]interface{}{ + "column": stringOrInt64, + }, + }, +} + +// RegisterPluginFuncs adds arg validation for plugin funcs. Not very good +// arg validation. +func RegisterPluginFuncs(ops []ext.BitmapOp) { + for _, op := range ops { + // ignore overlap for now. This should change. + if _, ok := callInfoByFunc[op.Name]; ok { + continue + } + ci := callInfo{allowUnknown: true} + if len(op.Reserved) > 0 { + // mark these as valid/known reserved words + ci.prototypes = make(map[string]interface{}) + for _, res := range op.Reserved { + ci.prototypes[res] = nil + } + } + t := op.Func.BitmapOpType() + if t.Precall == ext.OpPrecallGlobal { + ci.callType = PrecallGlobal + } + callInfoByFunc[op.Name] = ci + } +} + +// CheckCallInfo tries to validate that arguments are correct and valid for the +// given call. It does not guarantee checking all possible errors; for instance, +// if an argument is a field name, CheckCallInfo can't validate that the field +// exists. It also updates with information like whether the call is expected +// to require precalling. +func (c *Call) CheckCallInfo() error { + valid, ok := callInfoByFunc[c.Name] + if !ok { + return fmt.Errorf("no arg validation for '%s'", c.Name) + } + c.Type = valid.callType + for k, v := range c.Args { + acceptable, ok := valid.prototypes[k] + if !ok && !valid.allowUnknown { + return fmt.Errorf("'%s': unknown arg '%s'", c.String(), k) + } + if !ok && strings.HasPrefix(k, "_") { + return fmt.Errorf("'%s': unknown reserved arg '%s'", c.String(), k) + } + if acceptable == nil { + continue + } + // if the types are identical, that's fine + if reflect.TypeOf(acceptable) == reflect.TypeOf(v) { + continue + } + if reflect.TypeOf(acceptable) == reflect.TypeOf(stringOrInt64) { + switch v.(type) { + case string, int64: + continue + default: + return fmt.Errorf("'%s': arg '%s' needed a string or integer value, got %T.", + c.String(), k, v) + } + } + return fmt.Errorf("'%s': arg '%s' wrong type (got %T, expected %T)", + c.String(), k, v, acceptable) + } + // call-specific checking + for _, child := range c.Children { + if err := child.CheckCallInfo(); err != nil { + return err + } + } + return nil } // FieldArg determines which key-value pair contains the field and rowID, @@ -283,13 +525,29 @@ func IsReservedArg(name string) bool { return true } switch name { - case "from", "to": + case "from", "to", "index": return true default: return false } } +// CallIndex handles guessing whether we've been asked to apply this to a +// different index. An empty string means "no". +func (c *Call) CallIndex() string { + if index, ok := c.Args["_index"]; ok { + if index, ok := index.(string); ok { + return index + } + } + if index, ok := c.Args["index"]; ok && index != "" { + if index, ok := index.(string); ok { + return index + } + } + return "" +} + // BoolArg is for reading the value at key from call.Args as a bool. If the // key is not in Call.Args, the value of the returned bool will be false, and // the error will be nil. The value is assumed to be a bool. An error is diff --git a/pql/parser.go b/pql/parser.go index e733038d7..8482359fb 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -82,6 +82,14 @@ func (p *parser) Parse() (*Query, error) { panic(v) } } + for _, call := range p.Query.Calls { + if call == nil { + return nil, fmt.Errorf("unexpected nil Call in query's call list") + } + if err := call.CheckCallInfo(); err != nil { + return nil, err + } + } return &p.Query, nil } diff --git a/pql/parser_test.go b/pql/parser_test.go index 3e3ef719c..8e662eb66 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -75,12 +75,12 @@ func TestParser_Parse(t *testing.T) { // Parse with only arguments. t.Run("ArgumentsOnly", func(t *testing.T) { - q, err := pql.ParseString(`MyCall( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\" )`) + q, err := pql.ParseString(`Row( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\" )`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], &pql.Call{ - Name: "MyCall", + Name: "Row", Args: map[string]interface{}{ "key": "value", "foo": "bar", @@ -98,12 +98,12 @@ func TestParser_Parse(t *testing.T) { // Parse with float arguments. t.Run("WithFloatArgs", func(t *testing.T) { - q, err := pql.ParseString(`MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9)`) + q, err := pql.ParseString(`Row( key=12.25, foo= 13.167, bar=2., baz=0.9)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], &pql.Call{ - Name: "MyCall", + Name: "Row", Args: map[string]interface{}{ "key": 12.25, "foo": 13.167, @@ -118,12 +118,12 @@ func TestParser_Parse(t *testing.T) { // Parse with float arguments. t.Run("WithNegativeArgs", func(t *testing.T) { - q, err := pql.ParseString(`MyCall( key=-12.25, foo= -13)`) + q, err := pql.ParseString(`Row( key=-12.25, foo= -13)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], &pql.Call{ - Name: "MyCall", + Name: "Row", Args: map[string]interface{}{ "key": -12.25, "foo": int64(-13), @@ -173,12 +173,12 @@ func TestParser_Parse(t *testing.T) { // Parse with condition arguments. t.Run("WithCondition", func(t *testing.T) { - q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`) + q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], &pql.Call{ - Name: "MyCall", + Name: "Row", Args: map[string]interface{}{ "key": "foo", "x": &pql.Condition{Op: pql.EQ, Value: 12.25}, diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 3ceff075d..079d4abf4 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -46,7 +46,7 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9 t.Fatalf("Failed, got: %s", q) } - _, err = ParseString("C(a=falsen0)") + _, err = ParseString("Row(a=falsen0)") if err != nil { t.Fatalf("falsen0 should have been parsed as a string") } @@ -109,15 +109,15 @@ func TestPEGWorking(t *testing.T) { ncalls: 2}, { name: "SetWithArbCall", - input: "Set(1, a=4)Blerg(z=ha)", + input: "Set(1, a=4)Row(z=ha)", ncalls: 2}, { name: "SetArbSet", - input: "Set(1, a=4)Blerg(z=ha)Set(2, z=99)", + input: "Set(1, a=4)Row(z=ha)Set(2, z=99)", ncalls: 3}, { name: "ArbSetArb", - input: "Arb(q=1, a=4)Set(1, z=9)Arb(z=99)", + input: "Row(q=1, a=4)Set(1, z=9)Row(z=99)", ncalls: 3}, { name: "SetStringArg", @@ -161,11 +161,11 @@ func TestPEGWorking(t *testing.T) { ncalls: 1}, { name: "double quoted args", - input: `B(a="zm''e")`, + input: `Row(a="zm''e")`, ncalls: 1}, { name: "single quoted args", - input: `B(a='zm""e')`, + input: `Row(a='zm""e')`, ncalls: 1}, { name: "SetRowAttrs", @@ -320,7 +320,7 @@ func TestPEGErrors(t *testing.T) { input: "Set(, 1, a=4)"}, { name: "StartinCommaArb", - input: "Zeeb(, a=4)"}, + input: "Row(, a=4)"}, { name: "SetRowAttrs0args", input: "SetRowAttrs(blah, 9)"}, @@ -585,11 +585,11 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "Weird dash", - call: "Sum(field-=f)", + call: "Count(dashy-=f)", exp: &Call{ - Name: "Sum", + Name: "Count", Args: map[string]interface{}{ - "field-": "f", + "dashy-": "f", }, }}, { diff --git a/roaring/container_stash.go b/roaring/container_stash.go index c1dee9fa4..8aab2f4d3 100644 --- a/roaring/container_stash.go +++ b/roaring/container_stash.go @@ -416,6 +416,45 @@ func (c *Container) bitmap() []uint64 { return *(*[]uint64)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(c.pointer)), Len: int(c.len), Cap: int(c.cap)})) } +// AsBitmap yields a 65k-bit bitmap, storing it in the target if a target +// is provided. The target should be zeroed, or this becomes an implicit +// union. +func (c *Container) AsBitmap(target []uint64) (out []uint64) { + if c.typeID == containerBitmap { + return c.bitmap() + } + // Reminder: len(nil) == 0. + if len(target) < 1024 { + out = make([]uint64, 1024) + } else { + out = target + } + if c.typeID == containerArray { + a := c.array() + for _, v := range a { + out[v/64] |= 1 << (v % 64) + } + return out + } + if c.typeID == containerRun { + runs := c.runs() + for _, r := range runs { + splatRun(out, r) + } + return out + } + // in theory this shouldn't happen? + return out +} + +func splatRun(into []uint64, from interval16) { + // TODO this can be ~64x faster for long runs by setting maxBitmap instead of single bits + //note v must be int or will overflow + for v := int(from.start); v <= int(from.last); v++ { + into[v/64] |= (uint64(1) << uint(v%64)) + } +} + // setBitmap stores a set of uint64s as data. func (c *Container) setBitmap(bitmap []uint64) { if c == nil || c.frozen() { diff --git a/roaring/roaring.go b/roaring/roaring.go index 2c985358b..884889a90 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -21,6 +21,7 @@ import ( "hash/fnv" "io" "math/bits" + "reflect" "sort" "unsafe" @@ -433,6 +434,13 @@ func (b *Bitmap) Min() (uint64, bool) { return v, !eof } +// MinAt returns the lowest value in the bitmap at least equal to its argument. +// Second return value is true if containers exist in the bitmap. +func (b *Bitmap) MinAt(start uint64) (uint64, bool) { + v, eof := b.IteratorAt(start).Next() + return v, !eof +} + // Max returns the highest value in the bitmap. // Returns zero if the bitmap is empty. func (b *Bitmap) Max() uint64 { @@ -1718,6 +1726,12 @@ func (b *Bitmap) Iterator() *Iterator { return itr } +func (b *Bitmap) IteratorAt(start uint64) *Iterator { + itr := &Iterator{bitmap: b} + itr.Seek(start) + return itr +} + // Ops returns the number of write ops the bitmap is aware of in its ops // log, and their total bit count. func (b *Bitmap) Ops() (ops int, opN int) { @@ -1730,6 +1744,167 @@ func (b *Bitmap) SetOps(ops int, opN int) { b.ops, b.opN = ops, opN } +// RoaringToBitmaps yields a series of bitmaps with specified shard +// keys, based on a single roaring file, with splits at multiples of +// shardWidth, which should be a multiple of container size. +func RoaringToBitmaps(data []byte, shardWidth uint64) ([]*Bitmap, []uint64) { + if data == nil { + return nil, nil + } + var itr roaringIterator + var itrKey uint64 + var itrCType byte + var itrN int + var itrLen int + var itrPointer *uint16 + var itrErr error + currentShard := ^uint64(0) + var currentBitmap *Bitmap + var bitmaps []*Bitmap + var shards []uint64 + keysPerShard := shardWidth >> 16 + + itr, err := newRoaringIterator(data) + if err != nil || itr == nil { + return nil, nil + } + + itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next() + for itrErr == nil { + newC := &Container{ + typeID: itrCType, + n: int32(itrN), + len: int32(itrLen), + cap: int32(itrLen), + pointer: itrPointer, + flags: flagMapped, + } + shard := itrKey / keysPerShard + if shard != currentShard { + if currentBitmap != nil { + bitmaps = append(bitmaps, currentBitmap) + shards = append(shards, currentShard) + } + currentBitmap = NewFileBitmap() + currentShard = shard + } + currentBitmap.Containers.Put(itrKey, newC) + itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next() + } + if currentBitmap != nil { + bitmaps = append(bitmaps, currentBitmap) + shards = append(shards, currentShard) + } + // we don't support ops logs for this + return bitmaps, shards +} + +// BitmapsToRoaring renders a series of non-overlapping bitmaps as a +// unified roaring file. +func BitmapsToRoaring(bitmaps []*Bitmap) []byte { + count := int64(0) + size := int64(0) + for i, bm := range bitmaps { + c, s := bm.roaringSize() + // skip this bitmap during the next pass, since it's empty + if c == 0 { + bitmaps[i] = nil + continue + } + count += c + size += s + } + if count == 0 { + return nil + } + // we have count headers, which need 12 bytes, plus a magic number, + // plus offsets (4 bytes per container), plus size bytes of data to + // write. + out := make([]byte, headerBaseSize+(12*count)+(4*count)+size) + binary.LittleEndian.PutUint16(out[0:2], uint16(MagicNumber)) + out[3] = byte(storageVersion) + binary.LittleEndian.PutUint32(out[4:8], uint32(count)) + headerEnd := 8 + (12 * count) + offsetEnd := headerEnd + (4 * count) + headers := out[8:headerEnd] + offsets := out[headerEnd:offsetEnd] + data := out[offsetEnd:] + headerOffset := 0 + offsetOffset := 0 + dataOffset := 0 + prevKey := uint64(0) + for _, bm := range bitmaps { + if bm == nil { + continue + } + citer, _ := bm.Containers.Iterator(0) + for citer.Next() { + k, c := citer.Value() + n := c.N() + if n == 0 { + continue + } + if roaringParanoia { + if k < prevKey { + panic("unsorted keys in multiple-bitmap roaring conversion") + } + } + // place header at header offset, and data at data + // offset + header := headers[headerOffset : headerOffset+12] + offset := offsets[offsetOffset : offsetOffset+4] + headerOffset += 12 + offsetOffset += 4 + binary.LittleEndian.PutUint64(header[0:8], k) + binary.LittleEndian.PutUint16(header[8:10], uint16(c.typeID)) + binary.LittleEndian.PutUint16(header[10:12], uint16(n-1)) + binary.LittleEndian.PutUint32(offset[0:4], uint32(dataOffset+int(offsetEnd))) + nextData := data[dataOffset:] + switch c.typeID { + case containerArray: + asUint16 := *(*[]uint16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&nextData[0])), Len: int(c.len), Cap: int(c.len)})) + copy(asUint16, c.array()) + dataOffset += 2 * int(c.len) + case containerBitmap: + asUint64 := *(*[]uint64)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&nextData[0])), Len: 1024, Cap: 1024})) + copy(asUint64, c.bitmap()) + dataOffset += 8192 + case containerRun: + asInterval16 := *(*[]interval16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&nextData[2])), Len: int(c.len), Cap: int(c.len)})) + copy(asInterval16, c.runs()) + binary.LittleEndian.PutUint16(nextData[0:2], uint16(c.len)) + dataOffset += int(4*c.len) + 2 + } + } + } + return out +} + +// roaringSize yields the count of non-empty containers, and the size +// of the storage *only* -- not the headers. +func (b *Bitmap) roaringSize() (int64, int64) { + count := int64(0) + size := int64(0) + citer, _ := b.Containers.Iterator(0) + for citer.Next() { + _, c := citer.Value() + if c.N() == 0 { + continue + } + count++ + switch c.typeID { + case containerArray: + size += 2 * int64(c.N()) + case containerBitmap: + size += 8192 + case containerRun: + // 2 bytes for the count of runs, plus 4 bytes per run + size += 2 + (4 * int64(c.len)) + } + } + return count, size +} + // Info returns stats for the bitmap. func (b *Bitmap) Info() bitmapInfo { info := bitmapInfo{ diff --git a/row.go b/row.go index 601d5101e..d9a4f9cc9 100644 --- a/row.go +++ b/row.go @@ -18,6 +18,7 @@ import ( "encoding/json" "sort" + "github.com/pilosa/pilosa/v2/ext" "github.com/pilosa/pilosa/v2/roaring" "github.com/pkg/errors" ) @@ -43,6 +44,55 @@ func NewRow(columns ...uint64) *Row { return r } +// NewRowFromBitmap divides a bitmap into rows, which it now calls shards. This +// transposes; data that was in any shard for Row 0 is now considered shard 0, +// etcetera. +func NewRowFromBitmap(b *roaring.Bitmap) *Row { + r := &Row{} + if b == nil { + return r + } + rowNum := uint64(0) + for col, ok := b.MinAt(rowNum * ShardWidth); ok; col, ok = b.MinAt(rowNum * ShardWidth) { + rowNum = col / ShardWidth + seg := rowSegment{ + shard: rowNum, + data: b.OffsetRange(rowNum*ShardWidth, rowNum*ShardWidth, (rowNum+1)*ShardWidth), + writable: true, + } + seg.n = seg.data.Count() + r.segments = append(r.segments, seg) + rowNum++ + } + return r +} + +// NewRowFromRoaring parses a roaring data file as a row, dividing it into +// bitmaps and rowSegments based on shard width. +func NewRowFromRoaring(data []byte) *Row { + bitmaps, shards := roaring.RoaringToBitmaps(data, ShardWidth) + r := &Row{segments: make([]rowSegment, len(bitmaps))} + for i := range bitmaps { + segment := rowSegment{ + shard: shards[i], + data: bitmaps[i], + writable: false, + n: bitmaps[i].Count(), + } + r.segments[i] = segment + } + return r +} + +// Roaring returns the row treated as a unified roaring bitmap. +func (r *Row) Roaring() []byte { + bitmaps := make([]*roaring.Bitmap, len(r.segments)) + for i := range r.segments { + bitmaps[i] = r.segments[i].data + } + return roaring.BitmapsToRoaring(bitmaps) +} + // IsEmpty returns true if the row doesn't contain any set bits. func (r *Row) IsEmpty() bool { if len(r.segments) == 0 { @@ -194,6 +244,65 @@ func (r *Row) Union(others ...*Row) *Row { return &Row{segments: output} } +// GenericBinaryOp returns the output of a generic op on r and other. +func (r *Row) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *Row, args map[string]interface{}) *Row { + var segments []rowSegment + itr := newMergeSegmentIterator(r.segments, other.segments) + for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { + if s1 == nil { + segments = append(segments, *s0) + continue + } else if s0 == nil { + segments = append(segments, *s1) + continue + } + segments = append(segments, *s0.GenericBinaryOp(op, s1, args)) + } + + return &Row{segments: segments} +} + +// GenericNaryOp returns the output of an nary op on r and others. +func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args map[string]interface{}) *Row { + segments := make([][]rowSegment, 0, len(others)+1) + if len(r.segments) > 0 { + segments = append(segments, r.segments) + } + nextSegs := make([][]rowSegment, 0, len(others)+1) + toProcess := make([]*rowSegment, 0, len(others)+1) + var output []rowSegment + for _, other := range others { + if len(other.segments) > 0 { + segments = append(segments, other.segments) + } + } + for len(segments) > 0 { + shard := segments[0][0].shard + for _, segs := range segments { + if segs[0].shard < shard { + shard = segs[0].shard + } + } + nextSegs = nextSegs[:0] + toProcess := toProcess[:0] + for _, segs := range segments { + if segs[0].shard == shard { + toProcess = append(toProcess, &segs[0]) + segs = segs[1:] + } + if len(segs) > 0 { + nextSegs = append(nextSegs, segs) + } + } + // at this point, "toProcess" is a list of all the segments + // sharing the lowest ID, and nextSegs is a list of all the others. + // Swap the segment lists (so we don't have to reallocate it) + segments, nextSegs = nextSegs, segments + output = append(output, *toProcess[0].GenericNaryOp(op, toProcess[1:], args)) + } + return &Row{segments: output} +} + // Difference returns the diff of r and other. func (r *Row) Difference(other *Row) *Row { var segments []rowSegment @@ -212,6 +321,17 @@ func (r *Row) Difference(other *Row) *Row { return &Row{segments: segments} } +// GenericUnary returns the results of a generic op on r. +func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row { + work := r + var segments []rowSegment + for _, segment := range work.segments { + opped := segment.GenericUnaryOp(op, args) + segments = append(segments, *opped) + } + return &Row{segments: segments} +} + // Shift returns the bitwise shift of r by n bits. // Currently only positive shift values are supported. func (r *Row) Shift(n int64) (*Row, error) { @@ -299,6 +419,15 @@ func (r *Row) Count() uint64 { return n } +// GenericCount applies an op to lots of things. +func (r *Row) GenericCount(op ext.BitmapOpUnaryCount, args map[string]interface{}) uint64 { + var n int64 + for i := range r.segments { + n += op([]ext.Bitmap{WrapBitmap(r.segments[i].data)}, args) + } + return uint64(n) +} + // MarshalJSON returns a JSON-encoded byte slice of r. func (r *Row) MarshalJSON() ([]byte, error) { var o struct { @@ -419,6 +548,33 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment { } } +// GenericOp performs a generic op on s and other +func (s *rowSegment) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *rowSegment, args map[string]interface{}) *rowSegment { + data := op([]ext.Bitmap{WrapBitmap(s.data), WrapBitmap(other.data)}, args) + + return &rowSegment{ + data: UnwrapBitmap(data), + shard: s.shard, + n: data.Count(), + } +} + +// GenericOp performs a generic op on s and others +func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSegment, args map[string]interface{}) *rowSegment { + bitmaps := make([]ext.Bitmap, len(others)+1) + bitmaps[0] = WrapBitmap(s.data) + for i, seg := range others { + bitmaps[i+1] = WrapBitmap(seg.data) + } + data := op(bitmaps, args) + + return &rowSegment{ + data: UnwrapBitmap(data), + shard: s.shard, + n: data.Count(), + } +} + // Difference returns the diff of s and other. func (s *rowSegment) Difference(other *rowSegment) *rowSegment { data := s.data.Difference(other.data) @@ -462,6 +618,18 @@ func (s *rowSegment) Shift() (*rowSegment, error) { }, nil } +// GenericUnary returns s subject to op. +func (s *rowSegment) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *rowSegment { + //TODO deal with overflow + data := UnwrapBitmap(op([]ext.Bitmap{WrapBitmap(s.data)}, args)) + + return &rowSegment{ + data: data, + shard: s.shard, + n: data.Count(), + } +} + // SetBit sets the i-th column of the row. func (s *rowSegment) SetBit(i uint64) (changed bool) { s.ensureWritable() diff --git a/server.go b/server.go index bb4faf3e8..ddf43ed50 100644 --- a/server.go +++ b/server.go @@ -17,17 +17,21 @@ package pilosa import ( "context" "fmt" + "io" "log" "os" "os/exec" "path/filepath" + "plugin" "runtime" "strconv" "strings" "sync" "time" + "github.com/pilosa/pilosa/v2/ext" "github.com/pilosa/pilosa/v2/logger" + "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/stats" "github.com/pkg/errors" @@ -57,6 +61,8 @@ type Server struct { // nolint: maligned hosts []string clusterDisabled bool serializer Serializer + extensionPath string + extensions []*ext.ExtensionInfo // External systemInfo SystemInfo @@ -335,6 +341,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { if err != nil { return nil, err } + s.extensionPath = filepath.Join(path, ".extensions") s.holder.Path = path // s.holder.translateFile.Path = filepath.Join(path, ".keys") @@ -376,6 +383,30 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.cluster.broadcaster = s s.cluster.maxWritesPerRequest = s.maxWritesPerRequest s.holder.broadcaster = s + err = s.loadPlugins() + if err != nil { + s.logger.Printf("not all plugins loaded successfully") + } + if len(s.extensions) > 0 { + s.logger.Printf("loaded extensions:") + for _, ext := range s.extensions { + if ext == nil { + s.logger.Printf(" inexplicably, a nil extension?!?") + continue + } + s.logger.Printf(" %s %s: %s", ext.Name, ext.Version, ext.Description) + if ext.License != "" { + s.logger.Printf(" License: %s", ext.License) + } + if len(ext.BitmapOps) > 0 { + opList := make([]string, len(ext.BitmapOps)) + for i := range ext.BitmapOps { + opList[i] = ext.BitmapOps[i].Name + } + s.logger.Printf(" Ops: %s", strings.Join(opList, ", ")) + } + } + } err = s.cluster.setup() if err != nil { @@ -389,6 +420,95 @@ func (s *Server) InternalClient() InternalClient { return s.defaultClient } +func (s *Server) loadPlugins() error { + var anyError error + dir, err := os.Open(s.extensionPath) + if err != nil { + // don't complain about it not existing, that's fine. + if os.IsNotExist(err) { + s.logger.Printf("extension interface v0: no extensions directory.") + return nil + } + return errors.Wrap(err, "opening extension path:") + } + defer dir.Close() + for files, err := dir.Readdir(64); err != io.EOF; files, err = dir.Readdir(64) { + if err != nil { + return errors.Wrap(err, "searching extension directory:") + } + for _, file := range files { + name := file.Name() + // only .so files are likely plugins. + if !strings.HasSuffix(name, ".so") { + continue + } + // only regular files are candidates for loading. + mode := file.Mode() + if !mode.IsRegular() { + s.logger.Printf("extension file '%s' is not a regular file", name) + continue + } + err = s.loadPlugin(name) + if err != nil { + s.logger.Printf("loading extension %s: %v", name, err) + anyError = err + } + } + } + return anyError +} + +func (s *Server) loadPlugin(name string) error { + path := filepath.Join(s.extensionPath, name) + p, err := plugin.Open(path) + if err != nil { + return err + } + pluginExtInfo, err := p.Lookup("ExtensionInfo") + if err != nil { + return fmt.Errorf("%s: no ExtensionInfo found", name) + } + extInfoFunc, ok := pluginExtInfo.(func(string) (*ext.ExtensionInfo, error)) + if !ok { + return fmt.Errorf("%s: unexpected %T instead of ExtensionInfo object", name, pluginExtInfo) + } + extInfo, err := extInfoFunc("v0") + if err != nil { + return errors.Wrap(err, name) + } + if extInfo == nil { + return fmt.Errorf("%s: nil ExtensionInfo", name) + } + if extInfo.ExtensionAPI != "v0" { + return fmt.Errorf("%s: unsupported extension API %s", name, extInfo.ExtensionAPI) + } + s.extensions = append(s.extensions, extInfo) + bitmapOps := extInfo.BitmapOps + bmOps, countOps, fieldOps, unknownOps := 0, 0, 0, 0 + for i := range bitmapOps { + // title-case the name + bitmapOps[i].Name = strings.Title(bitmapOps[i].Name) + typ := bitmapOps[i].Func.BitmapOpType() + switch { + case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount: + countOps++ + case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap: + bmOps++ + case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap: + fieldOps++ + default: + unknownOps++ + } + } + err = s.executor.registerOps(bitmapOps) + if err != nil { + s.logger.Printf("warning: extension registration failed: %v", err) + } else { + pql.RegisterPluginFuncs(bitmapOps) + } + return nil +} + // UpAndDown brings the server up minimally and shuts it down // again; basically, it exists for testing holder open and close. func (s *Server) UpAndDown() error { From c5136b14db6ce0c3cbbf674a45c636945ab972d1 Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 17 Oct 2019 14:27:41 -0500 Subject: [PATCH 5/8] ensmarten snapshot queue The snapshot queue needs a bit more subtlety. In some cases, we really do want to do a snapshot right now -- these shouldn't have to wait for possibly a hundred or more other snapshots to complete. In other cases, we don't really care that much whether we do a snapshot, and just dropping it is probably fine. To accommodate this, we distinguish between "urgent" and "normal" snapshots, and between "Immediate" (does an urgent snapshot, waits for it) and "Enqueue" (might enqueue a snapshot but *also might not* if we're already busy). There's a corresponding "Await" to wait for a snapshot, if one is pending, but not if one isn't. We also have a background scan that checks the holder. It will scan pretty actively when it's finding fragments that need snapshots (no enqueued snapshot, opN > MaxOpN). It pauses for a second after every hundred fragments that didn't need snapshots, and for a minute after each holder scan that didn't find any. So, if you don't need snapshots, it does basically nothing, if you do, it'll be moderately aggressive about submitting tasks -- but it always waits if there's *any* requested snapshots in the queues. Updates since initial draft: Check results from Await more consistently, and in one case, use Immediate instead and then check its error. Fix a race condition. The race condition comes about if: 1. You have a limited enough worker pool that this can happen. (In testing we tend to have a worker pool of 1.) 2. A fragment is in the normal, non-urgent, queue already. 3. An immediate request comes in for that fragment. This always happens *with the fragment lock held*. 4. A worker thread grabs that fragment from the queue. 5. The worker thread now waits on the lock. Meanwhile, the immediate request blocks on sending the fragment to the urgent queue. 6. The worker can't read the urgent queue, and the immediate request can't send it, so the immediate request can't proceed. What's supposed to happen is that the immediate request sends the thing, and gets into Await(), which sleeps on a condition variable using the lock, which is to say, releases the lock. The obvious resolution is to let go of the lock, send the message, and then reclaim the lock. But then we have the possibility that the message sent ends up with a timestamp right after a snapshot that happened *after* the Immediate request was started. Oops. So we create the request, then let go of the lock, then send the request, then reclaim the lock and go into the Await state. All is well. This is on top of more general use of wait groups, etcetera, to allow us to ensure that any holder scans terminate *before* we close the channels they might otherwise be trying to write to. So, shutdown process is now: * grab lock on queue (workers and scanners don't use the lock) * mark snapshotqueue done * wait for holder scans to complete/exit * close and nil out all the channels * release lock Anything trying to submit to this needs to hold the lock, unless it's a holder scan, so either it got the lock before we did and already submitted the thing, or it will get the lock after this and not find a channel to write to; it's just the holder scanner that has an ongoing thing that might have started a write to the channel *without* a lock held, because it's expected that it might have to wait minutes or hours before the write will complete because it's a background task. Also, rework the background holder scan to grab lists of indexes/fields/views/fragments, then scan the grabbed/copied lists, rather than iterating over maps, allowing us to grab the lock when we're about to access a thing and let it go when done. There might be a simpler/cleaner way to do this but opinions on how safe it is are very mixed, so in the mean time, I'm making the range behavior not depend at all on there being no writes to the various tiers of holder/index/view/fragment during the background scans. --- field.go | 7 +- fragment.go | 149 +++----------- fragment_internal_test.go | 60 ++++-- holder.go | 8 +- index.go | 6 +- snapshotqueue.go | 405 ++++++++++++++++++++++++++++++++++++++ view.go | 6 +- 7 files changed, 495 insertions(+), 146 deletions(-) create mode 100644 snapshotqueue.go diff --git a/field.go b/field.go index 341695816..b6047e3a8 100644 --- a/field.go +++ b/field.go @@ -91,8 +91,7 @@ type Field struct { logger logger.Logger - snapshotQueue chan *fragment - + snapshotQueue snapshotQueue // Instantiates new translation store on open. OpenTranslateStore OpenTranslateStoreFunc } @@ -924,7 +923,9 @@ func (f *Field) newView(path, name string) *view { view.rowAttrStore = f.rowAttrStore view.stats = f.Stats view.broadcaster = f.broadcaster - view.snapshotQueue = f.snapshotQueue + if f.snapshotQueue != nil { + view.snapshotQueue = f.snapshotQueue + } return view } diff --git a/fragment.go b/fragment.go index 852b11389..794a5b43b 100644 --- a/fragment.go +++ b/fragment.go @@ -107,20 +107,16 @@ type fragment struct { shard uint64 // File-backed storage - path string - flags byte // user-defined flags passed to roaring - gen generation - storage *roaring.Bitmap - totalOpN int64 // total opN values - totalOps int64 // total ops (across all snapshots) - opN int // number of ops since snapshot (may be approximate for imports) - ops int // number of higher-level operations, as opposed to bit changes - snapshotsRequested int // number of times we've requested a snapshot - snapshotsTaken int // number of actual snapshot operations - snapshotting bool // set to true when requesting a snapshot, set to false after snapshot completes - snapshotCond sync.Cond - snapshotDelays int - snapshotDelayTime time.Duration + path string + flags byte // user-defined flags passed to roaring + gen generation + storage *roaring.Bitmap + opN int // number of ops since snapshot (may be approximate for imports) + ops int // number of higher-level operations, as opposed to bit changes + snapshotPending bool // set to true when requesting a snapshot, set to false after snapshot completes + snapshotCond sync.Cond + snapshotErr error // error yielded by the last snapshot operation + snapshotStamp time.Time // timestamp of last snapshot // Cache for row counts. CacheType string // passed in by field @@ -154,7 +150,7 @@ type fragment struct { stats stats.StatsClient - snapshotQueue chan *fragment + snapshotQueue snapshotQueue } // newFragment returns a new instance of Fragment. @@ -172,7 +168,8 @@ func newFragment(path, index, field, view string, shard uint64, flags byte) *fra Logger: logger.NopLogger, MaxOpN: defaultFragmentMaxOpN, - stats: stats.NopStatsClient, + stats: stats.NopStatsClient, + snapshotQueue: defaultSnapshotQueue, } f.snapshotCond = sync.Cond{L: &f.mu} return f @@ -181,62 +178,6 @@ func newFragment(path, index, field, view string, shard uint64, flags byte) *fra // cachePath returns the path to the fragment's cache data. func (f *fragment) cachePath() string { return f.path + cacheExt } -// newSnapshotQueue makes a new snapshot queue, of depth N, and spawns a -// goroutine for it. -func newSnapshotQueue(n int, w int, l logger.Logger) chan *fragment { - ch := make(chan *fragment, n) - for i := 0; i < w; i++ { - go snapshotQueueWorker(ch, l) - } - return ch -} - -func snapshotQueueWorker(snapshotQueue chan *fragment, l logger.Logger) { - for f := range snapshotQueue { - err := f.protectedSnapshot(true) - if err != nil { - l.Printf("snapshot error: %v", err) - } - f.snapshotCond.Broadcast() - } -} - -// enqueueSnapshot requests that the fragment be snapshotted at some point -// in the future, if this has not already been requested. Call this only when -// the mutex is held. -func (f *fragment) enqueueSnapshot() { - f.snapshotsRequested++ - if f.snapshotting { - return - } - f.snapshotting = true - if f.snapshotQueue != nil { - select { - case f.snapshotQueue <- f: - default: - before := time.Now() - // wait forever, but notice that we're waiting - f.snapshotQueue <- f - f.snapshotDelays++ - f.snapshotDelayTime += time.Since(before) - if f.snapshotDelays >= 10 { - f.Logger.Printf("snapshotting %s: last ten enqueue delays took %v", f.path, f.snapshotDelayTime) - f.snapshotDelays = 0 - f.snapshotDelayTime = 0 - } - } - } else { - // in testing, for instance, there may be no holder, thus no one - // to handle these snapshots. - err := f.snapshot() - if err != nil { - f.Logger.Printf("snapshot failed: %v", err) - } - f.snapshotting = false - f.snapshotCond.Broadcast() - } -} - // Open opens the underlying storage. func (f *fragment) Open() error { f.mu.Lock() @@ -445,31 +386,12 @@ func (f *fragment) openCache() error { func (f *fragment) Close() error { f.mu.Lock() defer f.mu.Unlock() - for f.snapshotting { + for f.snapshotPending { f.snapshotCond.Wait() } return f.close() } -// awaitSnapshot lets us delay until the snapshot gets written, preventing tests -// from misleadingly showing amazingly fast performance because the snapshots they -// trigger haven't happened yet. -func (f *fragment) awaitSnapshot() { - f.mu.Lock() - defer f.mu.Unlock() - for f.snapshotting { - f.snapshotCond.Wait() - } -} - -// unprotectedAwaitSnapshot assumes you already hold the lock, and waits for -// the snapshot fairy to come along. -func (f *fragment) unprotectedAwaitSnapshot() { - for f.snapshotting { - f.snapshotCond.Wait() - } -} - func (f *fragment) close() error { // Flush cache if closing gracefully. if err := f.flushCache(); err != nil { @@ -727,7 +649,7 @@ func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err f.rowCache.Add(rowID, nil) // Snapshot storage. - f.enqueueSnapshot() + f.snapshotQueue.Enqueue(f) f.stats.Count("setRow", 1, 1.0) return changed, nil @@ -768,7 +690,7 @@ func (f *fragment) unprotectedClearRow(rowID uint64) (changed bool, err error) { f.rowCache.Add(rowID, nil) // Snapshot storage. - f.enqueueSnapshot() + f.snapshotQueue.Enqueue(f) f.stats.Count("clearRow", 1, 1.0) @@ -2126,19 +2048,20 @@ func (f *fragment) importValue(columnIDs []uint64, values []int64, bitDepth uint _ = f.openStorage(true) return err } - // We don't actually care, except we want our stats to be accurate. - f.incrementOpN(totalChanges) + // Keep stats accurate. We don't call incrementOpN here because it may + // or may not enqueue a request, which would then be in the queue + // taking up space and otherwise being a possible nuisance, when we're + // about to force a snapshot anyway. + f.opN += totalChanges + f.ops++ // Reset the rowCache. f.rowCache = &simpleCache{make(map[uint64]*Row)} - // in theory, this should probably have happened anyway, but if enough + // in theory, this should probably have been queued anyway, but if enough // of the bits matched existing bits, we'll be under our opN estimate, and // we want to ensure that the snapshot happens. - f.enqueueSnapshot() - f.unprotectedAwaitSnapshot() - - return nil + return f.snapshotQueue.Immediate(f) } // importRoaring imports from the official roaring data format defined at @@ -2198,14 +2121,14 @@ func (f *fragment) incrementOpN(changed int) { f.opN += changed f.ops++ if f.opN > f.MaxOpN { - f.enqueueSnapshot() + f.snapshotQueue.Enqueue(f) } } // Snapshot writes the storage bitmap to disk and reopens it. This may // coexist with existing background-queue snapshotting; it does not remove // things from the queue. You probably don't want to do this; use -// enqueueSnapshot/awaitSnapshot. +// the snapshotQueue's Enqueue/Await. func (f *fragment) Snapshot() error { f.mu.Lock() defer f.mu.Unlock() @@ -2218,25 +2141,13 @@ func track(start time.Time, message string, stats stats.StatsClient, logger logg stats.Histogram("snapshot", elapsed.Seconds(), 1.0) } -// protectedSnapshot grabs the lock and unconditionally calls snapshot(). If -// fromQueue is true, the snapshotting state is also cleared. -func (f *fragment) protectedSnapshot(fromQueue bool) error { - f.mu.Lock() - defer f.mu.Unlock() - err := f.snapshot() - if fromQueue { - f.snapshotting = false - } - return err -} - // snapshot does the actual snapshot operation. it does not check or care -// about f.snapshotting. +// about f.snapshotPending. func (f *fragment) snapshot() error { - f.totalOpN += int64(f.opN) - f.totalOps += int64(f.ops) - f.snapshotsTaken++ _, err := unprotectedWriteToFragment(f, f.storage) + if err == nil { + f.snapshotStamp = time.Now() + } return err } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 13d47181b..229021907 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -2070,7 +2070,11 @@ func BenchmarkImportRoaring(b *testing.B) { b.StartTimer() err := f.importRoaringT(data, false) if err != nil { - f.awaitSnapshot() + // we don't actually particularly + // care whether this succeeds, + // but if it's happening we want + // it to be done. + _ = f.snapshotQueue.Await(f) f.Clean(b) b.Fatalf("import error: %v", err) } @@ -2109,7 +2113,9 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) { j := j eg.Go(func() error { err := frags[j].importRoaringT(data[j], false) - frags[j].awaitSnapshot() + // error unimportant if it happened, but we want + // any snapshots to have finished. + _ = frags[j].snapshotQueue.Await(frags[j]) return err }) } @@ -2147,11 +2153,13 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { // is excessive. force storage into snapshotted state, then use import // to generate an op log and/or snapshot. _, _, err := frags[j].storage.ImportRoaringBits(data, false, false, 0) - frags[j].enqueueSnapshot() - frags[j].awaitSnapshot() if err != nil { b.Fatalf("importing roaring: %v", err) } + err = frags[j].snapshotQueue.Immediate(frags[j]) + if err != nil { + b.Fatalf("snapshot after import: %v", err) + } } eg := errgroup.Group{} b.StartTimer() @@ -2159,7 +2167,10 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { j := j eg.Go(func() error { err := frags[j].importRoaringT(updata, false) - frags[j].awaitSnapshot() + err2 := frags[j].snapshotQueue.Await(frags[j]) + if err == nil { + err = err2 + } return err }) } @@ -2221,18 +2232,23 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { // is excessive. force storage into snapshotted state, then use import // to generate an op log and/or snapshot. _, _, err := f.storage.ImportRoaringBits(data, false, false, 0) - f.enqueueSnapshot() - f.awaitSnapshot() if err != nil { b.Errorf("import error: %v", err) } + err = f.snapshotQueue.Immediate(f) + if err != nil { + b.Errorf("snapshot after import error: %v", err) + } b.StartTimer() err = f.importRoaringT(updata, false) - f.awaitSnapshot() if err != nil { f.Clean(b) b.Errorf("import error: %v", err) } + err = f.snapshotQueue.Await(f) + if err != nil { + b.Errorf("snapshot after import error: %v", err) + } b.StopTimer() var stat os.FileInfo var statTarget io.Writer @@ -2542,7 +2558,12 @@ func (f *fragment) sanityCheck(t testing.TB) { } func (f *fragment) Clean(t testing.TB) { - f.awaitSnapshot() + f.mu.Lock() + err := f.snapshotQueue.Await(f) + f.mu.Unlock() + if err != nil { + t.Fatalf("snapshot failed before sanity check: %v", err) + } f.sanityCheck(t) if f.storage != nil && f.storage.Source != nil { if f.storage.Source.Dead() { @@ -2558,7 +2579,7 @@ func (f *fragment) Clean(t testing.TB) { t.Fatal("cleaning up fragment: ", errc, errf, errp) } if f.snapshotQueue != nil { - close(f.snapshotQueue) + f.snapshotQueue.Stop() f.snapshotQueue = nil } // not all fragments have cache files @@ -2581,7 +2602,7 @@ func (f *fragment) CleanKeep(t testing.TB) { t.Fatal("closing fragment: ", errc, errp) } if f.snapshotQueue != nil { - close(f.snapshotQueue) + f.snapshotQueue.Stop() f.snapshotQueue = nil } // not all fragments have cache files @@ -3069,10 +3090,10 @@ func TestFragmentRowIterator(t *testing.T) { func TestUnionInPlaceMapped(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeNone) + // note: clean has to be deferred first, because it has to run with + // the lock *not* held, because it is sometimes so it has to grab the + // lock... defer f.Clean(t) - // I know this doesn't actually matter in our current context, but - // strictly speaking, we do say you have to hold the lock while calling - // unprotectedWriteToFragment... f.mu.Lock() defer f.mu.Unlock() r0 := rand.New(rand.NewSource(2)) @@ -3105,8 +3126,15 @@ func TestUnionInPlaceMapped(t *testing.T) { f.storage.UnionInPlace(setBM1) countUnion := f.storage.Count() // UnionInPlace produces no ops log, we have to make it snapshot, to - // ensure that the on-disk representation is correct. - f.enqueueSnapshot() + // ensure that the on-disk representation is correct. Note, UIP is + // not used for things that are modifying real fragments, usually; + // it's used only in computation of things that usually don't go to + // disk, which is why we handle this specially in testing and not + // generically. + err = f.snapshotQueue.Immediate(f) + if err != nil { + t.Fatalf("snapshot after union-in-place: %v", err) + } if count0 != countF { t.Fatalf("writing bitmap to storage changed count: %d => %d", count0, countF) diff --git a/holder.go b/holder.go index d54fae43e..5045bff4d 100644 --- a/holder.go +++ b/holder.go @@ -75,7 +75,7 @@ type Holder struct { Logger logger.Logger - snapshotQueue chan *fragment + snapshotQueue snapshotQueue // Manages replication from the primary node. primaryTranslateNode *Node @@ -167,7 +167,7 @@ func (h *Holder) Open() error { // Run snapshots asynchronously. The snapshotQueue will have a background // task associated with it which flushes it and waits until this channel // is closed, so we should always close this channel when done. - h.snapshotQueue = newSnapshotQueue(100, 2, h.Logger) + h.snapshotQueue = newSnapshotQueue(10, 2, h.Logger) for _, fi := range fis { // Skip files or hidden directories. @@ -203,6 +203,7 @@ func (h *Holder) Open() error { go func() { defer h.wg.Done(); h.monitorCacheFlush() }() h.Stats.Open() + h.snapshotQueue.ScanHolder(h) h.opened.Close() return nil @@ -222,8 +223,7 @@ func (h *Holder) Close() error { } } if h.snapshotQueue != nil { - close(h.snapshotQueue) - // assuming the snapshotQueueWorker has already started, this is safe. + h.snapshotQueue.Stop() h.snapshotQueue = nil } diff --git a/index.go b/index.go index 5c73809c1..c96e3bd5f 100644 --- a/index.go +++ b/index.go @@ -58,7 +58,7 @@ type Index struct { Stats stats.StatsClient logger logger.Logger - snapshotQueue chan *fragment + snapshotQueue snapshotQueue // Used for notifying holder when a field is added. holder *Holder @@ -462,7 +462,9 @@ func (i *Index) newField(path, name string) (*Field, error) { f.Stats = i.Stats f.broadcaster = i.broadcaster f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data")) - f.snapshotQueue = i.snapshotQueue + if i.snapshotQueue != nil { + f.snapshotQueue = i.snapshotQueue + } f.OpenTranslateStore = i.OpenTranslateStore return f, nil } diff --git a/snapshotqueue.go b/snapshotqueue.go new file mode 100644 index 000000000..210c2b772 --- /dev/null +++ b/snapshotqueue.go @@ -0,0 +1,405 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "fmt" + "os" + "sync" + "time" + + "github.com/pilosa/pilosa/v2/logger" + "github.com/pkg/errors" +) + +// snapshotQueue is a thing which can handle enqueuing snapshots. A snapshot +// queue distinguishes between high-priority requests, which get satisfied +// by the next available worker, and regular requests, which get enqueued +// if there's space in the queue, and otherwise dropped. There's also a +// separate background task to scan a holder for fragments which may need +// snapshots, but which is processed only when the queue is empty, and only +// slowly. "Await" awaits an existing snapshot if one is already enqueued. +// "Immediate" tries to do one right away. (If one's already enqueued, this +// can leave it in the queue, which will ignore anything that shows up with +// the request flag cleared.) +// +// Await, Enqueue, and Immediate should be called only with the fragment lock +// held. +// +// ScanHolder spawns a new goroutine. You don't need to use `go` on it. +type snapshotQueue interface { + Immediate(*fragment) error + Enqueue(*fragment) + Await(*fragment) error + ScanHolder(*Holder) + Stop() +} + +// queuelessSnapshotQueue isn't a snapshot queue, but it satisfies the +// interface. +type queuelessSnapshotQueue struct{} + +func (q *queuelessSnapshotQueue) Enqueue(f *fragment) { + _ = f.snapshot() +} + +func (q *queuelessSnapshotQueue) Await(f *fragment) error { + return nil +} + +func (q *queuelessSnapshotQueue) Immediate(f *fragment) error { + return f.snapshot() +} + +func (q *queuelessSnapshotQueue) ScanHolder(h *Holder) { +} + +func (q *queuelessSnapshotQueue) Stop() { +} + +// defaultSnapshotQueue is the fallback to use if none is available, +// and currently uses queueless -- it runs all snapshots immediately. +var defaultSnapshotQueue *queuelessSnapshotQueue + +// newSnapshotQueue makes a new snapshot queue, of depth N, with +// w worker threads. +func newSnapshotQueue(n int, w int, l logger.Logger) snapshotQueue { + sq := prioritySnapshotQueue{normal: make(chan snapshotRequest, n), urgent: make(chan snapshotRequest), background: make(chan snapshotRequest), done: make(chan struct{}), logger: l} + if sq.logger == nil { + sq.logger = logger.NewStandardLogger(os.Stderr) + } + sq.spawnWorkers(w) + return &sq +} + +type snapshotRequest struct { + frag *fragment + when time.Time +} + +// prioritySnapshotQueue gives preference to "immediate" requests, and +// dispreference to "background" requests from ScanHolder. It timestamps +// requests, so it can discard a request if the most recent snapshot is +// newer than the request. The snapshotPending flag in the fragment is +// used to track that a given fragment thinks it has been successfully +// enqueued. Background requests are not considered enqueued, since +// they'll never get processed if there's anything else. In normal workloads, +// immediate/urgent snapshots should be rare, but we'll happily drop +// most requests on the floor; the scanner should pick them up once things +// are quiet. +type prioritySnapshotQueue struct { + logger logger.Logger + urgent chan snapshotRequest + normal chan snapshotRequest + background chan snapshotRequest + done chan struct{} + mu sync.RWMutex + scanWG, workerWG sync.WaitGroup + stats struct { + enqueued int64 + skipped int64 + } +} + +func (sq *prioritySnapshotQueue) spawnWorkers(w int) { + sq.mu.Lock() + defer sq.mu.Unlock() + if sq.done == nil { + sq.logger.Printf("prioritySnapshotQueue worker: no done channel, already done?") + return + } + sq.workerWG.Add(w) + for i := 0; i < w; i++ { + go sq.worker(sq.urgent, sq.normal, sq.background, sq.done) + } +} + +func (sq *prioritySnapshotQueue) worker(urgent, normal, background chan snapshotRequest, done chan struct{}) { + // We don't want a race condition on these. If they're non-nil when + // we get them, they should get closed at some point. If done is + // already nil, we shouldn't do anything. + defer sq.workerWG.Done() + ok := true + var req snapshotRequest + for ok { + req.frag = nil + + select { + case req, ok = <-urgent: + default: + select { + case req, ok = <-urgent: + case req, ok = <-normal: + default: + select { + case req, ok = <-urgent: + case req, ok = <-normal: + case req, ok = <-background: + case _, ok = <-done: + } + } + } + if req.frag != nil { + sq.process(req) + } + } +} + +// process actually runs a fragment. it will do this if either the fragment +// has a pending snapshot, or the force flag is set. +func (sq *prioritySnapshotQueue) process(req snapshotRequest) { + f := req.frag + f.mu.Lock() + defer f.mu.Unlock() + if f.snapshotStamp.Before(req.when) { + f.snapshotErr = f.snapshot() + if f.snapshotErr != nil { + fmt.Printf("snapshot error: %v\n", f.snapshotErr) + sq.logger.Printf("snapshot error: %v", f.snapshotErr) + } + f.snapshotPending = false + f.snapshotCond.Broadcast() + } +} + +// Stop shuts down the snapshot queue. It first marks it as done, causing +// the background scanner(s), if any, to shut down, then waits for them, then +// closes and nils the queues. The background scanner has to get stopped +// because otherwise it might try to write to those closed queues. +func (sq *prioritySnapshotQueue) Stop() { + sq.mu.Lock() + defer sq.mu.Unlock() + close(sq.done) + // scanners need to be done before we close the other channels. + sq.scanWG.Wait() + sq.done = nil + close(sq.normal) + sq.normal = nil + close(sq.urgent) + sq.urgent = nil + close(sq.background) + sq.background = nil + sq.logger.Printf("snapshot queue: enqueued %d, skipped %d\n", sq.stats.enqueued, sq.stats.skipped) +} + +// Enqueue tries to add a fragment to the queue, if the fragment is not already +// enqueued. You should hold a lock on the fragment when calling this. +func (sq *prioritySnapshotQueue) Enqueue(f *fragment) { + if f.snapshotPending { + return + } + sq.mu.Lock() + defer sq.mu.Unlock() + if sq.normal == nil { + sq.logger.Printf("requested snapshot after snapshot queue was closed") + return + } + // we have to set this before enqueing, because it's + // otherwise possible that we're at the head of the queue, + // and the recipient gets the fragment before we execute the + // line after the send. + f.snapshotPending = true + // try to enqueue snapshot + select { + case sq.normal <- snapshotRequest{frag: f, when: time.Now()}: + sq.stats.enqueued++ + return + default: + sq.stats.skipped++ + f.snapshotPending = false + return + } +} + +// Await returns when f is not pending a snapshot. Call with the fragment lock +// held. Await waits on a condition variable inside f, associated with the +// fragment's lock, so this does not conflict with the lock being used for +// snapshots. +func (sq *prioritySnapshotQueue) Await(f *fragment) (err error) { + for f.snapshotPending { + f.snapshotCond.Wait() + } + err, f.snapshotErr = f.snapshotErr, nil + return err +} + +// Immediate forces an immediate snapshot of the given fragment. Call with +// the fragment locked. If the queue is already closing, the fragment does +// not get snapshotted. +func (sq *prioritySnapshotQueue) Immediate(f *fragment) error { + sq.mu.RLock() + // no deferred unlock, because we want to unlock this before calling Await. + // Not because that needs this lock, but because once we're that far, we + // *don't* need this lock anymore so someone else should have it. + if sq.urgent == nil { + sq.mu.RUnlock() + sq.logger.Printf("requested immediate snapshot after snapshot queue was closed") + return errors.New("requested immediate snapshot after snapshot queue was closed") + } + f.snapshotPending = true + req := snapshotRequest{frag: f, when: time.Now()} + // if the fragment was already in the work queue, it's *possible* + // that the only available worker just picked it off the queue, and + // is now waiting on getting the fragment's lock, so it can run + // a snapshot. So we let go of the lock on the fragment, send the + // request, then request the fragment lock again, because Await will + // be sleeping on the condition variable associated with the lock, + // which means it needs to hold the lock so it can let it go during + // the wait... No, really, this made sense. + f.mu.Unlock() + sq.urgent <- req + sq.mu.RUnlock() + f.mu.Lock() + return sq.Await(f) +} + +// needsSnapshot determines whether a fragment probably wants snapshotting. +// Specifically, it looks for fragments not already marked to receive +// snapshots, but which have a high enough opN to justify a snapshot. This +// is only used from the background scan. +func (sq *prioritySnapshotQueue) needsSnapshot(f *fragment) bool { + if f == nil { + return false + } + f.mu.Lock() + defer f.mu.Unlock() + if f.snapshotPending { + return false + } + if f.opN > f.MaxOpN { + return true + } + return false +} + +// ScanHolder spawns a goroutine which iterates through the holder's +// indexes/fields/views/fragments, looking for fragments which have OpN +// high enough to justify a snapshot but don't seem to have one pending. +// It then dumps these in the low priority background queue. +func (sq *prioritySnapshotQueue) ScanHolder(h *Holder) { + sq.mu.Lock() + sq.scanWG.Add(1) + go sq.scanHolderWorker(h, sq.background, sq.done) + sq.mu.Unlock() +} + +// scanHolderWorker is a background task that scans a holder looking for +// fragments which need snapshots taken. It's the cleanup task for snapshots +// that would have been requested by Enqueue, but the queue was full. +func (sq *prioritySnapshotQueue) scanHolderWorker(h *Holder, background chan snapshotRequest, done chan struct{}) { + defer sq.scanWG.Done() + var indexNames, fieldNames, viewNames []string + var fragNums []uint64 + for { + // To avoid abusing things, cap activity rate; every time we finish + // the holder, or every couple hundred fragments considered, we + // pause for a bit. + counter := 0 + hits := 0 + h.mu.Lock() + indexNames = indexNames[:0] + for indexName := range h.indexes { + indexNames = append(indexNames, indexName) + } + h.mu.Unlock() + for _, indexName := range indexNames { + h.mu.Lock() + index := h.indexes[indexName] + h.mu.Unlock() + if index == nil { + continue + } + fieldNames = fieldNames[:0] + index.mu.Lock() + for fieldName := range index.fields { + fieldNames = append(fieldNames, fieldName) + } + index.mu.Unlock() + for _, fieldName := range fieldNames { + index.mu.Lock() + field := index.fields[fieldName] + index.mu.Unlock() + if field == nil { + continue + } + viewNames = viewNames[:0] + field.mu.Lock() + for viewName := range field.viewMap { + viewNames = append(viewNames, viewName) + } + field.mu.Unlock() + for _, viewName := range viewNames { + field.mu.Lock() + view := field.viewMap[viewName] + field.mu.Unlock() + if view == nil { + continue + } + fragNums := fragNums[:0] + view.mu.Lock() + for fragNum := range view.fragments { + fragNums = append(fragNums, fragNum) + } + view.mu.Unlock() + for _, fragNum := range fragNums { + view.mu.Lock() + frag := view.fragments[fragNum] + view.mu.Unlock() + if sq.needsSnapshot(frag) { + hits++ + select { + case background <- snapshotRequest{frag: frag, when: time.Now()}: + sq.logger.Debugf("found fragment needing snapshot: %s\n", frag.path) + case <-done: + return + } + } else { + // Count fragments examined *without* finding anything that + // needed a snapshot. When we find things that need snapshots, + // the time it takes the workers to respond to us is enough + // of a delay to keep us from eating every CPU. So, if a lot + // of things need snapshots, and the workers aren't doing + // anything else, ScanHolder will mostly keep them saturated. + // If they're busy, we'll block forever in the write to the + // background queue. If there's nothing that needs snapshots, + // we pause frequently for a second or so at a time. + counter++ + if counter == 100 { + select { + case <-time.After(1 * time.Second): + case <-done: + return + } + counter = 0 + } + } + } + } + } + } + if hits > 0 { + sq.logger.Printf("background scan: %d fragments needed snapshots\n", hits) + hits = 0 + } else { + sq.logger.Printf("background scan: no fragments needed snapshots, waiting\n") + // No reason to be active if we're not finding anything. + select { + case <-time.After(60 * time.Second): + case <-done: + return + } + } + } +} diff --git a/view.go b/view.go index 69ae1b22f..e1648a838 100644 --- a/view.go +++ b/view.go @@ -59,7 +59,7 @@ type view struct { stats stats.StatsClient rowAttrStore AttrStore logger logger.Logger - snapshotQueue chan *fragment + snapshotQueue snapshotQueue } // newView returns a new instance of View. @@ -309,7 +309,9 @@ func (v *view) newFragment(path string, shard uint64) *fragment { frag.CacheSize = v.cacheSize frag.Logger = v.logger frag.stats = v.stats - frag.snapshotQueue = v.snapshotQueue + if v.snapshotQueue != nil { + frag.snapshotQueue = v.snapshotQueue + } if v.fieldType == FieldTypeMutex { frag.mutexVector = newRowsVector(frag) } else if v.fieldType == FieldTypeBool { From 8cc7a176b59dc275915b64aa53016ff8afbb4b98 Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 6 Nov 2019 12:37:54 -0600 Subject: [PATCH 6/8] license header fixups Fix up license headers for the extension code, and add the proto file to the list of things we don't check license headers for. --- ext/ext.go | 14 ++++++++++++++ ext/samples/some/some.go | 14 ++++++++++++++ extension.go | 14 ++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/ext/ext.go b/ext/ext.go index 0d01a9970..b6daa3ef0 100644 --- a/ext/ext.go +++ b/ext/ext.go @@ -1,3 +1,17 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Package ext provides an EXPERIMENTAL AND TEMPORARY interface to use for // plugin extensions to Pilosa. DO NOT DEVELOP NEW PLUGINS WITH THIS. The // replacement design is already in process, but it needs more refinement diff --git a/ext/samples/some/some.go b/ext/samples/some/some.go index 421a51bbb..07088d255 100644 --- a/ext/samples/some/some.go +++ b/ext/samples/some/some.go @@ -1,3 +1,17 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( diff --git a/extension.go b/extension.go index f9758a451..d449f3f0c 100644 --- a/extension.go +++ b/extension.go @@ -1,3 +1,17 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( From 03f3f424aa81498d7cc33a9615b56448af9955b5 Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 6 Nov 2019 13:04:59 -0600 Subject: [PATCH 7/8] don't lint PEG files I was pretty sure I'd done this, but I guess not: Skip linting the PEG files. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 241480b41..a25195ef8 100644 --- a/Makefile +++ b/Makefile @@ -141,7 +141,7 @@ docker-test: # Run golangci-lint golangci-lint: require-golangci-lint - golangci-lint run + golangci-lint run --skip-files '.*\.peg\.go' # Run gometalinter with custom flags gometalinter: require-gometalinter vendor From 1e0873c70bb4ea0385ba7532bc46f009b61e3233 Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 6 Nov 2019 13:51:53 -0600 Subject: [PATCH 8/8] lock BufferLogger for reads/writes With the new addition of the holder background scan, it's possible for an open holder to write log messages at arbitrary times. The TestHolder_Open/ErrIndexName test checks the contents of the output buffer, but those contents could be changing if the background task happens to run at the right time. Use trivial locking around that so that this shouldn't happen. --- test/logger.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/logger.go b/test/logger.go index 60af32acb..ecdfbfc40 100644 --- a/test/logger.go +++ b/test/logger.go @@ -18,12 +18,14 @@ import ( "bytes" "fmt" "io/ioutil" + "sync" ) // bufferLogger represents a test Logger that holds log messages // in a buffer for review. type bufferLogger struct { buf *bytes.Buffer + mu sync.Mutex } // NewBufferLogger returns a new instance of BufferLogger. @@ -34,6 +36,8 @@ func NewBufferLogger() *bufferLogger { } func (b *bufferLogger) Printf(format string, v ...interface{}) { + b.mu.Lock() + defer b.mu.Unlock() s := fmt.Sprintf(format, v...) _, err := b.buf.WriteString(s) if err != nil { @@ -44,5 +48,7 @@ func (b *bufferLogger) Printf(format string, v ...interface{}) { func (b *bufferLogger) Debugf(format string, v ...interface{}) {} func (b *bufferLogger) ReadAll() ([]byte, error) { + b.mu.Lock() + defer b.mu.Unlock() return ioutil.ReadAll(b.buf) }