From 6e3e5134251e105a1ff1814a72caef9367984db2 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Tue, 19 May 2020 11:35:30 -0400 Subject: [PATCH 01/45] roaring: fix use-after-free in b-tree bitmap update --- roaring/btree.go | 9 +++++++++ roaring/btree_test.go | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/roaring/btree.go b/roaring/btree.go index 1051f8985..bbc429704 100644 --- a/roaring/btree.go +++ b/roaring/btree.go @@ -925,6 +925,15 @@ func (e *enumerator) Every(upd func(key uint64, oldV *Container, exists bool) (n if write { if nv == nil { e.t.Delete(i.k) + f, _ := e.t.Seek(e.k) + *e = *f + f.Close() + // we don't want to e.next() here; we'll + // already be on an item with key >= i.k, + // and since we just deleted the item with + // key i.k, that means key is > i.k, which + // makes it the next item. + continue } else { e.q.d[e.i].v = nv } diff --git a/roaring/btree_test.go b/roaring/btree_test.go index 422a7b767..c3c02af3d 100644 --- a/roaring/btree_test.go +++ b/roaring/btree_test.go @@ -11,6 +11,7 @@ import ( "math" "math/rand" "path" + "reflect" "runtime" "runtime/debug" "strings" @@ -998,6 +999,28 @@ func TestBtreeEnumeratorPrevSanity(t *testing.T) { } } +// TestBtreeEnumeratorEveryRegression is a regression test for a "use-after-free" bug. +// Previously, deleting a container would cause some values to be skipped (and sometimes trigger a race condition). +func TestBtreeEnumeratorEveryRegression(t *testing.T) { + r := treeNew() + + r.Set(uint64(10), getDummyC(100)) + r.Set(uint64(20), getDummyC(200)) + r.Set(uint64(30), getDummyC(300)) + + e, _ := r.Seek(0) + expect := []uint64{10, 20, 30} + var found []uint64 + _ = e.Every(func(key uint64, oldV *Container, exists bool) (*Container, bool) { + found = append(found, key) + return nil, true + }) + + if !reflect.DeepEqual(expect, found) { // Before the fix, this skipped the 20. + t.Errorf("had %v in bitmap; only found %v", expect, found) + } +} + func BenchmarkBtreeSeekSeq1e3(b *testing.B) { benchmarkSeekSeq(b, 1e3) } From 0a94f8393f9970035d9bcc6e730f55a4c48c6296 Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 21 May 2020 13:26:55 -0500 Subject: [PATCH 02/45] Address TODOs in roaring tests In addition to adding some tests, this commit moves the `GenerateUint64Slice()` helper function into a new `generator` package so that it can be used in both internal and non-internal tests. --- generator/slice.go | 51 ++++++++++++++ roaring/roaring_internal_test.go | 8 ++- roaring/roaring_test.go | 114 +++++++++++++++++++------------ 3 files changed, 127 insertions(+), 46 deletions(-) create mode 100644 generator/slice.go diff --git a/generator/slice.go b/generator/slice.go new file mode 100644 index 000000000..595f5a132 --- /dev/null +++ b/generator/slice.go @@ -0,0 +1,51 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package generator + +import ( + "math/rand" + "sort" +) + +// Uint64Slice generates between [0, n) random uint64 numbers between min and max. +func Uint64Slice(n int, min, max uint64, sorted bool, rand *rand.Rand) []uint64 { + a := make([]uint64, rand.Intn(n)) + for i := range a { + a[i] = min + uint64(rand.Int63n(int64(max-min))) + } + + if sorted { + sort.Sort(uint64Slice(a)) + } + + return a +} + +// Uint64SetSlice returns the values in a uint64 set. +func Uint64SetSlice(m map[uint64]struct{}) []uint64 { + a := make([]uint64, 0, len(m)) + for v := range m { + a = append(a, v) + } + sort.Sort(uint64Slice(a)) + return a +} + +// uint64Slice represents a sortable slice of uint64 numbers. +type uint64Slice []uint64 + +func (u uint64Slice) Swap(i, j int) { u[i], u[j] = u[j], u[i] } +func (u uint64Slice) Len() int { return len(u) } +func (u uint64Slice) Less(i, j int) bool { return u[i] < u[j] } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 9e7a468fc..7784064ab 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -19,11 +19,13 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "math/rand" "reflect" "runtime" "strings" "testing" + "github.com/pilosa/pilosa/v2/generator" "github.com/pkg/errors" ) @@ -4026,7 +4028,11 @@ func TestDirectAddNVsAdd(t *testing.T) { {9384932, 101000, 2, 1, 0}, {3489, 19230, 394, 0, 893982, 890283, 14, 7}, } - // TODO generate more tests and fuzz + // Add some randomly created tests. + rand := rand.New(rand.NewSource(1)) + for i := 0; i < 100; i++ { + tests = append(tests, generator.Uint64Slice(1+rand.Intn(1000), 0, 10000000, i%2 == 0, rand)) + } testsCopy := make([][]uint64, len(tests)) copy(testsCopy, tests) for i, test := range testsCopy { diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index a5c4a9a25..4f079102a 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -20,12 +20,12 @@ import ( "math" "math/rand" "reflect" - "sort" "testing" "testing/quick" "time" "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/generator" "github.com/pilosa/pilosa/v2/roaring" _ "github.com/pilosa/pilosa/v2/test" ) @@ -280,11 +280,36 @@ func TestBitmap_Slice_Empty(t *testing.T) { } // Ensure a bitmap can return a slice of values within a range. -// TODO duplicate for all container types func TestBitmap_SliceRange(t *testing.T) { - if a := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) { - t.Fatalf("unexpected slice: %+v", a) - } + t.Run("array", func(t *testing.T) { + if a := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) { + t.Fatalf("unexpected slice: %+v", a) + } + }) + + t.Run("bitmap", func(t *testing.T) { + bm := roaring.NewFileBitmap() + for i := uint64(10); i < 10000; i++ { + _, _ = bm.Add(i * 2) + } + bm.Optimize() + + if a := bm.SliceRange(20, 30); !reflect.DeepEqual(a, []uint64{20, 22, 24, 26, 28}) { + t.Fatalf("unexpected slice: %+v", a) + } + }) + + t.Run("run", func(t *testing.T) { + bm := roaring.NewFileBitmap() + for i := uint64(0); i < 11; i++ { + _, _ = bm.Add(i) + } + bm.Optimize() + + if a := bm.SliceRange(6, 10); !reflect.DeepEqual(a, []uint64{6, 7, 8, 9}) { + t.Fatalf("unexpected slice: %+v", a) + } + }) } // Ensure a bitmap can loop over a set of values. @@ -1442,7 +1467,7 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) { // If `got` is nil and `exp` has zero length, don't perform the DeepEqual // because when `a` is empty (`a = []uint64{}`) then `got` is a nil slice // while `exp` is an empty slice. Therefore they will not be considered equal. - if got, exp := bm.Slice(), uint64SetSlice(m); !(got == nil && len(exp) == 0) && !reflect.DeepEqual(got, exp) { + if got, exp := bm.Slice(), generator.Uint64SetSlice(m); !(got == nil && len(exp) == 0) && !reflect.DeepEqual(got, exp) { t.Fatalf("unexpected values:\n\ngot=%+v\n\nexp=%+v\n\n", got, exp) } @@ -1467,7 +1492,7 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) { return true }, &quick.Config{ Values: func(values []reflect.Value, rand *rand.Rand) { - values[0] = reflect.ValueOf(GenerateUint64Slice(n, min, max, false, rand)) + values[0] = reflect.ValueOf(generator.Uint64Slice(n, min, max, false, rand)) }, }) if err != nil { @@ -1478,7 +1503,9 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) { func TestBitmap_Marshal_Quick_Array1(t *testing.T) { testBitmapMarshalQuick(t, 1000, 1000, 2000, false) } -func TestBitmap_Marshal_Quick_Array2(t *testing.T) { testBitmapMarshalQuick(t, 10000, 0, 1000, false) } +func TestBitmap_Marshal_Quick_Array2(t *testing.T) { + testBitmapMarshalQuick(t, 10000, 0, 1000, false) +} func TestBitmap_Marshal_Quick_Bitmap1(t *testing.T) { testBitmapMarshalQuick(t, 10000, 0, 10000, false) } @@ -1494,6 +1521,12 @@ func TestBitmap_Marshal_Quick_Bitmap_Sorted(t *testing.T) { } // TODO update for RLE +// (travis) - it's not clear to me how to generate a run container +// using `testBitmapMarshalQuick`. Because it's randomly generated, +// even some of the "Bitmap" tests generate array containers. Also, +// I think in order for the container to be a run, we would need +// to call bm.Optimize() on the bitmap, and I'm hesitant to add that +// because it's not clear to me how that would affect the tests. // Ensure a bitmap can be marshaled and unmarshaled. func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { @@ -1538,12 +1571,12 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { } // Verify the original bitmap has the correct set of values. - if exp, got := uint64SetSlice(set), bm.Slice(); !reflect.DeepEqual(exp, got) { + if exp, got := generator.Uint64SetSlice(set), bm.Slice(); !reflect.DeepEqual(exp, got) { t.Fatalf("mismatch: %s\n\nexp=%+v\n\ngot=%+v\n\n", diff(exp, got), exp, got) } // Verify the bitmap loaded with the ops log has the correct set of values. - if exp, got := uint64SetSlice(set), bm2.Slice(); !reflect.DeepEqual(exp, got) { + if exp, got := generator.Uint64SetSlice(set), bm2.Slice(); !reflect.DeepEqual(exp, got) { t.Fatalf("mismatch: %s\n\nexp=%+v\n\ngot=%+v\n\n", diff(exp, got), exp, got) } } @@ -1551,8 +1584,8 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { return true }, &quick.Config{ Values: func(values []reflect.Value, rand *rand.Rand) { - values[0] = reflect.ValueOf(GenerateUint64Slice(n, min, max, sorted, rand)) - values[1] = reflect.ValueOf(GenerateUint64Slice(100, min, max, sorted, rand)) + values[0] = reflect.ValueOf(generator.Uint64Slice(n, min, max, sorted, rand)) + values[1] = reflect.ValueOf(generator.Uint64Slice(100, min, max, sorted, rand)) }, }) if err != nil { @@ -1561,9 +1594,8 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { } // Ensure iterator can iterate over all the values on the bitmap. -// TODO duplicate for all container types func TestIterator(t *testing.T) { - t.Run("bitmap", func(t *testing.T) { + t.Run("array", func(t *testing.T) { itr := roaring.NewFileBitmap(1, 2, 3).Iterator() itr.Seek(0) @@ -1577,6 +1609,29 @@ func TestIterator(t *testing.T) { } }) + t.Run("bitmap", func(t *testing.T) { + bm := roaring.NewFileBitmap() + exp := []uint64{} + for i := uint64(0); i < 10000; i++ { + v := i * 2 + _, _ = bm.Add(v) + exp = append(exp, v) + } + bm.Optimize() + + itr := bm.Iterator() + itr.Seek(0) + + var a []uint64 + for v, eof := itr.Next(); !eof; v, eof = itr.Next() { + a = append(a, v) + } + + if !reflect.DeepEqual(a, exp) { + t.Fatalf("unexpected values: %+v", a) + } + }) + t.Run("run", func(t *testing.T) { bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 11; i++ { @@ -1784,37 +1839,6 @@ func getBenchData(tb testing.TB) *benchmarkSampleData { return data } -// GenerateUint64Slice generates between [0, n) random uint64 numbers between min and max. -func GenerateUint64Slice(n int, min, max uint64, sorted bool, rand *rand.Rand) []uint64 { - a := make([]uint64, rand.Intn(n)) - for i := range a { - a[i] = min + uint64(rand.Int63n(int64(max-min))) - } - - if sorted { - sort.Sort(uint64Slice(a)) - } - - return a -} - -// uint64SetSlice returns the values in a uint64 set. -func uint64SetSlice(m map[uint64]struct{}) []uint64 { - a := make([]uint64, 0, len(m)) - for v := range m { - a = append(a, v) - } - sort.Sort(uint64Slice(a)) - return a -} - -// uint64Slice represents a sortable slice of uint64 numbers. -type uint64Slice []uint64 - -func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p uint64Slice) Len() int { return len(p) } -func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] } - func diff(a, b []uint64) string { if len(a) != len(b) { return fmt.Sprintf("len: %d != %d", len(a), len(b)) From 041726fbf76868e2ddc0c95c2acea517293b2dc7 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 22 May 2020 10:59:54 -0500 Subject: [PATCH 03/45] clean up the TODOs and some comments --- roaring/roaring.go | 2 +- row.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 9605a6299..5924673ea 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -160,7 +160,7 @@ type Containers interface { // replace the given container. UpdateEvery(fn func(uint64, *Container, bool) (*Container, bool)) - // Iterator returns a Contiterator which after a call to Next(), a call to Value() will + // Iterator returns a ContainterIterator which after a call to Next(), a call to Value() will // return the first container at or after key. found will be true if a // container is found at key. Iterator(key uint64) (citer ContainerIterator, found bool) diff --git a/row.go b/row.go index e50c6ab67..f1c67123c 100644 --- a/row.go +++ b/row.go @@ -374,7 +374,7 @@ func (r *Row) Difference(others ...*Row) *Row { return &Row{segments: output} } -// GenericUnary returns the results of a generic op on r. +// GenericUnaryOp returns the results of a generic op on r. func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row { work := r var segments []rowSegment @@ -660,7 +660,8 @@ func (s *rowSegment) Xor(other *rowSegment) *rowSegment { // Shift returns s shifted by 1 bit. func (s *rowSegment) Shift() (*rowSegment, error) { - //TODO deal with overflow + // TODO: deal with overflow + // See issue: https://github.com/molecula/pilosa/issues/403 data, err := s.data.Shift(1) if err != nil { return nil, errors.Wrap(err, "shifting roaring data") @@ -675,9 +676,8 @@ func (s *rowSegment) Shift() (*rowSegment, error) { }, nil } -// GenericUnary returns s subject to op. +// GenericUnaryOp returns s subject to op. func (s *rowSegment) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *rowSegment { - //TODO deal with overflow data := UnwrapBitmap(op([]ext.Bitmap{WrapBitmap(s.data)}, args)) return &rowSegment{ From e4b9293f26e6b69ec406e1ff25cb1992fa63bf9a Mon Sep 17 00:00:00 2001 From: Travis Date: Sat, 16 May 2020 12:07:56 -0500 Subject: [PATCH 04/45] Add support for `int == null` --- executor.go | 41 +++++++++++++++++++++++++++++++++-------- executor_test.go | 15 ++++++++++++++- pql/parser_test.go | 3 ++- pql/pqlpeg_test.go | 8 ++++++++ 4 files changed, 57 insertions(+), 10 deletions(-) diff --git a/executor.go b/executor.go index 3aa677c44..28f0ea80f 100644 --- a/executor.go +++ b/executor.go @@ -2463,21 +2463,15 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c return nil, ErrFieldNotFound } - // EQ null (not implemented: flip frag.NotNull with max ColumnID) + // EQ null _exists - frag.NotNull() // NEQ null frag.NotNull() // BETWEEN a,b(in) BETWEEN/frag.RowBetween() // BETWEEN a,b(out) BETWEEN/frag.NotNull() // EQ frag.RangeOp // NEQ frag.RangeOp - // Handle `!= null`. + // Handle `!= null` and `== null`. if cond.Op == pql.NEQ && cond.Value == nil { - // Find bsiGroup. - bsig := f.bsiGroup(fieldName) - if bsig == nil { - return nil, ErrBSIGroupNotFound - } - // Retrieve fragment. frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if frag == nil { @@ -2486,6 +2480,37 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c return frag.notNull() + } else if cond.Op == pql.EQ && cond.Value == nil { + // Make sure the index supports existence tracking. + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } else if idx.existenceField() == nil { + return nil, errors.Errorf("index does not support existence tracking: %s", index) + } + + var existenceRow *Row + existenceFrag := e.Holder.fragment(index, existenceFieldName, viewStandard, shard) + if existenceFrag == nil { + existenceRow = NewRow() + } else { + existenceRow = existenceFrag.row(0) + } + + var notNull *Row + var err error + + // Retrieve notNull from fragment if it exists. + if frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard); frag != nil { + if notNull, err = frag.notNull(); err != nil { + return nil, errors.Wrap(err, "getting fragment not null") + } + } else { + notNull = NewRow() + } + + return existenceRow.Difference(notNull), nil + } else if cond.Op == pql.BETWEEN || cond.Op == pql.BTWN_LT_LT || cond.Op == pql.BTWN_LTE_LT || cond.Op == pql.BTWN_LT_LTE { predicates, err := getCondIntSlice(f, cond) diff --git a/executor_test.go b/executor_test.go index e29dd6b99..dfb5d3af3 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2371,7 +2371,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} - idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{TrackExistence: true}) if err != nil { t.Fatal(err) } @@ -2414,6 +2414,19 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { } t.Run("EQ", func(t *testing.T) { + // EQ null + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other == null)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{1, + 50, + ShardWidth, + ShardWidth + 1, + ShardWidth + 2, + (5 * ShardWidth) + 100, + }, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) + } + // EQ if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 20)`}); err != nil { t.Fatal(err) } else if got, exp := result.Results[0].(*pilosa.Row).Columns(), []uint64{50, (5 * ShardWidth) + 100}; !reflect.DeepEqual(exp, got) { diff --git a/pql/parser_test.go b/pql/parser_test.go index 29c358865..54bb9e332 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -173,7 +173,7 @@ func TestParser_Parse(t *testing.T) { // Parse with condition arguments. t.Run("WithCondition", func(t *testing.T) { - q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`) + q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null, n == null)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], @@ -185,6 +185,7 @@ func TestParser_Parse(t *testing.T) { "y": &pql.Condition{Op: pql.GTE, Value: int64(100)}, "z": &pql.Condition{Op: pql.BETWEEN, Value: []interface{}{int64(4), int64(8)}}, "m": &pql.Condition{Op: pql.NEQ, Value: nil}, + "n": &pql.Condition{Op: pql.EQ, Value: nil}, }, }, ) { diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 48eaf10c4..4ebf1dcc9 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -238,8 +238,16 @@ func TestPEGWorking(t *testing.T) { name: "RangeEQ", input: "Row(a == 4)", ncalls: 1}, + { + name: "RangeEQNULL", + input: "Row(a == null)", + ncalls: 1}, { name: "RangeNEQ", + input: "Row(a != 4)", + ncalls: 1}, + { + name: "RangeNEQNull", input: "Row(a != null)", ncalls: 1}, { From 5644fb2275c6feb4edb472089e050cc489d1a5a1 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Wed, 20 May 2020 12:27:27 -0400 Subject: [PATCH 05/45] add metrics for transactions --- api.go | 39 ++++++++++++++++++++++++++++++++++++--- metrics.go | 7 +++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/api.go b/api.go index 77a3fc34e..e617837c2 100644 --- a/api.go +++ b/api.go @@ -1594,14 +1594,41 @@ func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Du if err := api.validate(apiStartTransaction); err != nil { return nil, errors.Wrap(err, "validating api method") } - return api.server.StartTransaction(ctx, id, timeout, exclusive, remote) + t, err := api.server.StartTransaction(ctx, id, timeout, exclusive, remote) + if exclusive { + switch err { + case nil: + api.holder.Stats.Count(MetricExclusiveTransactionRequest, 1, 1.0) + case ErrTransactionExclusive: + api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0) + } + if t.Active { + api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0) + } + } else { + switch err { + case nil: + api.holder.Stats.Count(MetricTransactionStart, 1, 1.0) + case ErrTransactionExclusive: + api.holder.Stats.Count(MetricTransactionBlocked, 1, 1.0) + } + } + return t, err } func (api *API) FinishTransaction(ctx context.Context, id string, remote bool) (*Transaction, error) { if err := api.validate(apiFinishTransaction); err != nil { return nil, errors.Wrap(err, "validating api method") } - return api.server.FinishTransaction(ctx, id, remote) + t, err := api.server.FinishTransaction(ctx, id, remote) + if err == nil { + if t.Exclusive { + api.holder.Stats.Count(MetricExclusiveTransactionEnd, 1, 1.0) + } else { + api.holder.Stats.Count(MetricTransactionEnd, 1, 1.0) + } + } + return t, err } func (api *API) Transactions(ctx context.Context) (map[string]*Transaction, error) { @@ -1615,7 +1642,13 @@ func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Tr if err := api.validate(apiGetTransaction); err != nil { return nil, errors.Wrap(err, "validating api method") } - return api.server.GetTransaction(ctx, id, remote) + t, err := api.server.GetTransaction(ctx, id, remote) + if err == nil { + if t.Exclusive && t.Active { + api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0) + } + } + return t, err } type serverInfo struct { diff --git a/metrics.go b/metrics.go index c1888e0f6..8d58f8a73 100644 --- a/metrics.go +++ b/metrics.go @@ -58,4 +58,11 @@ const ( MetricStackInuse = "stack_inuse" MetricMallocs = "mallocs" MetricFrees = "frees" + MetricTransactionStart = "transaction_start" + MetricTransactionEnd = "trasaction_end" + MetricTransactionBlocked = "transaction_blocked" + MetricExclusiveTransactionRequest = "transaction_exclusive_request" + MetricExclusiveTransactionActive = "transaction_exclusive_active" + MetricExclusiveTransactionEnd = "trasaction_exclusive_end" + MetricExclusiveTransactionBlocked = "transaction_exclusive_blocked" ) From a2f825a32e5e0e15f59a9c7aee4f871aeda9c2b4 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 26 May 2020 17:18:23 -0500 Subject: [PATCH 06/45] added some context to tracing --- api.go | 11 ++++++++++- executor.go | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/api.go b/api.go index 77a3fc34e..eb1587a13 100644 --- a/api.go +++ b/api.go @@ -1009,6 +1009,9 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp if err != nil { return errors.Wrap(err, "getting index and field") } + span.LogKV( + "index", req.Index, + "field", req.Field) // Unless explicitly ignoring key validation (meaning keys have been // translated to ids in a previous step at the coordinator node), then @@ -1016,6 +1019,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp if !options.IgnoreKeyCheck { // Translate row keys. if field.Keys() { + span.LogKV("row_keys", true) if len(req.RowIDs) != 0 { return errors.New("row ids cannot be used because field uses string keys") } @@ -1026,6 +1030,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp // Translate column keys. if index.Keys() { + span.LogKV("column_keys", true) if len(req.ColumnIDs) != 0 { return errors.New("column ids cannot be used because index uses string keys") } @@ -1124,13 +1129,16 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . if err != nil { return errors.Wrap(err, "getting index and field") } - + span.LogKV( + "index", req.Index, + "field", req.Field) // Unless explicitly ignoring key validation (meaning keys have been // translate to ids in a previous step at the coordinator node), then // check to see if keys need translation. if !options.IgnoreKeyCheck { // Translate column keys. if index.Keys() { + span.LogKV("column_keys", true) if len(req.ColumnIDs) != 0 { return errors.New("column ids cannot be used because index uses string keys") } @@ -1144,6 +1152,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . // the field has a ForeignIndex with keys). if field.Keys() { // Perform translation. + span.LogKV("row_keys", true) uints, err := api.cluster.translateIndexKeys(ctx, field.ForeignIndex(), req.StringValues) if err != nil { return err diff --git a/executor.go b/executor.go index 3aa677c44..a26dd2fb2 100644 --- a/executor.go +++ b/executor.go @@ -155,6 +155,7 @@ func (e *executor) registerOps(ops []ext.BitmapOp) error { // Execute executes a PQL query. func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute") + span.LogKV("pql", q.String()) defer span.Finish() resp := QueryResponse{} @@ -1124,6 +1125,7 @@ func (e *executor) executePrecomputedCall(ctx context.Context, index string, c * // executeBitmapCall executes a call that returns a bitmap. func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCall") + span.LogKV("pql_call_name", c.Name) defer span.Finish() indexTag := "index:" + index @@ -1206,6 +1208,7 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * } span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard") + span.LogKV("pql_call_name", c.Name) defer span.Finish() if _, ok := e.additionalCountOps[c.Name]; ok { From 022019c6cc6828756ffbbd10673888bc2cdb21a2 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 26 May 2020 23:23:57 -0500 Subject: [PATCH 07/45] removed shard level tracing tag --- executor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/executor.go b/executor.go index a26dd2fb2..b8147a64c 100644 --- a/executor.go +++ b/executor.go @@ -1208,7 +1208,6 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * } span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard") - span.LogKV("pql_call_name", c.Name) defer span.Finish() if _, ok := e.additionalCountOps[c.Name]; ok { From 604f3b3373adca80453d22e08fa8a854abf5d3ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Wed, 27 May 2020 16:41:38 +0200 Subject: [PATCH 08/45] Add grpc uri to status --- cluster.go | 9 +- encoding/proto/proto.go | 2 + internal/private.pb.go | 257 +++++++++++++++++++++++++--------------- internal/private.proto | 1 + 4 files changed, 169 insertions(+), 100 deletions(-) diff --git a/cluster.go b/cluster.go index 6fce679fb..e979710d8 100644 --- a/cluster.go +++ b/cluster.go @@ -659,6 +659,7 @@ func (c *cluster) addNodeBasicSorted(node *Node) bool { n.State = node.State n.IsCoordinator = node.IsCoordinator n.URI = node.URI + n.GRPCURI = node.GRPCURI return true } return false @@ -1745,8 +1746,9 @@ func (j *resizeJob) distributeResizeInstructions() error { // Because the node may not be in the cluster yet, create // a dummy node object to use in the SendTo() method. node := &Node{ - ID: instr.Node.ID, - URI: instr.Node.URI, + ID: instr.Node.ID, + URI: instr.Node.URI, + GRPCURI: instr.Node.GRPCURI, } j.Logger.Printf("send resize instructions: %v", instr) if err := j.Broadcaster.SendTo(node, instr); err != nil { @@ -2046,6 +2048,9 @@ func (c *cluster) nodeJoin(node *Node) error { c.logger.Printf("node: %v changed URI from %s to %s", cnode.ID, cnode.URI, node.URI) cnode.URI = node.URI } + if cnode.GRPCURI != node.GRPCURI { + cnode.GRPCURI = node.GRPCURI + } return c.unprotectedSetStateAndBroadcast(c.determineClusterState()) } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index c41d38c53..a4508f652 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -655,6 +655,7 @@ func (s Serializer) encodeNode(n *pilosa.Node) *internal.Node { URI: s.encodeURI(n.URI), IsCoordinator: n.IsCoordinator, State: n.State, + GRPCURI: s.encodeURI(n.GRPCURI), } } @@ -996,6 +997,7 @@ func (s Serializer) decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.Cl func (s Serializer) decodeNode(node *internal.Node, m *pilosa.Node) { m.ID = node.ID s.decodeURI(node.URI, &m.URI) + s.decodeURI(node.GRPCURI, &m.GRPCURI) m.IsCoordinator = node.IsCoordinator m.State = node.State } diff --git a/internal/private.pb.go b/internal/private.pb.go index a1d0870bd..38eb7af3d 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1090,6 +1090,7 @@ type Node struct { URI *URI `protobuf:"bytes,2,opt,name=URI,proto3" json:"URI,omitempty"` IsCoordinator bool `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"` State string `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"` + GRPCURI *URI `protobuf:"bytes,5,opt,name=GRPCURI,proto3" json:"GRPCURI,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1156,6 +1157,13 @@ func (m *Node) GetState() string { return "" } +func (m *Node) GetGRPCURI() *URI { + if m != nil { + return m.GRPCURI + } + return nil +} + type NodeStateMessage struct { NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` @@ -2413,95 +2421,96 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1395 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x17, 0xcd, 0x72, 0xdb, 0x44, - 0x18, 0x59, 0x8e, 0x63, 0x7f, 0x8e, 0x53, 0x67, 0xdb, 0xa6, 0x6a, 0x60, 0x82, 0x59, 0x3a, 0xd4, - 0x74, 0x86, 0xd0, 0x69, 0x99, 0xe1, 0xb7, 0x33, 0x6d, 0xe2, 0xb4, 0x98, 0x92, 0xb4, 0x5d, 0xa7, - 0xbd, 0x71, 0xd8, 0xc8, 0x3b, 0x8d, 0x26, 0xb2, 0x64, 0xa4, 0x55, 0xea, 0xf4, 0xc0, 0x15, 0x66, - 0x78, 0x01, 0x8e, 0xbc, 0x07, 0x2f, 0xc0, 0x91, 0x47, 0x60, 0xca, 0x53, 0x70, 0x63, 0xf6, 0xdb, - 0x5d, 0x49, 0x76, 0x1c, 0x52, 0x52, 0x6e, 0xfb, 0xfd, 0xff, 0x7f, 0x9f, 0x04, 0xad, 0x71, 0x12, - 0x1c, 0x71, 0x29, 0x36, 0xc6, 0x49, 0x2c, 0x63, 0x52, 0x0f, 0x22, 0x29, 0x92, 0x88, 0x87, 0x6b, - 0x4b, 0xe3, 0x6c, 0x3f, 0x0c, 0x7c, 0x8d, 0xa7, 0x0f, 0xa0, 0xd1, 0x8f, 0x86, 0x62, 0xb2, 0x23, - 0x24, 0x27, 0x04, 0xaa, 0x0f, 0xc5, 0x71, 0xea, 0xb9, 0x1d, 0xa7, 0x5b, 0x67, 0xf8, 0x26, 0x1f, - 0xc0, 0xf2, 0x5e, 0xc2, 0xfd, 0xc3, 0xed, 0x49, 0x90, 0x4a, 0x11, 0xf9, 0xc2, 0xab, 0x22, 0x75, - 0x06, 0x4b, 0x7f, 0x75, 0x61, 0xe9, 0x7e, 0x20, 0xc2, 0xe1, 0xa3, 0xb1, 0x0c, 0xe2, 0x28, 0x55, - 0xca, 0xf6, 0x8e, 0xc7, 0xc2, 0xab, 0x77, 0x9c, 0x6e, 0x83, 0xe1, 0x9b, 0xbc, 0x03, 0x8d, 0x2d, - 0xee, 0x1f, 0x08, 0x24, 0xb8, 0x48, 0x28, 0x10, 0x39, 0x75, 0x10, 0xbc, 0xd4, 0x56, 0x5a, 0xac, - 0x40, 0x90, 0x0e, 0x34, 0xf7, 0x82, 0x91, 0x78, 0x92, 0xf1, 0x48, 0x66, 0x23, 0x6f, 0x01, 0xa5, - 0xcb, 0x28, 0xb2, 0x0a, 0xb5, 0x47, 0xe1, 0x70, 0x27, 0x88, 0xbc, 0x46, 0xc7, 0xe9, 0xba, 0xcc, - 0x40, 0x16, 0xcf, 0x27, 0x1e, 0x14, 0x78, 0x3e, 0xc9, 0xc3, 0x6d, 0x4e, 0x87, 0xbb, 0x1b, 0x0f, - 0x24, 0x8f, 0x86, 0x3c, 0x19, 0x3e, 0x0b, 0xc4, 0x0b, 0x6f, 0x49, 0x87, 0x3b, 0x8d, 0x55, 0xb2, - 0x9b, 0x3c, 0x15, 0x5e, 0x0b, 0x35, 0xe2, 0x9b, 0xac, 0x41, 0x7d, 0x33, 0x90, 0x3d, 0x31, 0x96, - 0x07, 0xde, 0x72, 0xc7, 0xe9, 0x56, 0x59, 0x0e, 0x93, 0x4b, 0xb0, 0x30, 0xf0, 0x79, 0x28, 0xbc, - 0x0b, 0x28, 0xa0, 0x01, 0x42, 0x61, 0xe9, 0x7e, 0x9c, 0x88, 0xe0, 0x79, 0x84, 0x45, 0xf0, 0xda, - 0x18, 0xd4, 0x14, 0x8e, 0xbc, 0x0f, 0xae, 0x0a, 0x69, 0xa5, 0xe3, 0x74, 0x9b, 0xb7, 0x56, 0x36, - 0x6c, 0x1d, 0x37, 0x7a, 0xc2, 0x0f, 0x46, 0x3c, 0x64, 0x8a, 0x8a, 0x4c, 0x7c, 0xe2, 0x91, 0xd3, - 0x99, 0xf8, 0x84, 0x52, 0x58, 0xee, 0x8f, 0xc6, 0x71, 0x22, 0x99, 0x48, 0xc7, 0x71, 0x94, 0x0a, - 0xd2, 0x06, 0x77, 0x3b, 0x49, 0x3c, 0x07, 0xcd, 0xaa, 0x27, 0xfd, 0x01, 0xda, 0x9b, 0x61, 0xec, - 0x1f, 0xf6, 0xb8, 0xe4, 0x4c, 0x7c, 0x9f, 0x89, 0x54, 0x2a, 0xdf, 0xb5, 0x7b, 0x9a, 0x4f, 0x03, - 0x0a, 0x8b, 0xf5, 0xf6, 0x2a, 0x1a, 0x8b, 0x80, 0xca, 0x0b, 0x66, 0x4d, 0x97, 0x07, 0xdf, 0x18, - 0xfb, 0x01, 0x4f, 0x86, 0x58, 0xd3, 0x2a, 0xd3, 0x80, 0xc2, 0xa2, 0x25, 0xec, 0x83, 0x2a, 0xd3, - 0x00, 0xed, 0xc3, 0x4a, 0xc9, 0xbe, 0x71, 0x73, 0x15, 0x6a, 0x2c, 0x7e, 0xd1, 0xef, 0xa5, 0x9e, - 0xd3, 0x71, 0xbb, 0x55, 0x66, 0x20, 0x6c, 0x98, 0x38, 0xcc, 0x46, 0x91, 0x22, 0x55, 0x90, 0x54, - 0x20, 0xe8, 0x55, 0x58, 0xc0, 0xee, 0x51, 0x51, 0x16, 0xb2, 0xea, 0x49, 0x7f, 0x74, 0xa0, 0xb1, - 0xc3, 0x27, 0xe8, 0x48, 0x4a, 0xee, 0x40, 0xdd, 0xd6, 0x16, 0x99, 0x9a, 0xb7, 0xde, 0x2b, 0x32, - 0x98, 0xb3, 0x6d, 0x58, 0x9e, 0xed, 0x48, 0x26, 0xc7, 0x2c, 0x17, 0x59, 0xfb, 0x12, 0x5a, 0x53, - 0x24, 0x65, 0xef, 0x50, 0x1c, 0xdb, 0xac, 0x1e, 0x8a, 0x63, 0x15, 0xeb, 0x11, 0x0f, 0x33, 0x81, - 0xb9, 0xaa, 0x32, 0x0d, 0x7c, 0x51, 0xf9, 0xcc, 0xa1, 0xcf, 0x80, 0x6c, 0x25, 0x82, 0x4b, 0x81, - 0x46, 0x76, 0x44, 0x9a, 0xf2, 0xe7, 0xe2, 0xac, 0x8c, 0xbb, 0xe5, 0x8c, 0xe7, 0xd9, 0xad, 0x94, - 0xb2, 0x4b, 0x6f, 0x00, 0xe9, 0x89, 0x50, 0x48, 0x61, 0xa6, 0xfb, 0x5f, 0xf4, 0xd2, 0x81, 0xf5, - 0xe1, 0x6c, 0x5e, 0x72, 0x1d, 0xaa, 0x6a, 0x55, 0xa0, 0xb1, 0xe6, 0xad, 0x8b, 0x45, 0x9e, 0xf2, - 0x2d, 0xc2, 0x90, 0x81, 0x86, 0x56, 0x29, 0x7a, 0xf9, 0x9a, 0x81, 0x4d, 0xb5, 0xd2, 0x0d, 0x63, - 0xca, 0x45, 0x53, 0xab, 0x85, 0xa9, 0xf2, 0x9a, 0x31, 0xd6, 0xee, 0xda, 0x70, 0xcf, 0x6b, 0x8d, - 0xfa, 0xf0, 0xb6, 0xd6, 0x70, 0xef, 0x88, 0x07, 0x21, 0xdf, 0x0f, 0xff, 0x53, 0x45, 0xa6, 0x1c, - 0xf7, 0x60, 0x11, 0x65, 0xfb, 0x3d, 0xd3, 0xdb, 0x16, 0xa4, 0xdf, 0x41, 0x31, 0x26, 0xbb, 0x7c, - 0x24, 0x8c, 0x36, 0x7c, 0xe7, 0xf1, 0x56, 0xce, 0x8e, 0x57, 0x19, 0x56, 0xa3, 0xa5, 0x56, 0xb5, - 0xab, 0x0c, 0x23, 0x40, 0x6f, 0x43, 0x6d, 0xe0, 0x1f, 0x88, 0x11, 0x27, 0x1f, 0xc2, 0x22, 0x7a, - 0x28, 0x52, 0xd3, 0xd1, 0x17, 0x66, 0x2a, 0xc5, 0x2c, 0x9d, 0xa6, 0x26, 0xb2, 0xb9, 0x3e, 0x7d, - 0x04, 0x8b, 0xc6, 0x30, 0x4e, 0xf4, 0x29, 0x15, 0xb7, 0x3c, 0xe4, 0x3a, 0xd4, 0xd0, 0xd9, 0xd4, - 0xab, 0xce, 0x5a, 0x45, 0x3c, 0x33, 0x64, 0xba, 0x0d, 0xee, 0x53, 0xd6, 0x57, 0x83, 0x8d, 0x0e, - 0x5b, 0xa3, 0x06, 0x52, 0xae, 0x7c, 0x1d, 0xa7, 0xd2, 0xa4, 0x15, 0xdf, 0x0a, 0xf7, 0x38, 0x4e, - 0x24, 0xa6, 0xb4, 0xc5, 0xf0, 0x4d, 0x53, 0xa8, 0xee, 0xc6, 0x43, 0x41, 0x96, 0xa1, 0xd2, 0xef, - 0x19, 0x1d, 0x95, 0x7e, 0x8f, 0xbc, 0x8b, 0xea, 0x4d, 0x26, 0x5b, 0x85, 0x13, 0x4f, 0x59, 0x9f, - 0xa1, 0xe1, 0x6b, 0xd0, 0xea, 0xa7, 0x5b, 0x71, 0x9c, 0x0c, 0x83, 0x88, 0xcb, 0x38, 0x31, 0x27, - 0x6f, 0x1a, 0x89, 0xa3, 0x25, 0xb9, 0xd4, 0xc7, 0xa8, 0xc1, 0x34, 0x40, 0xef, 0x42, 0x5b, 0x19, - 0x45, 0xc0, 0xb6, 0xc7, 0x2a, 0xd4, 0x14, 0x2e, 0x77, 0xc2, 0x40, 0x85, 0x86, 0x4a, 0x59, 0xc3, - 0xb7, 0x5a, 0xc3, 0xf6, 0x91, 0x88, 0x64, 0xa9, 0xc1, 0x10, 0x46, 0x05, 0x2d, 0xa6, 0x01, 0x42, - 0x75, 0x80, 0x26, 0x92, 0xe5, 0x22, 0x12, 0x85, 0x65, 0x48, 0xa3, 0x3f, 0x3b, 0x00, 0xd6, 0xa1, - 0x2c, 0xcd, 0x45, 0x9c, 0xd3, 0x45, 0x48, 0xd7, 0x36, 0x8a, 0x19, 0xae, 0x76, 0xc1, 0xa5, 0xf1, - 0xcc, 0x36, 0xd2, 0xc7, 0x45, 0x23, 0xe9, 0x92, 0x5e, 0x9e, 0x69, 0x00, 0x6d, 0xb5, 0x68, 0xa7, - 0xc7, 0xd0, 0x2c, 0xe1, 0x4f, 0x69, 0x2a, 0xdb, 0x25, 0x95, 0x59, 0x95, 0x88, 0x37, 0x2a, 0x6d, - 0xaf, 0x3c, 0x84, 0x66, 0x09, 0x3d, 0x57, 0x63, 0x17, 0x2e, 0x4c, 0x8f, 0xad, 0x3d, 0x07, 0xb3, - 0x68, 0x1a, 0x40, 0x6b, 0x2b, 0xcc, 0x52, 0x29, 0x12, 0xa3, 0x4e, 0xdd, 0x10, 0x8d, 0xc8, 0x8b, - 0x57, 0x20, 0xe6, 0xd7, 0x8f, 0x5c, 0x83, 0x05, 0x95, 0x46, 0x3d, 0x7d, 0x27, 0x73, 0xac, 0x89, - 0xf4, 0x19, 0xd4, 0x37, 0x07, 0xfd, 0x07, 0x49, 0x9c, 0x8d, 0xe7, 0x3a, 0x6d, 0x3f, 0x90, 0x2a, - 0xa5, 0x0f, 0xa4, 0xb6, 0x3e, 0xf6, 0x2e, 0x7e, 0x24, 0xe0, 0x65, 0x6f, 0xeb, 0xcb, 0x5e, 0x35, - 0x18, 0xae, 0xd6, 0xf5, 0x8a, 0xde, 0xac, 0x6a, 0xe8, 0xcf, 0xb3, 0x9f, 0xec, 0x8d, 0x76, 0x8b, - 0x1b, 0xad, 0x94, 0xea, 0xf5, 0xf7, 0x7f, 0x2a, 0xfd, 0xbb, 0x02, 0x2b, 0x4c, 0xa4, 0xc1, 0x4b, - 0xd1, 0x8f, 0x52, 0x99, 0x64, 0xbe, 0xda, 0x12, 0x4a, 0xfe, 0x9b, 0x78, 0xdf, 0x64, 0xdb, 0x65, - 0x1a, 0x78, 0x9d, 0x4e, 0x27, 0x37, 0xa1, 0x39, 0x3b, 0xb3, 0x27, 0x59, 0xcb, 0x2c, 0xe4, 0x26, - 0x2c, 0x0e, 0xe2, 0x2c, 0xf1, 0xf3, 0xf6, 0x2d, 0xad, 0x55, 0xed, 0x99, 0x26, 0x33, 0xcb, 0x46, - 0x9e, 0x00, 0xd9, 0x4b, 0x78, 0x94, 0x86, 0x5c, 0x39, 0x6b, 0x85, 0xeb, 0xb3, 0x9f, 0x05, 0x25, - 0x9e, 0x29, 0x3d, 0x73, 0x84, 0xc9, 0x27, 0xe5, 0xf9, 0xf4, 0x16, 0xd1, 0xeb, 0x4b, 0xd3, 0x5e, - 0x9b, 0x96, 0x2f, 0xcf, 0xf1, 0x9d, 0x99, 0x4e, 0xf5, 0x6a, 0x28, 0x78, 0xa5, 0x10, 0x9c, 0x22, - 0xb3, 0x69, 0x6e, 0xfa, 0x93, 0x03, 0x4b, 0x65, 0xcf, 0x5e, 0x6b, 0x2f, 0xe4, 0x05, 0xaf, 0x9c, - 0xfd, 0xdd, 0x61, 0x0b, 0x5e, 0x9d, 0xf7, 0xa5, 0xb7, 0x50, 0xfe, 0x16, 0xc9, 0xe0, 0xca, 0x29, - 0xe9, 0x7a, 0x03, 0xa7, 0x3a, 0xd0, 0x7c, 0xcc, 0x13, 0x19, 0x28, 0x95, 0xe6, 0xd0, 0x2e, 0xb0, - 0x32, 0x8a, 0x1e, 0xc2, 0xd5, 0x13, 0xcd, 0xb7, 0x15, 0x8f, 0xc6, 0xaa, 0xcb, 0xdf, 0xa0, 0x09, - 0xd5, 0xa2, 0x4e, 0x12, 0xd3, 0x7e, 0x0d, 0xa6, 0x01, 0xfa, 0x39, 0x5c, 0x1e, 0x08, 0x59, 0x6a, - 0x3d, 0x3b, 0x43, 0x1d, 0x70, 0x77, 0xc5, 0x8b, 0x53, 0x02, 0x54, 0x24, 0xfa, 0x15, 0x78, 0x4f, - 0xc7, 0x43, 0x2e, 0xc5, 0xb9, 0xa4, 0x37, 0xa1, 0xbe, 0x17, 0x8f, 0xe3, 0x30, 0x7e, 0x7e, 0x7c, - 0xc6, 0x2e, 0xf3, 0x60, 0x51, 0x5f, 0x25, 0xbd, 0x1c, 0x1b, 0xcc, 0x82, 0xf4, 0xa2, 0x1a, 0x53, - 0x9f, 0x87, 0x7e, 0x16, 0x2a, 0x37, 0xd4, 0x47, 0x73, 0x4a, 0x85, 0x19, 0x04, 0x8e, 0x89, 0x2b, - 0x1d, 0xba, 0x7b, 0x88, 0xb0, 0x87, 0x4e, 0x43, 0xe4, 0x53, 0x68, 0x96, 0xb8, 0x4d, 0x02, 0x2f, - 0xcf, 0xcc, 0x8b, 0x26, 0xb2, 0x32, 0x27, 0xfd, 0xcd, 0x99, 0x92, 0x3c, 0x71, 0xca, 0x8d, 0xc1, - 0x23, 0x5d, 0x94, 0x3a, 0x33, 0x90, 0x8a, 0x75, 0x7b, 0xe2, 0x87, 0x59, 0xaa, 0x48, 0xfa, 0x7a, - 0x17, 0x08, 0x15, 0xab, 0xfa, 0x33, 0x8c, 0x33, 0x69, 0x36, 0xa7, 0x05, 0xd5, 0x4f, 0x5a, 0x4f, - 0xf0, 0x61, 0x18, 0x44, 0x02, 0xbb, 0xd4, 0x65, 0x39, 0x4c, 0x6e, 0xea, 0x6d, 0x6f, 0x47, 0x6d, - 0x6d, 0xae, 0xfb, 0xc8, 0xa1, 0x2f, 0x41, 0x4a, 0x09, 0xb4, 0x67, 0x49, 0x9b, 0xed, 0xdf, 0x5f, - 0xad, 0x3b, 0x7f, 0xbc, 0x5a, 0x77, 0xfe, 0x7c, 0xb5, 0xee, 0xfc, 0xf2, 0xd7, 0xfa, 0x5b, 0xfb, - 0x35, 0xfc, 0xd7, 0xbe, 0xfd, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2f, 0x76, 0xf5, 0x59, 0x94, - 0x0f, 0x00, 0x00, + // 1418 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x72, 0x1b, 0xc5, + 0x13, 0xff, 0xaf, 0x56, 0xb6, 0xa4, 0x96, 0xe5, 0xc8, 0x93, 0xc4, 0xd9, 0xf8, 0x4f, 0x19, 0x31, + 0xa4, 0x88, 0x48, 0x15, 0x26, 0x95, 0x50, 0xc5, 0x67, 0xaa, 0x12, 0x5b, 0x4e, 0x10, 0xc1, 0x8e, + 0x33, 0x72, 0x72, 0xe3, 0x30, 0x5e, 0x4d, 0xc5, 0x5b, 0x5e, 0xed, 0x8a, 0xdd, 0x59, 0x47, 0xce, + 0x81, 0x2b, 0x54, 0xf1, 0x02, 0x1c, 0x38, 0xf0, 0x1e, 0xbc, 0x00, 0x47, 0x1e, 0x81, 0x0a, 0x4f, + 0xc1, 0x8d, 0x9a, 0x9e, 0x99, 0xdd, 0x95, 0x2c, 0xe3, 0x90, 0x70, 0xdb, 0xfe, 0xf5, 0x77, 0x4f, + 0x77, 0xcf, 0x2c, 0xb4, 0xc6, 0x49, 0x70, 0xcc, 0xa5, 0xd8, 0x18, 0x27, 0xb1, 0x8c, 0x49, 0x3d, + 0x88, 0xa4, 0x48, 0x22, 0x1e, 0xae, 0x2d, 0x8d, 0xb3, 0x83, 0x30, 0xf0, 0x35, 0x4e, 0x1f, 0x40, + 0xa3, 0x1f, 0x0d, 0xc5, 0x64, 0x47, 0x48, 0x4e, 0x08, 0x54, 0x1f, 0x8a, 0x93, 0xd4, 0x73, 0x3b, + 0x4e, 0xb7, 0xce, 0xf0, 0x9b, 0xbc, 0x07, 0xcb, 0xfb, 0x09, 0xf7, 0x8f, 0xb6, 0x27, 0x41, 0x2a, + 0x45, 0xe4, 0x0b, 0xaf, 0x8a, 0xdc, 0x19, 0x94, 0xfe, 0xe2, 0xc2, 0xd2, 0xfd, 0x40, 0x84, 0xc3, + 0x47, 0x63, 0x19, 0xc4, 0x51, 0xaa, 0x8c, 0xed, 0x9f, 0x8c, 0x85, 0x57, 0xef, 0x38, 0xdd, 0x06, + 0xc3, 0x6f, 0xf2, 0x16, 0x34, 0xb6, 0xb8, 0x7f, 0x28, 0x90, 0xe1, 0x22, 0xa3, 0x00, 0x72, 0xee, + 0x20, 0x78, 0xa1, 0xbd, 0xb4, 0x58, 0x01, 0x90, 0x0e, 0x34, 0xf7, 0x83, 0x91, 0x78, 0x9c, 0xf1, + 0x48, 0x66, 0x23, 0x6f, 0x01, 0xb5, 0xcb, 0x10, 0x59, 0x85, 0xc5, 0x47, 0xe1, 0x70, 0x27, 0x88, + 0xbc, 0x46, 0xc7, 0xe9, 0xba, 0xcc, 0x50, 0x16, 0xe7, 0x13, 0x0f, 0x0a, 0x9c, 0x4f, 0xf2, 0x74, + 0x9b, 0xd3, 0xe9, 0xee, 0xc6, 0x03, 0xc9, 0xa3, 0x21, 0x4f, 0x86, 0x4f, 0x03, 0xf1, 0xdc, 0x5b, + 0xd2, 0xe9, 0x4e, 0xa3, 0x4a, 0x77, 0x93, 0xa7, 0xc2, 0x6b, 0xa1, 0x45, 0xfc, 0x26, 0x6b, 0x50, + 0xdf, 0x0c, 0x64, 0x4f, 0x8c, 0xe5, 0xa1, 0xb7, 0xdc, 0x71, 0xba, 0x55, 0x96, 0xd3, 0xe4, 0x12, + 0x2c, 0x0c, 0x7c, 0x1e, 0x0a, 0xef, 0x02, 0x2a, 0x68, 0x82, 0x50, 0x58, 0xba, 0x1f, 0x27, 0x22, + 0x78, 0x16, 0xe1, 0x21, 0x78, 0x6d, 0x4c, 0x6a, 0x0a, 0x23, 0xef, 0x82, 0xab, 0x52, 0x5a, 0xe9, + 0x38, 0xdd, 0xe6, 0xad, 0x95, 0x0d, 0x7b, 0x8e, 0x1b, 0x3d, 0xe1, 0x07, 0x23, 0x1e, 0x32, 0xc5, + 0x45, 0x21, 0x3e, 0xf1, 0xc8, 0xd9, 0x42, 0x7c, 0x42, 0x29, 0x2c, 0xf7, 0x47, 0xe3, 0x38, 0x91, + 0x4c, 0xa4, 0xe3, 0x38, 0x4a, 0x05, 0x69, 0x83, 0xbb, 0x9d, 0x24, 0x9e, 0x83, 0x6e, 0xd5, 0x27, + 0xfd, 0x0e, 0xda, 0x9b, 0x61, 0xec, 0x1f, 0xf5, 0xb8, 0xe4, 0x4c, 0x7c, 0x9b, 0x89, 0x54, 0xaa, + 0xd8, 0x75, 0x78, 0x5a, 0x4e, 0x13, 0x0a, 0xc5, 0xf3, 0xf6, 0x2a, 0x1a, 0x45, 0x42, 0xd5, 0x05, + 0xab, 0xa6, 0x8f, 0x07, 0xbf, 0x31, 0xf7, 0x43, 0x9e, 0x0c, 0xf1, 0x4c, 0xab, 0x4c, 0x13, 0x0a, + 0x45, 0x4f, 0xd8, 0x07, 0x55, 0xa6, 0x09, 0xda, 0x87, 0x95, 0x92, 0x7f, 0x13, 0xe6, 0x2a, 0x2c, + 0xb2, 0xf8, 0x79, 0xbf, 0x97, 0x7a, 0x4e, 0xc7, 0xed, 0x56, 0x99, 0xa1, 0xb0, 0x61, 0xe2, 0x30, + 0x1b, 0x45, 0x8a, 0x55, 0x41, 0x56, 0x01, 0xd0, 0xab, 0xb0, 0x80, 0xdd, 0xa3, 0xb2, 0x2c, 0x74, + 0xd5, 0x27, 0xfd, 0xde, 0x81, 0xc6, 0x0e, 0x9f, 0x60, 0x20, 0x29, 0xb9, 0x03, 0x75, 0x7b, 0xb6, + 0x28, 0xd4, 0xbc, 0xf5, 0x4e, 0x51, 0xc1, 0x5c, 0x6c, 0xc3, 0xca, 0x6c, 0x47, 0x32, 0x39, 0x61, + 0xb9, 0xca, 0xda, 0xe7, 0xd0, 0x9a, 0x62, 0x29, 0x7f, 0x47, 0xe2, 0xc4, 0x56, 0xf5, 0x48, 0x9c, + 0xa8, 0x5c, 0x8f, 0x79, 0x98, 0x09, 0xac, 0x55, 0x95, 0x69, 0xe2, 0xb3, 0xca, 0x27, 0x0e, 0x7d, + 0x0a, 0x64, 0x2b, 0x11, 0x5c, 0x0a, 0x74, 0xb2, 0x23, 0xd2, 0x94, 0x3f, 0x13, 0xe7, 0x55, 0xdc, + 0x2d, 0x57, 0x3c, 0xaf, 0x6e, 0xa5, 0x54, 0x5d, 0x7a, 0x03, 0x48, 0x4f, 0x84, 0x42, 0x0a, 0x33, + 0xdd, 0xff, 0x60, 0x97, 0x0e, 0x6c, 0x0c, 0xe7, 0xcb, 0x92, 0xeb, 0x50, 0x55, 0xab, 0x02, 0x9d, + 0x35, 0x6f, 0x5d, 0x2c, 0xea, 0x94, 0x6f, 0x11, 0x86, 0x02, 0x34, 0xb4, 0x46, 0x31, 0xca, 0x57, + 0x4c, 0x6c, 0xaa, 0x95, 0x6e, 0x18, 0x57, 0x2e, 0xba, 0x5a, 0x2d, 0x5c, 0x95, 0xd7, 0x8c, 0xf1, + 0x76, 0xd7, 0xa6, 0xfb, 0xba, 0xde, 0xa8, 0x0f, 0xff, 0xd7, 0x16, 0xee, 0x1d, 0xf3, 0x20, 0xe4, + 0x07, 0xe1, 0xbf, 0x3a, 0x91, 0xa9, 0xc0, 0x3d, 0xa8, 0xa1, 0x6e, 0xbf, 0x67, 0x7a, 0xdb, 0x92, + 0xf4, 0x1b, 0x28, 0xc6, 0x64, 0x97, 0x8f, 0x84, 0xb1, 0x86, 0xdf, 0x79, 0xbe, 0x95, 0xf3, 0xf3, + 0x55, 0x8e, 0xd5, 0x68, 0xa9, 0x55, 0xed, 0x2a, 0xc7, 0x48, 0xd0, 0xdb, 0xb0, 0x38, 0xf0, 0x0f, + 0xc5, 0x88, 0x93, 0xf7, 0xa1, 0x86, 0x11, 0x8a, 0xd4, 0x74, 0xf4, 0x85, 0x99, 0x93, 0x62, 0x96, + 0x4f, 0x53, 0x93, 0xd9, 0xdc, 0x98, 0x3e, 0x80, 0x9a, 0x71, 0x8c, 0x13, 0x7d, 0xc6, 0x89, 0x5b, + 0x19, 0x72, 0x1d, 0x16, 0x31, 0xd8, 0xd4, 0xab, 0xce, 0x7a, 0x45, 0x9c, 0x19, 0x36, 0xdd, 0x06, + 0xf7, 0x09, 0xeb, 0xab, 0xc1, 0xc6, 0x80, 0xad, 0x53, 0x43, 0xa9, 0x50, 0xbe, 0x8c, 0x53, 0x69, + 0xca, 0x8a, 0xdf, 0x0a, 0xdb, 0x8b, 0x13, 0x89, 0x25, 0x6d, 0x31, 0xfc, 0xa6, 0x3f, 0x3b, 0x50, + 0xdd, 0x8d, 0x87, 0x82, 0x2c, 0x43, 0xa5, 0xdf, 0x33, 0x46, 0x2a, 0xfd, 0x1e, 0x79, 0x1b, 0xed, + 0x9b, 0x52, 0xb6, 0x8a, 0x28, 0x9e, 0xb0, 0x3e, 0x43, 0xcf, 0xd7, 0xa0, 0xd5, 0x4f, 0xb7, 0xe2, + 0x38, 0x19, 0x06, 0x11, 0x97, 0x71, 0x62, 0xee, 0xbc, 0x69, 0x10, 0x67, 0x4b, 0x72, 0xa9, 0x6f, + 0xa3, 0x06, 0xd3, 0x04, 0xb9, 0x0e, 0xb5, 0x07, 0x6c, 0x6f, 0x4b, 0x39, 0x58, 0x98, 0xe7, 0xc0, + 0x72, 0xe9, 0x5d, 0x68, 0xab, 0xe8, 0x50, 0xcb, 0x36, 0xd2, 0x2a, 0x2c, 0x2a, 0x2c, 0x8f, 0xd6, + 0x50, 0x85, 0xab, 0x4a, 0xc9, 0x15, 0xfd, 0x5a, 0x5b, 0xd8, 0x3e, 0x16, 0x91, 0x2c, 0xb5, 0x22, + 0xd2, 0x68, 0xa0, 0xc5, 0x34, 0x41, 0xa8, 0xae, 0x84, 0x49, 0x79, 0xb9, 0x88, 0x48, 0xa1, 0x0c, + 0x79, 0xf4, 0x47, 0x07, 0xc0, 0x06, 0x94, 0xa5, 0xb9, 0x8a, 0x73, 0xb6, 0x0a, 0xe9, 0xda, 0x96, + 0x32, 0x63, 0xd8, 0x2e, 0xa4, 0x34, 0xce, 0x6c, 0xcb, 0x7d, 0x58, 0xb4, 0x9c, 0x3e, 0xfc, 0xcb, + 0x33, 0xad, 0xa2, 0xbd, 0x16, 0x8d, 0xb7, 0x07, 0xcd, 0x12, 0x7e, 0x46, 0xfb, 0xd9, 0x7e, 0xaa, + 0xcc, 0x9a, 0x44, 0xdc, 0x98, 0xb4, 0x5d, 0xf5, 0x10, 0x9a, 0x25, 0x78, 0xae, 0xc5, 0x2e, 0x5c, + 0x98, 0x1e, 0x70, 0x7b, 0x71, 0xcc, 0xc2, 0x34, 0x80, 0xd6, 0x56, 0x98, 0xa5, 0x52, 0x24, 0xc6, + 0x9c, 0xba, 0x6d, 0x34, 0x90, 0x1f, 0x5e, 0x01, 0xcc, 0x3f, 0x3f, 0x72, 0x0d, 0x16, 0x54, 0x19, + 0xf5, 0x9c, 0x9e, 0xae, 0xb1, 0x66, 0xd2, 0xa7, 0x50, 0xdf, 0x1c, 0xf4, 0x1f, 0x24, 0x71, 0x36, + 0x9e, 0x1b, 0xb4, 0x7d, 0x4a, 0x55, 0x4a, 0x4f, 0xa9, 0xb6, 0x7e, 0x16, 0xb8, 0xf8, 0x9c, 0xc0, + 0x37, 0x40, 0x5b, 0xbf, 0x01, 0xaa, 0x06, 0xe1, 0x6a, 0xb1, 0xaf, 0xe8, 0x1d, 0xac, 0xd6, 0xc3, + 0xeb, 0x6c, 0x32, 0x7b, 0x9b, 0xbb, 0xc5, 0x6d, 0xae, 0x8c, 0xea, 0x45, 0xf9, 0x5f, 0x1a, 0xfd, + 0xab, 0x02, 0x2b, 0x4c, 0xa4, 0xc1, 0x0b, 0xd1, 0x8f, 0x52, 0x99, 0x64, 0xbe, 0xda, 0x27, 0x4a, + 0xff, 0xab, 0xf8, 0xc0, 0x54, 0xdb, 0x65, 0x9a, 0x78, 0x95, 0x4e, 0x27, 0x37, 0xa1, 0x39, 0x3b, + 0xdc, 0xa7, 0x45, 0xcb, 0x22, 0xe4, 0x26, 0xd4, 0x06, 0x71, 0x96, 0xf8, 0x79, 0xfb, 0x96, 0x16, + 0xb0, 0x8e, 0x4c, 0xb3, 0x99, 0x15, 0x23, 0x8f, 0x81, 0xec, 0x27, 0x3c, 0x4a, 0x43, 0xae, 0x82, + 0xb5, 0xca, 0xf5, 0xd9, 0x07, 0x44, 0x49, 0x66, 0xca, 0xce, 0x1c, 0x65, 0xf2, 0x51, 0x79, 0x3e, + 0xbd, 0x1a, 0x46, 0x7d, 0x69, 0x3a, 0x6a, 0xd3, 0xf2, 0xe5, 0x39, 0xbe, 0x33, 0xd3, 0xa9, 0xde, + 0x22, 0x2a, 0x5e, 0x29, 0x14, 0xa7, 0xd8, 0x6c, 0x5a, 0x9a, 0xfe, 0xe0, 0xc0, 0x52, 0x39, 0xb2, + 0x57, 0xda, 0x0b, 0xf9, 0x81, 0x57, 0xce, 0x7f, 0xa1, 0xd8, 0x03, 0xaf, 0xce, 0x7b, 0x13, 0x2e, + 0x94, 0x5f, 0x2d, 0x19, 0x5c, 0x39, 0xa3, 0x5c, 0x6f, 0x10, 0x54, 0x07, 0x9a, 0x7b, 0x3c, 0x91, + 0x81, 0x32, 0x69, 0xae, 0xe4, 0x05, 0x56, 0x86, 0xe8, 0x11, 0x5c, 0x3d, 0xd5, 0x7c, 0x5b, 0xf1, + 0x68, 0xac, 0xba, 0xfc, 0x0d, 0x9a, 0x50, 0x2d, 0xea, 0x24, 0x31, 0xed, 0xd7, 0x60, 0x9a, 0xa0, + 0x9f, 0xc2, 0xe5, 0x81, 0x90, 0xa5, 0xd6, 0xb3, 0x33, 0xd4, 0x01, 0x77, 0x57, 0x3c, 0x3f, 0x23, + 0x41, 0xc5, 0xa2, 0x5f, 0x80, 0xf7, 0x64, 0x3c, 0xe4, 0x52, 0xbc, 0x96, 0xf6, 0x26, 0xd4, 0xf7, + 0xe3, 0x71, 0x1c, 0xc6, 0xcf, 0x4e, 0xce, 0xd9, 0x65, 0x1e, 0xd4, 0xf4, 0xad, 0xa4, 0x97, 0x63, + 0x83, 0x59, 0x92, 0x5e, 0x54, 0x63, 0xea, 0xf3, 0xd0, 0xcf, 0x42, 0x15, 0x86, 0x7a, 0x5e, 0xa7, + 0x54, 0x98, 0x41, 0xe0, 0x58, 0xb8, 0xd2, 0x45, 0x77, 0x0f, 0x01, 0x7b, 0xd1, 0x69, 0x8a, 0x7c, + 0x0c, 0xcd, 0x92, 0xb4, 0x29, 0xe0, 0xe5, 0x99, 0x79, 0xd1, 0x4c, 0x56, 0x96, 0xa4, 0xbf, 0x3a, + 0x53, 0x9a, 0xa7, 0xee, 0x7c, 0xe3, 0xf0, 0x58, 0x1f, 0x4a, 0x9d, 0x19, 0x4a, 0xe5, 0xba, 0x3d, + 0xf1, 0xc3, 0x2c, 0x55, 0x2c, 0x7d, 0xcd, 0x17, 0x80, 0xca, 0x55, 0xfd, 0x43, 0xc6, 0x99, 0x34, + 0x9b, 0xd3, 0x92, 0xea, 0x77, 0xae, 0x27, 0xf8, 0x30, 0x0c, 0x22, 0x81, 0x5d, 0xea, 0xb2, 0x9c, + 0x26, 0x37, 0xf5, 0xb6, 0xb7, 0xa3, 0xb6, 0x36, 0x37, 0x7c, 0x94, 0xd0, 0x37, 0x41, 0x4a, 0x09, + 0xb4, 0x67, 0x59, 0x9b, 0xed, 0xdf, 0x5e, 0xae, 0x3b, 0xbf, 0xbf, 0x5c, 0x77, 0xfe, 0x78, 0xb9, + 0xee, 0xfc, 0xf4, 0xe7, 0xfa, 0xff, 0x0e, 0x16, 0xf1, 0xaf, 0xfc, 0xf6, 0xdf, 0x01, 0x00, 0x00, + 0xff, 0xff, 0x8a, 0x80, 0x2a, 0x36, 0xbe, 0x0f, 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -3422,6 +3431,18 @@ func (m *Node) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.GRPCURI != nil { + { + size, err := m.GRPCURI.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + } if len(m.State) > 0 { i -= len(m.State) copy(dAtA[i:], m.State) @@ -3684,20 +3705,20 @@ func (m *FieldStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { copy(dAtA[i:], m.XXX_unrecognized) } if len(m.AvailableShards) > 0 { - dAtA18 := make([]byte, len(m.AvailableShards)*10) - var j17 int + dAtA19 := make([]byte, len(m.AvailableShards)*10) + var j18 int for _, num := range m.AvailableShards { for num >= 1<<7 { - dAtA18[j17] = uint8(uint64(num)&0x7f | 0x80) + dAtA19[j18] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j17++ + j18++ } - dAtA18[j17] = uint8(num) - j17++ + dAtA19[j18] = uint8(num) + j18++ } - i -= j17 - copy(dAtA[i:], dAtA18[:j17]) - i = encodeVarintPrivate(dAtA, i, uint64(j17)) + i -= j18 + copy(dAtA[i:], dAtA19[:j18]) + i = encodeVarintPrivate(dAtA, i, uint64(j18)) i-- dAtA[i] = 0x12 } @@ -4925,6 +4946,10 @@ func (m *Node) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.GRPCURI != nil { + l = m.GRPCURI.Size() + n += 1 + l + sovPrivate(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -8114,6 +8139,42 @@ func (m *Node) Unmarshal(dAtA []byte) error { } m.State = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GRPCURI", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.GRPCURI == nil { + m.GRPCURI = &URI{} + } + if err := m.GRPCURI.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) diff --git a/internal/private.proto b/internal/private.proto index 804c3f2cd..849fff04a 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -110,6 +110,7 @@ message Node { URI URI = 2; bool IsCoordinator = 3; string State = 4; + URI GRPCURI = 5; } message NodeStateMessage { From 4274d2d1418a1cc2967be22802b9b17b55400bd6 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 27 May 2020 15:23:13 -0500 Subject: [PATCH 09/45] convert to camelCase --- api.go | 8 ++++---- executor.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api.go b/api.go index eb1587a13..90d3e2e96 100644 --- a/api.go +++ b/api.go @@ -1019,7 +1019,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp if !options.IgnoreKeyCheck { // Translate row keys. if field.Keys() { - span.LogKV("row_keys", true) + span.LogKV("rowKeys", true) if len(req.RowIDs) != 0 { return errors.New("row ids cannot be used because field uses string keys") } @@ -1030,7 +1030,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp // Translate column keys. if index.Keys() { - span.LogKV("column_keys", true) + span.LogKV("columnKeys", true) if len(req.ColumnIDs) != 0 { return errors.New("column ids cannot be used because index uses string keys") } @@ -1138,7 +1138,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . if !options.IgnoreKeyCheck { // Translate column keys. if index.Keys() { - span.LogKV("column_keys", true) + span.LogKV("columnKeys", true) if len(req.ColumnIDs) != 0 { return errors.New("column ids cannot be used because index uses string keys") } @@ -1152,7 +1152,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . // the field has a ForeignIndex with keys). if field.Keys() { // Perform translation. - span.LogKV("row_keys", true) + span.LogKV("rowKeys", true) uints, err := api.cluster.translateIndexKeys(ctx, field.ForeignIndex(), req.StringValues) if err != nil { return err diff --git a/executor.go b/executor.go index b8147a64c..23c0835f6 100644 --- a/executor.go +++ b/executor.go @@ -1125,7 +1125,7 @@ func (e *executor) executePrecomputedCall(ctx context.Context, index string, c * // executeBitmapCall executes a call that returns a bitmap. func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCall") - span.LogKV("pql_call_name", c.Name) + span.LogKV("pqlCallName", c.Name) defer span.Finish() indexTag := "index:" + index From 4d1ce90b32664529b7049da78ba5446bd0303a22 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 28 May 2020 15:36:00 -0500 Subject: [PATCH 10/45] correction to endpoint --- transaction.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transaction.md b/transaction.md index f8f7f5723..4a3bae6d4 100644 --- a/transaction.md +++ b/transaction.md @@ -30,7 +30,7 @@ The base transaction endpoints are `/transactions`, for listing or creating transactions, and `/transaction/[id]`, for listing, creating, finishing, or cancelling a transaction. -A POST to `/transactions` attempts to create a transaction, assigning it an +A POST to `/transaction` attempts to create a transaction, assigning it an arbitrary ID that is not the ID of any existing transaction. A `GET` from `/transactions` lists existing transactions. From 1ca9435af923898bb2d68242293b9c22dd459cf8 Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 22 Apr 2020 15:23:29 -0500 Subject: [PATCH 11/45] Handle file sizes over 4GB We only have 4 bytes for offsets, but what if a file is over 4GB? Someone came to us with a file with 265 *million* containers, in a single fragment, which means that over 3GB of their 4.7GB file is actually just the container headers alone. But we can't easily make the offsets larger, or change the file format. So we don't. We just track how many 4GB hunks of the file we've been through and bump that every time the 32-bit offset wraps. And this appears to... just work. This is fixed for both the roaring iterator and the old unmarshalBinary logic. The logic to handle this will work on 32-bit hosts in the sense that it will correctly error out for excessively large file sizes or container counts, but it doesn't actually handle the large files since it can't. --- roaring/roaring.go | 37 +++++++++++++++++++++++++------------ roaring/unmarshal_binary.go | 27 ++++++++++++++++++--------- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 5924673ea..22d2821db 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1738,7 +1738,9 @@ type baseRoaringIterator struct { currentN int currentLen int currentPointer *uint16 - currentDataOffset uint32 + currentDataOffset uint64 + prevOffset32 uint32 + chunkOffset uint64 lastDataOffset int64 lastErr error } @@ -1752,6 +1754,8 @@ func (b *baseRoaringIterator) SilenceLint() { _ = b.offsets _ = b.headers _ = b.currentIdx + _ = b.chunkOffset + _ = b.prevOffset32 } type pilosaRoaringIterator struct { @@ -1787,14 +1791,14 @@ func newOfficialRoaringIterator(data []byte) (*officialRoaringIterator, error) { r.headers = data[headerOffset:offsetOffset] // note: offsets are only actually used with the no-run headers. if r.haveRuns { - r.currentDataOffset = uint32(offsetOffset) + r.currentDataOffset = uint64(offsetOffset) } else { if len(r.data) < offsetOffset+int(r.keys*4) { return nil, fmt.Errorf("insufficient data for offsets (need %d bytes, found %d)", r.keys*4, len(r.data)-offsetOffset) } r.offsets = data[offsetOffset : offsetOffset+int(r.keys*4)] - r.currentDataOffset = uint32(offsetOffset) + r.currentDataOffset = uint64(offsetOffset) } // set key to -1; user should call Next first. r.currentIdx = -1 @@ -1838,7 +1842,11 @@ func newPilosaRoaringIterator(data []byte) (*pilosaRoaringIterator, error) { // if there's no containers, we want to act as though data started at the end // of the list of offsets, which was also empty, so we don't think the entire thing // is actually a malformed op - r.currentDataOffset = uint32(offsetEnd) + r.prevOffset32 = uint32(offsetEnd) + r.currentDataOffset = uint64(offsetEnd) + // it's possible that there's so many headers that we're actually over + // 4GB into the file already. + r.chunkOffset = r.currentDataOffset &^ ((1 << 32) - 1) // set key to -1; user should call Next first. r.currentIdx = -1 r.currentKey = ^uint64(0) @@ -1895,7 +1903,12 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in r.currentKey = binary.LittleEndian.Uint64(header[0:8]) r.currentType = byte(binary.LittleEndian.Uint16(header[8:10])) r.currentN = int(binary.LittleEndian.Uint16(header[10:12])) + 1 - r.currentDataOffset = binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:]) + offset32 := binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:]) + if offset32 < r.prevOffset32 { + r.chunkOffset += (1 << 32) + } + r.prevOffset32 = offset32 + r.currentDataOffset = r.chunkOffset + uint64(offset32) // a run container keeps its data after an initial 2 byte length header var runCount uint16 @@ -1903,7 +1916,7 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in runCount = binary.LittleEndian.Uint16(r.data[r.currentDataOffset : r.currentDataOffset+runCountHeaderSize]) r.currentDataOffset += 2 } - if r.currentDataOffset > uint32(len(r.data)) || r.currentDataOffset < headerBaseSize { + if r.currentDataOffset > uint64(len(r.data)) || r.currentDataOffset < headerBaseSize { r.Done(fmt.Errorf("container %d/%d, key %d, had offset %d, maximum %d", r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, len(r.data))) return r.Current() @@ -1926,7 +1939,7 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, size, len(r.data))) return r.Current() } - r.currentDataOffset += uint32(size) + r.currentDataOffset += uint64(size) r.lastErr = nil return r.Current() } @@ -1949,7 +1962,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length // with runs, we can't actually look up offsets; the format just stores // things sequentially. so we have to actually track the offset in that case. if !r.haveRuns { - r.currentDataOffset = binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:]) + r.currentDataOffset = uint64(binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:])) } // a run container keeps its data after an initial 2 byte length header var runCount uint16 @@ -1962,7 +1975,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length runCount = binary.LittleEndian.Uint16(r.data[r.currentDataOffset : r.currentDataOffset+runCountHeaderSize]) r.currentDataOffset += 2 } - if r.currentDataOffset > uint32(len(r.data)) || r.currentDataOffset < headerBaseSize { + if r.currentDataOffset > uint64(len(r.data)) || r.currentDataOffset < headerBaseSize { r.Done(fmt.Errorf("container %d/%d, key %d, had offset %d, maximum %d", r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, len(r.data))) return r.Current() @@ -1994,7 +2007,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, size, len(r.data))) return r.Current() } - r.currentDataOffset += uint32(size) + r.currentDataOffset += uint64(size) r.lastErr = nil return r.Current() } @@ -2012,14 +2025,14 @@ func (b *Bitmap) SanityCheckMapping(from, to uintptr) (mappedIn int64, mappedOut if c.Mapped() { mappedIn++ } else { - err = fmt.Errorf("container key %d, addr %x, inside %x+%d\n", + err = fmt.Errorf("container key %d, addr %x, inside %x+%d", key, dptr, from, to-from) errs++ unmappedIn++ } } else { if c.Mapped() { - err = fmt.Errorf("container key %d, addr %x, outside %x+%d, but mapped\n", + err = fmt.Errorf("container key %d, addr %x, outside %x+%d, but mapped", key, dptr, from, to-from) errs++ mappedOut++ diff --git a/roaring/unmarshal_binary.go b/roaring/unmarshal_binary.go index 208c15d53..c9071b1ca 100644 --- a/roaring/unmarshal_binary.go +++ b/roaring/unmarshal_binary.go @@ -159,8 +159,8 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { // Read key count in bytes sizeof(cookie)+sizeof(flag):(sizeof(cookie)+sizeof(uint32)). keyN := binary.LittleEndian.Uint32(data[3+1 : 8]) - if uint32(len(data)) < headerBaseSize+keyN*12 { - return fmt.Errorf("insufficient data for header + offsets: key-cardinality not provided for %d containers", int(keyN)/12) + if int64(len(data)) < headerBaseSize+int64(keyN)*12 { + return fmt.Errorf("insufficient data for header + offsets: key-cardinality not provided for %d containers", keyN) } headerSize := headerBaseSize @@ -173,14 +173,23 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { int(binary.LittleEndian.Uint16(buf[10:12]))+1, true) } - opsOffset := headerSize + int(keyN)*12 + opsOffset := int64(headerSize) + int64(keyN)*12 // Read container offsets and attach data. citer, _ := b.Containers.Iterator(0) + // if you have enough containers that the *headers alone* exceed 4GB, we + // need to start with a higher cycle offset. + cycleOffset := opsOffset &^ ((1 << 32) - 1) + prevOffset32 := uint32(opsOffset) for i, buf := 0, data[opsOffset:]; i < int(keyN); i, buf = i+1, buf[4:] { - offset := binary.LittleEndian.Uint32(buf[0:4]) + offset32 := binary.LittleEndian.Uint32(buf[0:4]) + if offset32 < prevOffset32 { + cycleOffset += (1 << 32) + } + prevOffset32 = offset32 + offset := int64(offset32) + cycleOffset // Verify the offset is within the bounds of the input data. - if int(offset) >= len(data) { + if offset >= int64(len(data)) { return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data)) } @@ -201,13 +210,13 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { case containerRun: runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize]) c.setRuns((*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount:runCount]) - opsOffset = int(offset) + runCountHeaderSize + len(c.runs())*interval16Size + opsOffset = offset + runCountHeaderSize + int64(len(c.runs()))*interval16Size case containerArray: c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.N():c.N()]) - opsOffset = int(offset) + len(c.array())*2 // sizeof(uint32) + opsOffset = offset + int64(len(c.array()))*2 // sizeof(uint32) case containerBitmap: c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN:bitmapN]) - opsOffset = int(offset) + len(c.bitmap())*8 // sizeof(uint64) + opsOffset = offset + int64(len(c.bitmap()))*8 // sizeof(uint64) } } @@ -228,7 +237,7 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { // Increase the op count. b.ops++ b.opN += opr.count() - opsOffset += opr.size() + opsOffset += int64(opr.size()) // Move the buffer forward. buf = data[opsOffset:] } From 1c1204fc77cfedff7f9173b7460f76cf9e3bcfa2 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 28 May 2020 23:53:28 -0500 Subject: [PATCH 12/45] modify PQL parser to handle escapes in string values This modifies the parser to properly "unquote" incoming strings. So if a string comes in double or single quoted, we approximately follow Go rules for removing the quotes and processing escape sequences. The differences from Go are: 1. we only support backslash, quote, tab and newline escape sequenences. 2. Single quoted strings are supported and work just like double quoted strings. 3. The peg parser won't actually accept backquoted strings (I don't think) Fixes: #411 --- pql/ast.go | 7 + pql/parser.go | 78 ++++ pql/parser_test.go | 67 ++++ pql/pql.peg | 16 +- pql/pql.peg.go | 811 +++++++++++++++++++++++------------------- server/server_test.go | 44 +++ 6 files changed, 642 insertions(+), 381 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index e4163a66e..e93a2aaf5 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -134,6 +134,13 @@ func (q *Query) validateArgField(elem *callStackElem) { } func (q *Query) addVal(val interface{}) { + if vs, ok := val.(string); ok { + vsu, err := Unquote(vs) + if err != nil { + panic(err) + } + val = vsu + } elem := q.lastCallStackElem() if elem == nil || elem.lastField == "" { panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val)) diff --git a/pql/parser.go b/pql/parser.go index b3ad984f9..8a5907dbf 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -18,7 +18,9 @@ import ( "fmt" "io" "io/ioutil" + "strconv" "strings" + "unicode/utf8" "github.com/pkg/errors" ) @@ -95,3 +97,79 @@ func (p *parser) Parse() (*Query, error) { return &p.Query, nil } + +// Unquote interprets s as a single-quoted, double-quoted, or +// backquoted Go string literal, returning the string value that s +// quotes. It is a copy of stdlib's strconv.Unquote, but modified so +// that if s is single-quoted, it can still be a string rather than +// only character literal. This version of Unquote also accepts +// unquoted strings and passes them back unchanged. +func Unquote(s string) (string, error) { + n := len(s) + if n < 2 { + return s, nil + } + quote := s[0] + if quote != '"' && quote != '\'' && quote != '`' { + return s, nil + } + if quote != s[n-1] { + return "", strconv.ErrSyntax + } + s = s[1 : n-1] + + if quote == '`' { + if contains(s, '`') { + return "", strconv.ErrSyntax + } + if contains(s, '\r') { + // -1 because we know there is at least one \r to remove. + buf := make([]byte, 0, len(s)-1) + for i := 0; i < len(s); i++ { + if s[i] != '\r' { + buf = append(buf, s[i]) + } + } + return string(buf), nil + } + return s, nil + } + if quote != '"' && quote != '\'' { + return "", strconv.ErrSyntax + } + if contains(s, '\n') { + return "", strconv.ErrSyntax + } + + // Is it trivial? Avoid allocation. + if !contains(s, '\\') && !contains(s, quote) { + switch quote { + case '"', '\'': + if utf8.ValidString(s) { + return s, nil + } + } + } + + var runeTmp [utf8.UTFMax]byte + buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations. + for len(s) > 0 { + c, multibyte, ss, err := strconv.UnquoteChar(s, quote) + if err != nil { + return "", err + } + s = ss + if c < utf8.RuneSelf || !multibyte { + buf = append(buf, byte(c)) + } else { + n := utf8.EncodeRune(runeTmp[:], c) + buf = append(buf, runeTmp[:n]...) + } + } + return string(buf), nil +} + +// contains reports whether the string contains the byte c. +func contains(s string, c byte) bool { + return strings.ContainsRune(s, rune(c)) +} diff --git a/pql/parser_test.go b/pql/parser_test.go index 54bb9e332..8e5cf8cc4 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -16,6 +16,7 @@ package pql_test import ( "reflect" + "strings" "testing" "github.com/pilosa/pilosa/v2/pql" @@ -192,4 +193,70 @@ func TestParser_Parse(t *testing.T) { t.Fatalf("unexpected call: %#v", q.Calls[0]) } }) + +} + +func TestUnquote(t *testing.T) { + tests := []struct { + name string + value string + exp string + expErr string + }{ + { + name: "simple double", + value: `"hello"`, + exp: "hello", + }, + { + name: "simple single", + value: `'hello'`, + exp: "hello", + }, + { + name: "double with esc", + value: `"he\"llo"`, + exp: "he\"llo", + }, + { + name: "single with esc", + value: `'he\'llo'`, + exp: "he'llo", + }, + { + name: "single with backslash and esc", + value: `'he\\\'llo'`, + exp: `he\'llo`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := pql.Unquote(test.value) + if testErr(t, test.expErr, err) { + return + } + if got != test.exp { + t.Errorf("exp: '%s'\ngot: '%s'", test.exp, got) + } + }) + } + +} + +func testErr(t *testing.T, exp string, actual error) (done bool) { + t.Helper() + if exp == "" && actual == nil { + return false + } + if exp == "" && actual != nil { + t.Fatalf("unexpected error: %v", actual) + } + if exp != "" && actual == nil { + t.Fatalf("expected error like '%s'", exp) + } + if !strings.Contains(actual.Error(), exp) { + t.Fatalf("unmatched errs exp/got\n%s\n%v", exp, actual) + } + return true } diff --git a/pql/pql.peg b/pql/pql.peg index ecb435549..7e6c039bd 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -64,8 +64,8 @@ itema <- ( 'null' &(comma / sp close) { p.addVal(nil) } ) itemb <- ( < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) } / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) } - / < '"' doublequotedstring '"' > { s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) } - / '\'' < singlequotedstring > '\'' { p.addVal(buffer[begin:end]) } + / < '"' doublequotedstring '"' > { p.addVal(buffer[begin:end]) } + / < '\'' singlequotedstring '\'' > { p.addVal(buffer[begin:end]) } ) float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], true) } / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], true) } @@ -74,8 +74,8 @@ decimal <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], false / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], false) } ) -doublequotedstring <- ( '\\"' / '\\\\' / [^"] )* -singlequotedstring <- ( '\\\'' / '\\\\' / [^'] )* +doublequotedstring <- ( '\\"' / '\\\\' / '\\n' / '\\t' / [^"\\] )* +singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' / [^'\\] )* fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )* field <- { p.addField(buffer[begin:end]) } @@ -83,12 +83,12 @@ reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field') posfield <- { p.addPosStr("_field", buffer[begin:end]) } uint <- [1-9] [0-9]* / '0' col <- ( {p.addPosNum("_col", buffer[begin:end])} - / '\'' '\'' {p.addPosStr("_col", buffer[begin:end])} - / '"' '"' {p.addPosStr("_col", buffer[begin:end])} + / < '\'' singlequotedstring '\'' > {p.addPosStr("_col", buffer[begin:end])} + / < '"' doublequotedstring '"' > {p.addPosStr("_col", buffer[begin:end])} ) row <- ( {p.addPosNum("_row", buffer[begin:end])} - / '\'' '\'' {p.addPosStr("_row", buffer[begin:end])} - / '"' '"' {p.addPosStr("_row", buffer[begin:end])} + / < '\'' singlequotedstring '\'' > {p.addPosStr("_row", buffer[begin:end])} + / < '"' doublequotedstring '"' > {p.addPosStr("_row", buffer[begin:end])} ) open <- '(' sp diff --git a/pql/pql.peg.go b/pql/pql.peg.go index c1e2db0c7..3cba1b702 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -536,8 +536,7 @@ func (p *PQL) Execute() { case ruleAction46: p.addVal(buffer[begin:end]) case ruleAction47: - s, _ := strconv.Unquote(buffer[begin:end]) - p.addVal(s) + p.addVal(buffer[begin:end]) case ruleAction48: p.addVal(buffer[begin:end]) case ruleAction49: @@ -836,42 +835,42 @@ func (p *PQL) Init(options ...func(*PQL) error) error { goto l19 l20: position, tokenIndex = position19, tokenIndex19 - if buffer[position] != rune('\'') { - goto l23 - } - position++ { position24 := position + if buffer[position] != rune('\'') { + goto l23 + } + position++ if !_rules[rulesinglequotedstring]() { goto l23 } + if buffer[position] != rune('\'') { + goto l23 + } + position++ add(rulePegText, position24) } - if buffer[position] != rune('\'') { - goto l23 - } - position++ { add(ruleAction59, position) } goto l19 l23: position, tokenIndex = position19, tokenIndex19 - if buffer[position] != rune('"') { - goto l16 - } - position++ { position26 := position + if buffer[position] != rune('"') { + goto l16 + } + position++ if !_rules[ruledoublequotedstring]() { goto l16 } + if buffer[position] != rune('"') { + goto l16 + } + position++ add(rulePegText, position26) } - if buffer[position] != rune('"') { - goto l16 - } - position++ { add(ruleAction60, position) } @@ -2433,7 +2432,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error { position, tokenIndex = position209, tokenIndex209 return false }, - /* 19 itemb <- <(( Action44 open allargs comma? close Action45) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action46) / (<('"' doublequotedstring '"')> Action47) / ('\'' '\'' Action48))> */ + /* 19 itemb <- <(( Action44 open allargs comma? close Action45) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action46) / (<('"' doublequotedstring '"')> Action47) / (<('\'' singlequotedstring '\'')> Action48))> */ func() bool { position228, tokenIndex228 := position, tokenIndex { @@ -2599,21 +2598,21 @@ func (p *PQL) Init(options ...func(*PQL) error) error { goto l230 l254: position, tokenIndex = position230, tokenIndex230 - if buffer[position] != rune('\'') { - goto l228 - } - position++ { position257 := position + if buffer[position] != rune('\'') { + goto l228 + } + position++ if !_rules[rulesinglequotedstring]() { goto l228 } + if buffer[position] != rune('\'') { + goto l228 + } + position++ add(rulePegText, position257) } - if buffer[position] != rune('\'') { - goto l228 - } - position++ { add(ruleAction48, position) } @@ -2630,7 +2629,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 21 decimal <- <((<('-'? [0-9]+ ('.' [0-9]*)?)> Action51) / (<('-'? '.' [0-9]+)> Action52))> */ nil, - /* 22 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / (!'"' .))*> */ + /* 22 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / ('\\' 'n') / ('\\' 't') / (!('"' / '\\') .))*> */ func() bool { { position262 := position @@ -2660,16 +2659,49 @@ func (p *PQL) Init(options ...func(*PQL) error) error { position++ goto l265 l267: + position, tokenIndex = position265, tokenIndex265 + if buffer[position] != rune('\\') { + goto l268 + } + position++ + if buffer[position] != rune('n') { + goto l268 + } + position++ + goto l265 + l268: + position, tokenIndex = position265, tokenIndex265 + if buffer[position] != rune('\\') { + goto l269 + } + position++ + if buffer[position] != rune('t') { + goto l269 + } + position++ + goto l265 + l269: position, tokenIndex = position265, tokenIndex265 { - position268, tokenIndex268 := position, tokenIndex - if buffer[position] != rune('"') { - goto l268 + position270, tokenIndex270 := position, tokenIndex + { + position271, tokenIndex271 := position, tokenIndex + if buffer[position] != rune('"') { + goto l272 + } + position++ + goto l271 + l272: + position, tokenIndex = position271, tokenIndex271 + if buffer[position] != rune('\\') { + goto l270 + } + position++ } - position++ + l271: goto l264 - l268: - position, tokenIndex = position268, tokenIndex268 + l270: + position, tokenIndex = position270, tokenIndex270 } if !matchDot() { goto l264 @@ -2684,795 +2716,828 @@ func (p *PQL) Init(options ...func(*PQL) error) error { } return true }, - /* 23 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / (!'\'' .))*> */ + /* 23 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / ('\\' 'n') / ('\\' 't') / (!('\'' / '\\') .))*> */ func() bool { { - position270 := position - l271: + position274 := position + l275: { - position272, tokenIndex272 := position, tokenIndex + position276, tokenIndex276 := position, tokenIndex { - position273, tokenIndex273 := position, tokenIndex + position277, tokenIndex277 := position, tokenIndex if buffer[position] != rune('\\') { - goto l274 + goto l278 } position++ if buffer[position] != rune('\'') { - goto l274 + goto l278 } position++ - goto l273 - l274: - position, tokenIndex = position273, tokenIndex273 + goto l277 + l278: + position, tokenIndex = position277, tokenIndex277 if buffer[position] != rune('\\') { - goto l275 + goto l279 } position++ if buffer[position] != rune('\\') { - goto l275 + goto l279 } position++ - goto l273 - l275: - position, tokenIndex = position273, tokenIndex273 + goto l277 + l279: + position, tokenIndex = position277, tokenIndex277 + if buffer[position] != rune('\\') { + goto l280 + } + position++ + if buffer[position] != rune('n') { + goto l280 + } + position++ + goto l277 + l280: + position, tokenIndex = position277, tokenIndex277 + if buffer[position] != rune('\\') { + goto l281 + } + position++ + if buffer[position] != rune('t') { + goto l281 + } + position++ + goto l277 + l281: + position, tokenIndex = position277, tokenIndex277 { - position276, tokenIndex276 := position, tokenIndex - if buffer[position] != rune('\'') { - goto l276 + position282, tokenIndex282 := position, tokenIndex + { + position283, tokenIndex283 := position, tokenIndex + if buffer[position] != rune('\'') { + goto l284 + } + position++ + goto l283 + l284: + position, tokenIndex = position283, tokenIndex283 + if buffer[position] != rune('\\') { + goto l282 + } + position++ } - position++ - goto l272 - l276: - position, tokenIndex = position276, tokenIndex276 + l283: + goto l276 + l282: + position, tokenIndex = position282, tokenIndex282 } if !matchDot() { - goto l272 + goto l276 } } - l273: - goto l271 - l272: - position, tokenIndex = position272, tokenIndex272 + l277: + goto l275 + l276: + position, tokenIndex = position276, tokenIndex276 } - add(rulesinglequotedstring, position270) + add(rulesinglequotedstring, position274) } return true }, /* 24 fieldExpr <- <(([a-z] / [A-Z] / '_') ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ func() bool { - position277, tokenIndex277 := position, tokenIndex + position285, tokenIndex285 := position, tokenIndex { - position278 := position + position286 := position { - position279, tokenIndex279 := position, tokenIndex + position287, tokenIndex287 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l280 + goto l288 } position++ - goto l279 - l280: - position, tokenIndex = position279, tokenIndex279 + goto l287 + l288: + position, tokenIndex = position287, tokenIndex287 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l281 + goto l289 } position++ - goto l279 - l281: - position, tokenIndex = position279, tokenIndex279 + goto l287 + l289: + position, tokenIndex = position287, tokenIndex287 if buffer[position] != rune('_') { - goto l277 + goto l285 } position++ } - l279: - l282: + l287: + l290: { - position283, tokenIndex283 := position, tokenIndex + position291, tokenIndex291 := position, tokenIndex { - position284, tokenIndex284 := position, tokenIndex + position292, tokenIndex292 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l285 + goto l293 } position++ - goto l284 - l285: - position, tokenIndex = position284, tokenIndex284 + goto l292 + l293: + position, tokenIndex = position292, tokenIndex292 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l286 + goto l294 } position++ - goto l284 - l286: - position, tokenIndex = position284, tokenIndex284 + goto l292 + l294: + position, tokenIndex = position292, tokenIndex292 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l287 + goto l295 } position++ - goto l284 - l287: - position, tokenIndex = position284, tokenIndex284 + goto l292 + l295: + position, tokenIndex = position292, tokenIndex292 if buffer[position] != rune('_') { - goto l288 + goto l296 } position++ - goto l284 - l288: - position, tokenIndex = position284, tokenIndex284 + goto l292 + l296: + position, tokenIndex = position292, tokenIndex292 if buffer[position] != rune('-') { - goto l283 + goto l291 } position++ } - l284: - goto l282 - l283: - position, tokenIndex = position283, tokenIndex283 + l292: + goto l290 + l291: + position, tokenIndex = position291, tokenIndex291 } - add(rulefieldExpr, position278) + add(rulefieldExpr, position286) } return true - l277: - position, tokenIndex = position277, tokenIndex277 + l285: + position, tokenIndex = position285, tokenIndex285 return false }, /* 25 field <- <(<(fieldExpr / reserved)> Action53)> */ func() bool { - position289, tokenIndex289 := position, tokenIndex + position297, tokenIndex297 := position, tokenIndex { - position290 := position + position298 := position { - position291 := position + position299 := position { - position292, tokenIndex292 := position, tokenIndex + position300, tokenIndex300 := position, tokenIndex if !_rules[rulefieldExpr]() { - goto l293 + goto l301 } - goto l292 - l293: - position, tokenIndex = position292, tokenIndex292 + goto l300 + l301: + position, tokenIndex = position300, tokenIndex300 { - position294 := position + position302 := position { - position295, tokenIndex295 := position, tokenIndex + position303, tokenIndex303 := position, tokenIndex if buffer[position] != rune('_') { - goto l296 + goto l304 } position++ if buffer[position] != rune('r') { - goto l296 + goto l304 } position++ if buffer[position] != rune('o') { - goto l296 + goto l304 } position++ if buffer[position] != rune('w') { - goto l296 + goto l304 } position++ - goto l295 - l296: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l304: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l297 + goto l305 } position++ if buffer[position] != rune('c') { - goto l297 + goto l305 } position++ if buffer[position] != rune('o') { - goto l297 + goto l305 } position++ if buffer[position] != rune('l') { - goto l297 + goto l305 } position++ - goto l295 - l297: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l305: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l298 + goto l306 } position++ if buffer[position] != rune('s') { - goto l298 + goto l306 } position++ if buffer[position] != rune('t') { - goto l298 + goto l306 } position++ if buffer[position] != rune('a') { - goto l298 + goto l306 } position++ if buffer[position] != rune('r') { - goto l298 + goto l306 } position++ if buffer[position] != rune('t') { - goto l298 + goto l306 } position++ - goto l295 - l298: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l306: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l299 + goto l307 } position++ if buffer[position] != rune('e') { - goto l299 + goto l307 } position++ if buffer[position] != rune('n') { - goto l299 + goto l307 } position++ if buffer[position] != rune('d') { - goto l299 + goto l307 } position++ - goto l295 - l299: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l307: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l300 + goto l308 } position++ if buffer[position] != rune('t') { - goto l300 + goto l308 } position++ if buffer[position] != rune('i') { - goto l300 + goto l308 } position++ if buffer[position] != rune('m') { - goto l300 + goto l308 } position++ if buffer[position] != rune('e') { - goto l300 + goto l308 } position++ if buffer[position] != rune('s') { - goto l300 + goto l308 } position++ if buffer[position] != rune('t') { - goto l300 + goto l308 } position++ if buffer[position] != rune('a') { - goto l300 + goto l308 } position++ if buffer[position] != rune('m') { - goto l300 + goto l308 } position++ if buffer[position] != rune('p') { - goto l300 + goto l308 } position++ - goto l295 - l300: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l308: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l289 + goto l297 } position++ if buffer[position] != rune('f') { - goto l289 + goto l297 } position++ if buffer[position] != rune('i') { - goto l289 + goto l297 } position++ if buffer[position] != rune('e') { - goto l289 + goto l297 } position++ if buffer[position] != rune('l') { - goto l289 + goto l297 } position++ if buffer[position] != rune('d') { - goto l289 + goto l297 } position++ } - l295: - add(rulereserved, position294) + l303: + add(rulereserved, position302) } } - l292: - add(rulePegText, position291) + l300: + add(rulePegText, position299) } { add(ruleAction53, position) } - add(rulefield, position290) + add(rulefield, position298) } return true - l289: - position, tokenIndex = position289, tokenIndex289 + l297: + position, tokenIndex = position297, tokenIndex297 return false }, /* 26 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, /* 27 posfield <- <( Action54)> */ func() bool { - position303, tokenIndex303 := position, tokenIndex + position311, tokenIndex311 := position, tokenIndex { - position304 := position + position312 := position { - position305 := position + position313 := position if !_rules[rulefieldExpr]() { - goto l303 + goto l311 } - add(rulePegText, position305) + add(rulePegText, position313) } { add(ruleAction54, position) } - add(ruleposfield, position304) + add(ruleposfield, position312) } return true - l303: - position, tokenIndex = position303, tokenIndex303 + l311: + position, tokenIndex = position311, tokenIndex311 return false }, /* 28 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position307, tokenIndex307 := position, tokenIndex + position315, tokenIndex315 := position, tokenIndex { - position308 := position + position316 := position { - position309, tokenIndex309 := position, tokenIndex + position317, tokenIndex317 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l310 + goto l318 } position++ - l311: + l319: { - position312, tokenIndex312 := position, tokenIndex + position320, tokenIndex320 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l312 + goto l320 } position++ - goto l311 - l312: - position, tokenIndex = position312, tokenIndex312 + goto l319 + l320: + position, tokenIndex = position320, tokenIndex320 } - goto l309 - l310: - position, tokenIndex = position309, tokenIndex309 + goto l317 + l318: + position, tokenIndex = position317, tokenIndex317 if buffer[position] != rune('0') { - goto l307 + goto l315 } position++ } - l309: - add(ruleuint, position308) + l317: + add(ruleuint, position316) } return true - l307: - position, tokenIndex = position307, tokenIndex307 + l315: + position, tokenIndex = position315, tokenIndex315 return false }, - /* 29 col <- <(( Action55) / ('\'' '\'' Action56) / ('"' '"' Action57))> */ + /* 29 col <- <(( Action55) / (<('\'' singlequotedstring '\'')> Action56) / (<('"' doublequotedstring '"')> Action57))> */ func() bool { - position313, tokenIndex313 := position, tokenIndex + position321, tokenIndex321 := position, tokenIndex { - position314 := position + position322 := position { - position315, tokenIndex315 := position, tokenIndex + position323, tokenIndex323 := position, tokenIndex { - position317 := position + position325 := position if !_rules[ruleuint]() { - goto l316 + goto l324 } - add(rulePegText, position317) + add(rulePegText, position325) } { add(ruleAction55, position) } - goto l315 - l316: - position, tokenIndex = position315, tokenIndex315 - if buffer[position] != rune('\'') { - goto l319 - } - position++ + goto l323 + l324: + position, tokenIndex = position323, tokenIndex323 { - position320 := position - if !_rules[rulesinglequotedstring]() { - goto l319 + position328 := position + if buffer[position] != rune('\'') { + goto l327 } - add(rulePegText, position320) + position++ + if !_rules[rulesinglequotedstring]() { + goto l327 + } + if buffer[position] != rune('\'') { + goto l327 + } + position++ + add(rulePegText, position328) } - if buffer[position] != rune('\'') { - goto l319 - } - position++ { add(ruleAction56, position) } - goto l315 - l319: - position, tokenIndex = position315, tokenIndex315 - if buffer[position] != rune('"') { - goto l313 - } - position++ + goto l323 + l327: + position, tokenIndex = position323, tokenIndex323 { - position322 := position - if !_rules[ruledoublequotedstring]() { - goto l313 + position330 := position + if buffer[position] != rune('"') { + goto l321 } - add(rulePegText, position322) + position++ + if !_rules[ruledoublequotedstring]() { + goto l321 + } + if buffer[position] != rune('"') { + goto l321 + } + position++ + add(rulePegText, position330) } - if buffer[position] != rune('"') { - goto l313 - } - position++ { add(ruleAction57, position) } } - l315: - add(rulecol, position314) + l323: + add(rulecol, position322) } return true - l313: - position, tokenIndex = position313, tokenIndex313 + l321: + position, tokenIndex = position321, tokenIndex321 return false }, - /* 30 row <- <(( Action58) / ('\'' '\'' Action59) / ('"' '"' Action60))> */ + /* 30 row <- <(( Action58) / (<('\'' singlequotedstring '\'')> Action59) / (<('"' doublequotedstring '"')> Action60))> */ nil, /* 31 open <- <('(' sp)> */ func() bool { - position325, tokenIndex325 := position, tokenIndex + position333, tokenIndex333 := position, tokenIndex { - position326 := position + position334 := position if buffer[position] != rune('(') { - goto l325 + goto l333 } position++ if !_rules[rulesp]() { - goto l325 + goto l333 } - add(ruleopen, position326) + add(ruleopen, position334) } return true - l325: - position, tokenIndex = position325, tokenIndex325 + l333: + position, tokenIndex = position333, tokenIndex333 return false }, /* 32 close <- <(')' sp)> */ func() bool { - position327, tokenIndex327 := position, tokenIndex + position335, tokenIndex335 := position, tokenIndex { - position328 := position + position336 := position if buffer[position] != rune(')') { - goto l327 + goto l335 } position++ if !_rules[rulesp]() { - goto l327 + goto l335 } - add(ruleclose, position328) + add(ruleclose, position336) } return true - l327: - position, tokenIndex = position327, tokenIndex327 + l335: + position, tokenIndex = position335, tokenIndex335 return false }, /* 33 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position330 := position - l331: + position338 := position + l339: { - position332, tokenIndex332 := position, tokenIndex + position340, tokenIndex340 := position, tokenIndex { - position333, tokenIndex333 := position, tokenIndex + position341, tokenIndex341 := position, tokenIndex if buffer[position] != rune(' ') { - goto l334 + goto l342 } position++ - goto l333 - l334: - position, tokenIndex = position333, tokenIndex333 + goto l341 + l342: + position, tokenIndex = position341, tokenIndex341 if buffer[position] != rune('\t') { - goto l335 + goto l343 } position++ - goto l333 - l335: - position, tokenIndex = position333, tokenIndex333 + goto l341 + l343: + position, tokenIndex = position341, tokenIndex341 if buffer[position] != rune('\n') { - goto l332 + goto l340 } position++ } - l333: - goto l331 - l332: - position, tokenIndex = position332, tokenIndex332 + l341: + goto l339 + l340: + position, tokenIndex = position340, tokenIndex340 } - add(rulesp, position330) + add(rulesp, position338) } return true }, /* 34 comma <- <(sp ',' sp)> */ func() bool { - position336, tokenIndex336 := position, tokenIndex + position344, tokenIndex344 := position, tokenIndex { - position337 := position + position345 := position if !_rules[rulesp]() { - goto l336 + goto l344 } if buffer[position] != rune(',') { - goto l336 + goto l344 } position++ if !_rules[rulesp]() { - goto l336 + goto l344 } - add(rulecomma, position337) + add(rulecomma, position345) } return true - l336: - position, tokenIndex = position336, tokenIndex336 + l344: + position, tokenIndex = position344, tokenIndex344 return false }, /* 35 lbrack <- <('[' sp)> */ func() bool { - position338, tokenIndex338 := position, tokenIndex + position346, tokenIndex346 := position, tokenIndex { - position339 := position + position347 := position if buffer[position] != rune('[') { - goto l338 + goto l346 } position++ if !_rules[rulesp]() { - goto l338 + goto l346 } - add(rulelbrack, position339) + add(rulelbrack, position347) } return true - l338: - position, tokenIndex = position338, tokenIndex338 + l346: + position, tokenIndex = position346, tokenIndex346 return false }, /* 36 rbrack <- <(sp ']' sp)> */ func() bool { - position340, tokenIndex340 := position, tokenIndex + position348, tokenIndex348 := position, tokenIndex { - position341 := position + position349 := position if !_rules[rulesp]() { - goto l340 + goto l348 } if buffer[position] != rune(']') { - goto l340 + goto l348 } position++ if !_rules[rulesp]() { - goto l340 + goto l348 } - add(rulerbrack, position341) + add(rulerbrack, position349) } return true - l340: - position, tokenIndex = position340, tokenIndex340 + l348: + position, tokenIndex = position348, tokenIndex348 return false }, /* 37 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ func() bool { - position342, tokenIndex342 := position, tokenIndex + position350, tokenIndex350 := position, tokenIndex { - position343 := position + position351 := position { - position344, tokenIndex344 := position, tokenIndex + position352, tokenIndex352 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l345 + goto l353 } position++ - goto l344 - l345: - position, tokenIndex = position344, tokenIndex344 + goto l352 + l353: + position, tokenIndex = position352, tokenIndex352 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l342 + goto l350 } position++ } - l344: - l346: + l352: + l354: { - position347, tokenIndex347 := position, tokenIndex + position355, tokenIndex355 := position, tokenIndex { - position348, tokenIndex348 := position, tokenIndex + position356, tokenIndex356 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l349 + goto l357 } position++ - goto l348 - l349: - position, tokenIndex = position348, tokenIndex348 + goto l356 + l357: + position, tokenIndex = position356, tokenIndex356 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l350 + goto l358 } position++ - goto l348 - l350: - position, tokenIndex = position348, tokenIndex348 + goto l356 + l358: + position, tokenIndex = position356, tokenIndex356 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l347 + goto l355 } position++ } - l348: - goto l346 - l347: - position, tokenIndex = position347, tokenIndex347 + l356: + goto l354 + l355: + position, tokenIndex = position355, tokenIndex355 } - add(ruleIDENT, position343) + add(ruleIDENT, position351) } return true - l342: - position, tokenIndex = position342, tokenIndex342 + l350: + position, tokenIndex = position350, tokenIndex350 return false }, /* 38 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position351, tokenIndex351 := position, tokenIndex + position359, tokenIndex359 := position, tokenIndex { - position352 := position + position360 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if buffer[position] != rune('-') { - goto l351 + goto l359 } position++ { - position353, tokenIndex353 := position, tokenIndex + position361, tokenIndex361 := position, tokenIndex if buffer[position] != rune('0') { - goto l354 + goto l362 } position++ - goto l353 - l354: - position, tokenIndex = position353, tokenIndex353 + goto l361 + l362: + position, tokenIndex = position361, tokenIndex361 if buffer[position] != rune('1') { - goto l351 + goto l359 } position++ } - l353: + l361: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if buffer[position] != rune('-') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if buffer[position] != rune('T') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if buffer[position] != rune(':') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ - add(ruletimestampbasicfmt, position352) + add(ruletimestampbasicfmt, position360) } return true - l351: - position, tokenIndex = position351, tokenIndex351 + l359: + position, tokenIndex = position359, tokenIndex359 return false }, /* 39 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position355, tokenIndex355 := position, tokenIndex + position363, tokenIndex363 := position, tokenIndex { - position356 := position + position364 := position { - position357, tokenIndex357 := position, tokenIndex + position365, tokenIndex365 := position, tokenIndex if buffer[position] != rune('"') { - goto l358 + goto l366 } position++ { - position359 := position + position367 := position if !_rules[ruletimestampbasicfmt]() { - goto l358 + goto l366 } - add(rulePegText, position359) + add(rulePegText, position367) } if buffer[position] != rune('"') { - goto l358 + goto l366 } position++ - goto l357 - l358: - position, tokenIndex = position357, tokenIndex357 + goto l365 + l366: + position, tokenIndex = position365, tokenIndex365 if buffer[position] != rune('\'') { - goto l360 + goto l368 } position++ { - position361 := position + position369 := position if !_rules[ruletimestampbasicfmt]() { - goto l360 + goto l368 } - add(rulePegText, position361) + add(rulePegText, position369) } if buffer[position] != rune('\'') { - goto l360 + goto l368 } position++ - goto l357 - l360: - position, tokenIndex = position357, tokenIndex357 + goto l365 + l368: + position, tokenIndex = position365, tokenIndex365 { - position362 := position + position370 := position if !_rules[ruletimestampbasicfmt]() { - goto l355 + goto l363 } - add(rulePegText, position362) + add(rulePegText, position370) } } - l357: - add(ruletimestampfmt, position356) + l365: + add(ruletimestampfmt, position364) } return true - l355: - position, tokenIndex = position355, tokenIndex355 + l363: + position, tokenIndex = position363, tokenIndex363 return false }, /* 40 timestamp <- <( Action61)> */ @@ -3572,7 +3637,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 89 Action46 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 90 Action47 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ + /* 90 Action47 <- <{ p.addVal(buffer[begin:end]) }> */ nil, /* 91 Action48 <- <{ p.addVal(buffer[begin:end]) }> */ nil, diff --git a/server/server_test.go b/server/server_test.go index d7cb9452e..96d0fa527 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -1092,6 +1092,50 @@ func TestClusterExhaustingConnections(t *testing.T) { } } +func TestQueryingWithQuotesAndStuff(t *testing.T) { + m := test.RunCommand(t) + defer m.Close() + + client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil)) + if err != nil { + t.Fatal(err) + } + + // Execute Set() commands. + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } + if err := client.CreateFieldWithOptions(context.Background(), "i", "fld", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } + + // Test escaped single quote gets set properly + if res, err := m.Query(t, "i", "", `Set('bl\'ah', fld=ha)`); err != nil { + t.Fatal(err) + } else if !strings.Contains(res, "[true]") { + t.Errorf("setting escaped single quote result: %s", res) + } + if res, err := m.Query(t, "i", "", `Row(fld=ha)`); err != nil { + t.Fatal(err) + } else if !strings.Contains(res, `bl'ah`) { + t.Errorf("value with escaped single quote set improperly: %s", res) + } + + // Test escaped double quote gets set properly + if res, err := m.Query(t, "i", "", `Set("d\"ah", fld=dq)`); err != nil { + t.Fatal(err) + } else if !strings.Contains(res, "[true]") { + t.Errorf("value with escaped double quote set improperly: %s", res) + } + if res, err := m.Query(t, "i", "", `Row(fld=dq)`); err != nil { + t.Fatal(err) + } else if !strings.Contains(res, `d\"ah`) { + // the backslash is there because JSON needs to escape the + // double quote since it uses double quotes + t.Errorf("value with escaped double quote set improperly: %s", res) + } +} + func TestClusterExhaustingConnectionsImport(t *testing.T) { if !runStress { t.Skip("stress") From fdaae31c2c36214bfc75eaa3f23902cfe9b434a3 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 2 Jun 2020 15:39:28 -0500 Subject: [PATCH 13/45] add address for listening --- server/server.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/server.go b/server/server.go index 5ff688096..614cfd1ab 100644 --- a/server/server.go +++ b/server/server.go @@ -274,6 +274,15 @@ func (m *Command) SetupServer() error { grpcURI.SetPort(uint16(m.grpcLn.Addr().(*net.TCPAddr).Port)) } + if grpcURI.Scheme == "http" { + grpcURI.Scheme = "grpc" + } + + // discover the address if not specified + if grpcURI.Host == "0.0.0.0" { + grpcURI.Host = outboundIP().String() + } + // Setup TLS if uri.Scheme == "https" { m.tlsConfig, err = GetTLSConfig(&m.Config.TLS, m.logger.Logger()) From 3d270f45d2df9e0affe48044ca99427ab4bceb4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Wed, 27 May 2020 21:56:11 +0200 Subject: [PATCH 14/45] Add (in memory) ETag to index and fields --- api.go | 38 +++- api_test.go | 25 ++- cluster.go | 11 +- encoding/proto/proto.go | 12 ++ field.go | 10 + gossip/gossip.go | 10 +- holder.go | 51 +++++- http/handler.go | 272 ++++++++++++++++++---------- index.go | 23 ++- internal/private.pb.go | 392 +++++++++++++++++++++++++++++++--------- internal/private.proto | 6 + pilosa.go | 5 + server.go | 13 +- server/handler_test.go | 60 ++++++ 14 files changed, 706 insertions(+), 222 deletions(-) diff --git a/api.go b/api.go index 210569f6a..43dba98f0 100644 --- a/api.go +++ b/api.go @@ -181,10 +181,15 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index if err != nil { return nil, errors.Wrap(err, "creating index") } + index.mu.Lock() + index.etag = newETag() + index.mu.Unlock() + // Send the create index message to all nodes. err = api.server.SendSync( &CreateIndexMessage{ Index: indexName, + ETag: index.ETag(), Meta: &options, }) if err != nil { @@ -269,14 +274,17 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str if err != nil { return nil, errors.Wrap(err, "creating field") } + field.mu.Lock() + field.etag = newETag() + field.mu.Unlock() // Send the create field message to all nodes. - err = api.server.SendSync( - &CreateFieldMessage{ - Index: indexName, - Field: fieldName, - Meta: &fo, - }) + err = api.server.SendSync(&CreateFieldMessage{ + Index: indexName, + Field: fieldName, + ETag: field.ETag(), + Meta: &fo, + }) if err != nil { api.server.logger.Printf("problem sending CreateField message: %s", err) return nil, errors.Wrap(err, "sending CreateField message") @@ -803,6 +811,18 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { return errors.Wrap(err, "validating api method") } + // set etags for indexes and fields (if empty), and then apply schema. + for _, index := range s.Indexes { + if index.ETag == 0 { + index.ETag = newETag() + } + for _, field := range index.Fields { + if field.ETag == 0 { + field.ETag = newETag() + } + } + } + if !remote { nodes := api.cluster.Nodes() for i, node := range nodes { @@ -813,7 +833,11 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { } } - return api.holder.applySchema(s) + if err := api.holder.applySchema(s); err != nil { + return errors.Wrap(err, "applying schema") + } + + return nil } // Views returns the views in the given field. diff --git a/api_test.go b/api_test.go index b7f0fc06f..c11ac4f8a 100644 --- a/api_test.go +++ b/api_test.go @@ -186,17 +186,24 @@ func TestAPI_Import(t *testing.T) { t.Run("RowIDColumnKey", func(t *testing.T) { ctx := context.Background() - index := "rick" - field := "f" + indexName := "rick" + fieldName := "f" - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}) + index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + if index.ETag() == 0 { + t.Fatal("index etag is empty") + } + + field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) if err != nil { t.Fatalf("creating field: %v", err) } + if field.ETag() == 0 { + t.Fatal("field etag is empty") + } rowID := uint64(1) timestamp := int64(0) @@ -215,8 +222,8 @@ func TestAPI_Import(t *testing.T) { // Import data with keys to the coordinator (node0) and verify that it gets // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) req := &pilosa.ImportRequest{ - Index: index, - Field: field, + Index: indexName, + Field: fieldName, Shard: 0, RowIDs: rowIDs, ColumnKeys: colKeys, @@ -226,10 +233,10 @@ func TestAPI_Import(t *testing.T) { t.Fatal(err) } - pql := fmt.Sprintf("Row(%s=%d)", field, rowID) + pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID) // Query node0. - if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil { t.Fatal(err) } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { t.Fatalf("unexpected column keys: %#v", keys) @@ -237,7 +244,7 @@ func TestAPI_Import(t *testing.T) { // Query node1. if err := test.RetryUntil(5*time.Second, func() error { - if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil { return err } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { return fmt.Errorf("unexpected column keys: %#v", keys) diff --git a/cluster.go b/cluster.go index e979710d8..72ce09b17 100644 --- a/cluster.go +++ b/cluster.go @@ -1272,9 +1272,7 @@ func (c *cluster) listenForJoins() { // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. // We use a bool `setNormal` to indicate when at least one node has joined. var setNormal bool - for { - // Handle all pending joins before changing state back to NORMAL. select { case nodeAction := <-c.joiningLeavingNodes: @@ -2146,7 +2144,7 @@ func (c *cluster) nodeStatus() *NodeStatus { } var availableShards *roaring.Bitmap for _, idx := range ns.Schema.Indexes { - is := &IndexStatus{Name: idx.Name} + is := &IndexStatus{Name: idx.Name, ETag: idx.ETag} for _, f := range idx.Fields { if field := c.holder.Field(idx.Name, f.Name); field != nil { availableShards = field.AvailableShards() @@ -2155,6 +2153,7 @@ func (c *cluster) nodeStatus() *NodeStatus { } is.Fields = append(is.Fields, &FieldStatus{ Name: f.Name, + ETag: f.ETag, AvailableShards: availableShards, }) } @@ -2491,7 +2490,7 @@ type translationResizeNode struct { // Schema contains information about indexes and their configuration. type Schema struct { - Indexes []*IndexInfo + Indexes []*IndexInfo `json:"indexes"` } func encodeTopology(topology *Topology) *internal.Topology { @@ -2530,6 +2529,7 @@ type CreateShardMessage struct { // CreateIndexMessage is an internal message indicating index creation. type CreateIndexMessage struct { Index string + ETag int64 Meta *IndexOptions } @@ -2542,6 +2542,7 @@ type DeleteIndexMessage struct { type CreateFieldMessage struct { Index string Field string + ETag int64 Meta *FieldOptions } @@ -2606,12 +2607,14 @@ type NodeStatus struct { // IndexStatus is an internal message representing the contents of an index. type IndexStatus struct { Name string + ETag int64 Fields []*FieldStatus } // FieldStatus is an internal message representing the contents of a field. type FieldStatus struct { Name string + ETag int64 AvailableShards *roaring.Bitmap } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index a4508f652..6eb389302 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -594,6 +594,7 @@ func (s Serializer) encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index func (s Serializer) encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index { return &internal.Index{ Name: idx.Name, + ETag: idx.ETag, Options: s.encodeIndexMeta(&idx.Options), Fields: s.encodeFieldInfos(idx.Fields), } @@ -610,6 +611,7 @@ func (s Serializer) encodeFieldInfos(fs []*pilosa.FieldInfo) []*internal.Field { func (s Serializer) encodeFieldInfo(f *pilosa.FieldInfo) *internal.Field { ifield := &internal.Field{ Name: f.Name, + ETag: f.ETag, Meta: s.encodeFieldOptions(&f.Options), Views: make([]string, 0, len(f.Views)), } @@ -686,6 +688,7 @@ func (s Serializer) encodeCreateShardMessage(m *pilosa.CreateShardMessage) *inte func (s Serializer) encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateIndexMessage { return &internal.CreateIndexMessage{ Index: m.Index, + ETag: m.ETag, Meta: s.encodeIndexMeta(m.Meta), } } @@ -707,6 +710,7 @@ func (s Serializer) encodeCreateFieldMessage(m *pilosa.CreateFieldMessage) *inte return &internal.CreateFieldMessage{ Index: m.Index, Field: m.Field, + ETag: m.ETag, Meta: s.encodeFieldOptions(m.Meta), } } @@ -787,6 +791,7 @@ func (s Serializer) encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus func (s Serializer) encodeIndexStatus(m *pilosa.IndexStatus) *internal.IndexStatus { return &internal.IndexStatus{ Name: m.Name, + ETag: m.ETag, Fields: s.encodeFieldStatuses(m.Fields), } } @@ -802,6 +807,7 @@ func (s Serializer) encodeIndexStatuses(a []*pilosa.IndexStatus) []*internal.Ind func (s Serializer) encodeFieldStatus(m *pilosa.FieldStatus) *internal.FieldStatus { return &internal.FieldStatus{ Name: m.Name, + ETag: m.ETag, AvailableShards: m.AvailableShards.Slice(), } } @@ -938,6 +944,7 @@ func (s Serializer) decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo) func (s Serializer) decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) { m.Name = idx.Name + m.ETag = idx.ETag m.Options = pilosa.IndexOptions{} s.decodeIndexMeta(idx.Options, &m.Options) m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields)) @@ -953,6 +960,7 @@ func (s Serializer) decodeFields(fs []*internal.Field, m []*pilosa.FieldInfo) { func (s Serializer) decodeField(f *internal.Field, m *pilosa.FieldInfo) { m.Name = f.Name + m.ETag = f.ETag m.Options = pilosa.FieldOptions{} s.decodeFieldOptions(f.Meta, &m.Options) m.Views = make([]*pilosa.ViewInfo, 0, len(f.Views)) @@ -1016,6 +1024,7 @@ func (s Serializer) decodeCreateShardMessage(pb *internal.CreateShardMessage, m func (s Serializer) decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) { m.Index = pb.Index + m.ETag = pb.ETag m.Meta = &pilosa.IndexOptions{} s.decodeIndexMeta(pb.Meta, m.Meta) } @@ -1034,6 +1043,7 @@ func (s Serializer) decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m func (s Serializer) decodeCreateFieldMessage(pb *internal.CreateFieldMessage, m *pilosa.CreateFieldMessage) { m.Index = pb.Index m.Field = pb.Field + m.ETag = pb.ETag m.Meta = &pilosa.FieldOptions{} s.decodeFieldOptions(pb.Meta, m.Meta) } @@ -1107,6 +1117,7 @@ func (s Serializer) decodeIndexStatuses(a []*internal.IndexStatus) []*pilosa.Ind func (s Serializer) decodeIndexStatus(pb *internal.IndexStatus, m *pilosa.IndexStatus) { m.Name = pb.Name + m.ETag = pb.ETag m.Fields = s.decodeFieldStatuses(pb.Fields) } @@ -1121,6 +1132,7 @@ func (s Serializer) decodeFieldStatuses(a []*internal.FieldStatus) []*pilosa.Fie func (s Serializer) decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldStatus) { m.Name = pb.Name + m.ETag = pb.ETag m.AvailableShards = roaring.NewBitmap(pb.AvailableShards...) } diff --git a/field.go b/field.go index d38aa2ffb..272717565 100644 --- a/field.go +++ b/field.go @@ -87,6 +87,7 @@ var availableShardFileFlushDuration = &protected{ // Field represents a container for views. type Field struct { mu sync.RWMutex + etag int64 path string index string name string @@ -382,6 +383,14 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) { // Name returns the name the field was initialized with. func (f *Field) Name() string { return f.name } +// ETag is an identifier for a specific version of field. +func (f *Field) ETag() int64 { + f.mu.RLock() + defer f.mu.RUnlock() + + return f.etag +} + // Index returns the index name the field was initialized with. func (f *Field) Index() string { return f.index } @@ -1972,6 +1981,7 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FieldInfo represents schema information for a field. type FieldInfo struct { Name string `json:"name"` + ETag int64 `json:"etag,omitempty"` Options FieldOptions `json:"options"` Views []*ViewInfo `json:"views,omitempty"` } diff --git a/gossip/gossip.go b/gossip/gossip.go index 84f914975..115c4274b 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -324,16 +324,20 @@ func (g *memberSet) LocalState(join bool) []byte { Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())}, } for _, idx := range m.Schema.Indexes { - is := &pilosa.IndexStatus{Name: idx.Name} + is := &pilosa.IndexStatus{Name: idx.Name, ETag: idx.ETag} + for _, f := range idx.Fields { availableShards := roaring.NewBitmap() if field, _ := g.papi.Field(context.Background(), idx.Name, f.Name); field != nil { availableShards = field.AvailableShards() } - is.Fields = append(is.Fields, &pilosa.FieldStatus{ + + fs := &pilosa.FieldStatus{ Name: f.Name, + ETag: f.ETag, AvailableShards: availableShards, - }) + } + is.Fields = append(is.Fields, fs) } m.Indexes = append(m.Indexes, is) } diff --git a/holder.go b/holder.go index ad52ee082..7434a0d0c 100644 --- a/holder.go +++ b/holder.go @@ -234,7 +234,13 @@ func (h *Holder) Open() error { return errors.Wrap(err, "opening index") } - if err := index.Open(); err != nil { + if h.isCoordinator() { + index.etag = newETag() + err = index.OpenWithETag() + } else { + err = index.Open() + } + if err != nil { if err == ErrName { h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err) continue @@ -381,10 +387,15 @@ func (h *Holder) Schema() []*IndexInfo { for _, index := range h.Indexes() { di := &IndexInfo{ Name: index.Name(), + ETag: index.ETag(), Options: index.Options(), } for _, field := range index.Fields() { - fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + fi := &FieldInfo{ + Name: field.Name(), + ETag: field.ETag(), + Options: field.Options(), + } for _, view := range field.views() { fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) } @@ -404,6 +415,7 @@ func (h *Holder) limitedSchema() []*IndexInfo { for _, index := range h.Indexes() { di := &IndexInfo{ Name: index.Name(), + ETag: index.ETag(), Options: index.Options(), ShardWidth: ShardWidth, } @@ -411,7 +423,11 @@ func (h *Holder) limitedSchema() []*IndexInfo { if strings.HasPrefix(field.name, "_") { continue } - fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + fi := &FieldInfo{ + Name: field.Name(), + ETag: field.ETag(), + Options: field.Options(), + } di.Fields = append(di.Fields, fi) } sort.Sort(fieldInfoSlice(di.Fields)) @@ -424,20 +440,32 @@ func (h *Holder) limitedSchema() []*IndexInfo { // applySchema applies an internal Schema to Holder. func (h *Holder) applySchema(schema *Schema) error { // Create indexes that don't exist. - for _, index := range schema.Indexes { - idx, err := h.CreateIndexIfNotExists(index.Name, index.Options) + for _, i := range schema.Indexes { + idx, err := h.CreateIndexIfNotExists(i.Name, i.Options) if err != nil { return errors.Wrap(err, "creating index") } + if i.ETag != 0 { + idx.mu.Lock() + idx.etag = i.ETag + idx.mu.Unlock() + } + // Create fields that don't exist. - for _, f := range index.Fields { - field, err := idx.createFieldIfNotExists(f.Name, &f.Options) + for _, f := range i.Fields { + fld, err := idx.createFieldIfNotExists(f.Name, &f.Options) if err != nil { return errors.Wrap(err, "creating field") } + if f.ETag != 0 { + fld.mu.Lock() + fld.etag = f.ETag + fld.mu.Unlock() + } + // Create views that don't exist. for _, v := range f.Views { - _, err := field.createViewIfNotExists(v.Name) + _, err := fld.createViewIfNotExists(v.Name) if err != nil { return errors.Wrap(err, "creating view") } @@ -652,6 +680,13 @@ func (h *Holder) recalculateCaches() { } } +func (h *Holder) isCoordinator() bool { + if s, ok := h.broadcaster.(*Server); ok { + return s.isCoordinator + } + return false +} + // setFileLimit attempts to set the open file limit to the FileLimit constant defined above. func (h *Holder) setFileLimit() { oldLimit := &syscall.Rlimit{} diff --git a/http/handler.go b/http/handler.go index 5b9eaf405..555c789c9 100644 --- a/http/handler.go +++ b/http/handler.go @@ -15,13 +15,13 @@ package http import ( + "bytes" "context" "crypto/tls" "encoding/json" "expvar" "fmt" "io" - "io/ioutil" "math" "net" "net/http" @@ -504,7 +504,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") schema := h.api.Schema(r.Context()) - if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { // TODO: use pilosa.Schema instead of map[string]interface{} here? + if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil { h.logger.Printf("write schema response error: %s", err) } } @@ -787,8 +787,14 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { resp.write(w, err) return } - _, err = h.api.CreateIndex(r.Context(), indexName, req.Options) - + index, err := h.api.CreateIndex(r.Context(), indexName, req.Options) + if index != nil { + w.Header().Add("ETag", strconv.FormatInt(index.ETag(), 10)) + } else if _, ok = err.(pilosa.ConflictError); ok { + if index, _ = h.api.Index(r.Context(), indexName); index != nil { + w.Header().Add("ETag", strconv.FormatInt(index.ETag(), 10)) + } + } resp.write(w, err) } @@ -925,11 +931,18 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { fos = append(fos, pilosa.OptFieldForeignIndex(*req.Options.ForeignIndex)) } - _, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...) + field, err := h.api.CreateField(r.Context(), indexName, fieldName, fos...) if _, ok := err.(pilosa.BadRequestError); ok { http.Error(w, err.Error(), http.StatusBadRequest) return } + if field != nil { + w.Header().Add("ETag", strconv.FormatInt(field.ETag(), 10)) + } else if _, ok = err.(pilosa.ConflictError); ok { + if field, _ = h.api.Field(r.Context(), indexName, fieldName); field != nil { + w.Header().Add("ETag", strconv.FormatInt(field.ETag(), 10)) + } + } resp.write(w, err) } @@ -1240,7 +1253,7 @@ func (h *Handler) readQueryRequest(r *http.Request) (*pilosa.QueryRequest, error // readProtobufQueryRequest parses query parameters in protobuf from r. func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { // Slurp the body. - body, err := ioutil.ReadAll(r.Body) + body, err := readBody(r) if err != nil { return nil, errors.Wrap(err, "reading") } @@ -1258,7 +1271,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er q := r.URL.Query() // Parse query string. - buf, err := ioutil.ReadAll(r.Body) + buf, err := readBody(r) if err != nil { return nil, errors.Wrap(err, "reading") } @@ -1319,95 +1332,38 @@ func (h *Handler) writeJSONQueryResponse(w io.Writer, resp *pilosa.QueryResponse return json.NewEncoder(w).Encode(resp) } -// handlePostImport handles /import requests. -func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. +func validateProtobufHeader(r *http.Request) (error string, code int) { if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - return + return "Unsupported media type", http.StatusUnsupportedMediaType } - indexName := mux.Vars(r)["index"] - fieldName := mux.Vars(r)["field"] - - // If the clear flag is true, treat the import as clear bits. - q := r.URL.Query() - doClear := q.Get("clear") == "true" - doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" - - opts := []pilosa.ImportOption{ - pilosa.OptImportOptionsClear(doClear), - pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck), + if r.Header.Get("Accept") != "application/x-protobuf" { + return "Not acceptable", http.StatusNotAcceptable } + return +} - // Get index and field type to determine how to handle the - // import data. - field, err := h.api.Field(r.Context(), indexName, fieldName) - if err != nil { - switch errors.Cause(err) { - case pilosa.ErrIndexNotFound: - fallthrough - case pilosa.ErrFieldNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return +func validateETagHeader(r *http.Request, index *pilosa.Index, field *pilosa.Field) (error string, code int) { + etags := strings.Split(r.Header.Get("If-Match"), ",") + netags := len(etags) + for i := 0; i < netags && i < 2; i++ { + etags[i] = strings.TrimLeft(strings.TrimSpace(etags[i]), "W/") } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return + if netags == 1 && etags[0] != "" && etags[0] != "*" { + return "Precondition Failed", http.StatusPreconditionFailed } - - // Unmarshal request based on field type. - if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal { - // Field type: Int - // Marshal into request object. - req := &pilosa.ImportValueRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return + if netags > 1 { + indexETag := etags[0] + if indexETag != "" && strconv.FormatInt(index.ETag(), 10) != indexETag { + return "Precondition Failed", http.StatusPreconditionFailed } - if err := h.api.ImportValue(r.Context(), req, opts...); err != nil { - switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - } else { - // Field type: set, time, mutex - // Marshal into request object. - req := &pilosa.ImportRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.api.Import(r.Context(), req, opts...); err != nil { - switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return + fieldETag := etags[1] + if fieldETag != "" && strconv.FormatInt(field.ETag(), 10) != fieldETag { + return "Precondition Failed", http.StatusPreconditionFailed } } - // Write response. - _, err = w.Write(importOk) - if err != nil { - h.logger.Printf("writing import response: %v", err) - } + return } // handleGetExport handles /export requests. @@ -1885,6 +1841,101 @@ func GetHTTPClient(t *tls.Config) *http.Client { return &http.Client{Transport: transport} } +// handlePostImport handles /import requests. +func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if error, code := validateProtobufHeader(r); error != "" { + http.Error(w, error, code) + return + } + + // Get index and field type to determine how to handle the + // import data. + indexName := mux.Vars(r)["index"] + index, err := h.api.Index(r.Context(), indexName) + if err != nil { + if errors.Cause(err) == pilosa.ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + fieldName := mux.Vars(r)["field"] + field := index.Field(fieldName) + if field == nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + // Verify if request matches etag + if error, code := validateETagHeader(r, index, field); error != "" { + http.Error(w, error, code) + return + } + + // If the clear flag is true, treat the import as clear bits. + q := r.URL.Query() + doClear := q.Get("clear") == "true" + doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" + + opts := []pilosa.ImportOption{ + pilosa.OptImportOptionsClear(doClear), + pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck), + } + + // Read entire body. + body, err := readBody(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + // Unmarshal request based on field type. + if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal { + // Field type: Int + // Marshal into request object. + req := &pilosa.ImportValueRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.api.ImportValue(r.Context(), req, opts...); err != nil { + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + } else { + // Field type: set, time, mutex + // Marshal into request object. + req := &pilosa.ImportRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.api.Import(r.Context(), req, opts...); err != nil { + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + } + + // Write response. + _, err = w.Write(importOk) + if err != nil { + h.logger.Printf("writing import response: %v", err) + } +} + // handlePostImportColumnAttrs func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. @@ -1898,7 +1949,7 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req opts := []pilosa.ImportOption{} - body, err := ioutil.ReadAll(r.Body) + body, err := readBody(r) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -1922,18 +1973,38 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req } } -// handlPostRoaringImport +// handlePostImportRoaring func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) + if error, code := validateProtobufHeader(r); error != "" { + http.Error(w, error, code) return } + + // Get index and field type to determine how to handle the + // import data. indexName := mux.Vars(r)["index"] + index, err := h.api.Index(r.Context(), indexName) + if err != nil { + if errors.Cause(err) == pilosa.ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } fieldName := mux.Vars(r)["field"] + field := index.Field(fieldName) + if field == nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + // Verify if request matches etag + if error, code := validateETagHeader(r, index, field); error != "" { + http.Error(w, error, code) + return + } q := r.URL.Query() remoteStr := q.Get("remote") @@ -1946,7 +2017,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request // Read entire body. span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body") - body, err := ioutil.ReadAll(r.Body) + body, err := readBody(r) span.LogKV("bodySize", len(body)) span.Finish() if err != nil { @@ -2040,3 +2111,18 @@ func (h *Handler) handlePostTranslateIDs(w http.ResponseWriter, r *http.Request) h.logger.Printf("writing translate keys response: %v", err) } } + +// Read entire request body. +func readBody(r *http.Request) ([]byte, error) { + var contentLength int64 = bytes.MinRead + if r.ContentLength > 0 { + contentLength = r.ContentLength + } + + buf := bytes.NewBuffer(make([]byte, 0, 1+contentLength)) + if _, err := buf.ReadFrom(r.Body); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/index.go b/index.go index a5f143bfc..aaf61e70f 100644 --- a/index.go +++ b/index.go @@ -37,6 +37,7 @@ import ( // Index represents a container for fields. type Index struct { mu sync.RWMutex + etag int64 path string name string keys bool // use string keys @@ -103,6 +104,13 @@ func NewIndex(path, name string, partitionN int) (*Index, error) { }, nil } +// ETag is an identifier for a specific version of an index. +func (i *Index) ETag() int64 { + i.mu.RLock() + defer i.mu.RUnlock() + return i.etag +} + // Name returns name of the index. func (i *Index) Name() string { return i.name } @@ -140,7 +148,12 @@ func (i *Index) options() IndexOptions { } // Open opens and initializes the index. -func (i *Index) Open() (err error) { +func (i *Index) Open() error { return i.open(false) } + +// OpenWithETag opens and initializes the index and set a new ETag for fields. +func (i *Index) OpenWithETag() error { return i.open(true) } + +func (i *Index) open(withETag bool) (err error) { // Ensure the path exists. i.logger.Debugf("ensure index path exists: %s", i.path) if err := os.MkdirAll(i.path, 0777); err != nil { @@ -154,7 +167,7 @@ func (i *Index) Open() (err error) { } i.logger.Debugf("open fields for index: %s", i.name) - if err := i.openFields(); err != nil { + if err := i.openFields(withETag); err != nil { return errors.Wrap(err, "opening fields") } @@ -197,7 +210,7 @@ func (i *Index) Open() (err error) { var indexQueue = make(chan struct{}, 8) // openFields opens and initializes the fields inside the index. -func (i *Index) openFields() error { +func (i *Index) openFields(withETag bool) error { f, err := os.Open(i.path) if err != nil { return errors.Wrap(err, "opening directory") @@ -229,6 +242,9 @@ fileLoop: i.logger.Debugf("open field: %s", fi.Name()) mu.Lock() fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + if withETag { + fld.etag = newETag() + } mu.Unlock() if err != nil { return errors.Wrapf(ErrName, "'%s'", fi.Name()) @@ -559,6 +575,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { Name string `json:"name"` + ETag int64 `json:"etag,omitempty"` Options IndexOptions `json:"options"` Fields []*FieldInfo `json:"fields"` ShardWidth uint64 `json:"shardWidth"` diff --git a/internal/private.pb.go b/internal/private.pb.go index 38eb7af3d..694935fb0 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -616,6 +616,7 @@ func (m *DeleteIndexMessage) GetIndex() string { type CreateIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` + ETag int64 `protobuf:"varint,3,opt,name=ETag,proto3" json:"ETag,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -668,10 +669,18 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { return nil } +func (m *CreateIndexMessage) GetETag() int64 { + if m != nil { + return m.ETag + } + return 0 +} + type CreateFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta,proto3" json:"Meta,omitempty"` + ETag int64 `protobuf:"varint,4,opt,name=ETag,proto3" json:"ETag,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -731,6 +740,13 @@ func (m *CreateFieldMessage) GetMeta() *FieldOptions { return nil } +func (m *CreateFieldMessage) GetETag() int64 { + if m != nil { + return m.ETag + } + return 0 +} + type DeleteFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -853,6 +869,7 @@ type Field struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` Views []string `protobuf:"bytes,3,rep,name=Views,proto3" json:"Views,omitempty"` + ETag int64 `protobuf:"varint,4,opt,name=ETag,proto3" json:"ETag,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -912,6 +929,13 @@ func (m *Field) GetViews() []string { return nil } +func (m *Field) GetETag() int64 { + if m != nil { + return m.ETag + } + return 0 +} + type Schema struct { Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes,proto3" json:"Indexes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -961,6 +985,7 @@ func (m *Schema) GetIndexes() []*Index { type Index struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + ETag int64 `protobuf:"varint,2,opt,name=ETag,proto3" json:"ETag,omitempty"` Options *IndexMeta `protobuf:"bytes,5,opt,name=Options,proto3" json:"Options,omitempty"` Fields []*Field `protobuf:"bytes,4,rep,name=Fields,proto3" json:"Fields,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1008,6 +1033,13 @@ func (m *Index) GetName() string { return "" } +func (m *Index) GetETag() int64 { + if m != nil { + return m.ETag + } + return 0 +} + func (m *Index) GetOptions() *IndexMeta { if m != nil { return m.Options @@ -1340,6 +1372,7 @@ func (m *NodeStatus) GetIndexes() []*IndexStatus { type IndexStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields,proto3" json:"Fields,omitempty"` + ETag int64 `protobuf:"varint,3,opt,name=ETag,proto3" json:"ETag,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1392,9 +1425,17 @@ func (m *IndexStatus) GetFields() []*FieldStatus { return nil } +func (m *IndexStatus) GetETag() int64 { + if m != nil { + return m.ETag + } + return 0 +} + type FieldStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards,proto3" json:"AvailableShards,omitempty"` + ETag int64 `protobuf:"varint,3,opt,name=ETag,proto3" json:"ETag,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1447,6 +1488,13 @@ func (m *FieldStatus) GetAvailableShards() []uint64 { return nil } +func (m *FieldStatus) GetETag() int64 { + if m != nil { + return m.ETag + } + return 0 +} + type ClusterStatus struct { ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` @@ -2421,96 +2469,98 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1418 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x72, 0x1b, 0xc5, - 0x13, 0xff, 0xaf, 0x56, 0xb6, 0xa4, 0x96, 0xe5, 0xc8, 0x93, 0xc4, 0xd9, 0xf8, 0x4f, 0x19, 0x31, - 0xa4, 0x88, 0x48, 0x15, 0x26, 0x95, 0x50, 0xc5, 0x67, 0xaa, 0x12, 0x5b, 0x4e, 0x10, 0xc1, 0x8e, - 0x33, 0x72, 0x72, 0xe3, 0x30, 0x5e, 0x4d, 0xc5, 0x5b, 0x5e, 0xed, 0x8a, 0xdd, 0x59, 0x47, 0xce, - 0x81, 0x2b, 0x54, 0xf1, 0x02, 0x1c, 0x38, 0xf0, 0x1e, 0xbc, 0x00, 0x47, 0x1e, 0x81, 0x0a, 0x4f, - 0xc1, 0x8d, 0x9a, 0x9e, 0x99, 0xdd, 0x95, 0x2c, 0xe3, 0x90, 0x70, 0xdb, 0xfe, 0xf5, 0x77, 0x4f, - 0x77, 0xcf, 0x2c, 0xb4, 0xc6, 0x49, 0x70, 0xcc, 0xa5, 0xd8, 0x18, 0x27, 0xb1, 0x8c, 0x49, 0x3d, - 0x88, 0xa4, 0x48, 0x22, 0x1e, 0xae, 0x2d, 0x8d, 0xb3, 0x83, 0x30, 0xf0, 0x35, 0x4e, 0x1f, 0x40, - 0xa3, 0x1f, 0x0d, 0xc5, 0x64, 0x47, 0x48, 0x4e, 0x08, 0x54, 0x1f, 0x8a, 0x93, 0xd4, 0x73, 0x3b, - 0x4e, 0xb7, 0xce, 0xf0, 0x9b, 0xbc, 0x07, 0xcb, 0xfb, 0x09, 0xf7, 0x8f, 0xb6, 0x27, 0x41, 0x2a, - 0x45, 0xe4, 0x0b, 0xaf, 0x8a, 0xdc, 0x19, 0x94, 0xfe, 0xe2, 0xc2, 0xd2, 0xfd, 0x40, 0x84, 0xc3, - 0x47, 0x63, 0x19, 0xc4, 0x51, 0xaa, 0x8c, 0xed, 0x9f, 0x8c, 0x85, 0x57, 0xef, 0x38, 0xdd, 0x06, - 0xc3, 0x6f, 0xf2, 0x16, 0x34, 0xb6, 0xb8, 0x7f, 0x28, 0x90, 0xe1, 0x22, 0xa3, 0x00, 0x72, 0xee, - 0x20, 0x78, 0xa1, 0xbd, 0xb4, 0x58, 0x01, 0x90, 0x0e, 0x34, 0xf7, 0x83, 0x91, 0x78, 0x9c, 0xf1, - 0x48, 0x66, 0x23, 0x6f, 0x01, 0xb5, 0xcb, 0x10, 0x59, 0x85, 0xc5, 0x47, 0xe1, 0x70, 0x27, 0x88, - 0xbc, 0x46, 0xc7, 0xe9, 0xba, 0xcc, 0x50, 0x16, 0xe7, 0x13, 0x0f, 0x0a, 0x9c, 0x4f, 0xf2, 0x74, - 0x9b, 0xd3, 0xe9, 0xee, 0xc6, 0x03, 0xc9, 0xa3, 0x21, 0x4f, 0x86, 0x4f, 0x03, 0xf1, 0xdc, 0x5b, - 0xd2, 0xe9, 0x4e, 0xa3, 0x4a, 0x77, 0x93, 0xa7, 0xc2, 0x6b, 0xa1, 0x45, 0xfc, 0x26, 0x6b, 0x50, - 0xdf, 0x0c, 0x64, 0x4f, 0x8c, 0xe5, 0xa1, 0xb7, 0xdc, 0x71, 0xba, 0x55, 0x96, 0xd3, 0xe4, 0x12, - 0x2c, 0x0c, 0x7c, 0x1e, 0x0a, 0xef, 0x02, 0x2a, 0x68, 0x82, 0x50, 0x58, 0xba, 0x1f, 0x27, 0x22, - 0x78, 0x16, 0xe1, 0x21, 0x78, 0x6d, 0x4c, 0x6a, 0x0a, 0x23, 0xef, 0x82, 0xab, 0x52, 0x5a, 0xe9, - 0x38, 0xdd, 0xe6, 0xad, 0x95, 0x0d, 0x7b, 0x8e, 0x1b, 0x3d, 0xe1, 0x07, 0x23, 0x1e, 0x32, 0xc5, - 0x45, 0x21, 0x3e, 0xf1, 0xc8, 0xd9, 0x42, 0x7c, 0x42, 0x29, 0x2c, 0xf7, 0x47, 0xe3, 0x38, 0x91, - 0x4c, 0xa4, 0xe3, 0x38, 0x4a, 0x05, 0x69, 0x83, 0xbb, 0x9d, 0x24, 0x9e, 0x83, 0x6e, 0xd5, 0x27, - 0xfd, 0x0e, 0xda, 0x9b, 0x61, 0xec, 0x1f, 0xf5, 0xb8, 0xe4, 0x4c, 0x7c, 0x9b, 0x89, 0x54, 0xaa, - 0xd8, 0x75, 0x78, 0x5a, 0x4e, 0x13, 0x0a, 0xc5, 0xf3, 0xf6, 0x2a, 0x1a, 0x45, 0x42, 0xd5, 0x05, - 0xab, 0xa6, 0x8f, 0x07, 0xbf, 0x31, 0xf7, 0x43, 0x9e, 0x0c, 0xf1, 0x4c, 0xab, 0x4c, 0x13, 0x0a, - 0x45, 0x4f, 0xd8, 0x07, 0x55, 0xa6, 0x09, 0xda, 0x87, 0x95, 0x92, 0x7f, 0x13, 0xe6, 0x2a, 0x2c, - 0xb2, 0xf8, 0x79, 0xbf, 0x97, 0x7a, 0x4e, 0xc7, 0xed, 0x56, 0x99, 0xa1, 0xb0, 0x61, 0xe2, 0x30, - 0x1b, 0x45, 0x8a, 0x55, 0x41, 0x56, 0x01, 0xd0, 0xab, 0xb0, 0x80, 0xdd, 0xa3, 0xb2, 0x2c, 0x74, - 0xd5, 0x27, 0xfd, 0xde, 0x81, 0xc6, 0x0e, 0x9f, 0x60, 0x20, 0x29, 0xb9, 0x03, 0x75, 0x7b, 0xb6, - 0x28, 0xd4, 0xbc, 0xf5, 0x4e, 0x51, 0xc1, 0x5c, 0x6c, 0xc3, 0xca, 0x6c, 0x47, 0x32, 0x39, 0x61, - 0xb9, 0xca, 0xda, 0xe7, 0xd0, 0x9a, 0x62, 0x29, 0x7f, 0x47, 0xe2, 0xc4, 0x56, 0xf5, 0x48, 0x9c, - 0xa8, 0x5c, 0x8f, 0x79, 0x98, 0x09, 0xac, 0x55, 0x95, 0x69, 0xe2, 0xb3, 0xca, 0x27, 0x0e, 0x7d, - 0x0a, 0x64, 0x2b, 0x11, 0x5c, 0x0a, 0x74, 0xb2, 0x23, 0xd2, 0x94, 0x3f, 0x13, 0xe7, 0x55, 0xdc, - 0x2d, 0x57, 0x3c, 0xaf, 0x6e, 0xa5, 0x54, 0x5d, 0x7a, 0x03, 0x48, 0x4f, 0x84, 0x42, 0x0a, 0x33, - 0xdd, 0xff, 0x60, 0x97, 0x0e, 0x6c, 0x0c, 0xe7, 0xcb, 0x92, 0xeb, 0x50, 0x55, 0xab, 0x02, 0x9d, - 0x35, 0x6f, 0x5d, 0x2c, 0xea, 0x94, 0x6f, 0x11, 0x86, 0x02, 0x34, 0xb4, 0x46, 0x31, 0xca, 0x57, - 0x4c, 0x6c, 0xaa, 0x95, 0x6e, 0x18, 0x57, 0x2e, 0xba, 0x5a, 0x2d, 0x5c, 0x95, 0xd7, 0x8c, 0xf1, - 0x76, 0xd7, 0xa6, 0xfb, 0xba, 0xde, 0xa8, 0x0f, 0xff, 0xd7, 0x16, 0xee, 0x1d, 0xf3, 0x20, 0xe4, - 0x07, 0xe1, 0xbf, 0x3a, 0x91, 0xa9, 0xc0, 0x3d, 0xa8, 0xa1, 0x6e, 0xbf, 0x67, 0x7a, 0xdb, 0x92, - 0xf4, 0x1b, 0x28, 0xc6, 0x64, 0x97, 0x8f, 0x84, 0xb1, 0x86, 0xdf, 0x79, 0xbe, 0x95, 0xf3, 0xf3, - 0x55, 0x8e, 0xd5, 0x68, 0xa9, 0x55, 0xed, 0x2a, 0xc7, 0x48, 0xd0, 0xdb, 0xb0, 0x38, 0xf0, 0x0f, - 0xc5, 0x88, 0x93, 0xf7, 0xa1, 0x86, 0x11, 0x8a, 0xd4, 0x74, 0xf4, 0x85, 0x99, 0x93, 0x62, 0x96, - 0x4f, 0x53, 0x93, 0xd9, 0xdc, 0x98, 0x3e, 0x80, 0x9a, 0x71, 0x8c, 0x13, 0x7d, 0xc6, 0x89, 0x5b, - 0x19, 0x72, 0x1d, 0x16, 0x31, 0xd8, 0xd4, 0xab, 0xce, 0x7a, 0x45, 0x9c, 0x19, 0x36, 0xdd, 0x06, - 0xf7, 0x09, 0xeb, 0xab, 0xc1, 0xc6, 0x80, 0xad, 0x53, 0x43, 0xa9, 0x50, 0xbe, 0x8c, 0x53, 0x69, - 0xca, 0x8a, 0xdf, 0x0a, 0xdb, 0x8b, 0x13, 0x89, 0x25, 0x6d, 0x31, 0xfc, 0xa6, 0x3f, 0x3b, 0x50, - 0xdd, 0x8d, 0x87, 0x82, 0x2c, 0x43, 0xa5, 0xdf, 0x33, 0x46, 0x2a, 0xfd, 0x1e, 0x79, 0x1b, 0xed, - 0x9b, 0x52, 0xb6, 0x8a, 0x28, 0x9e, 0xb0, 0x3e, 0x43, 0xcf, 0xd7, 0xa0, 0xd5, 0x4f, 0xb7, 0xe2, - 0x38, 0x19, 0x06, 0x11, 0x97, 0x71, 0x62, 0xee, 0xbc, 0x69, 0x10, 0x67, 0x4b, 0x72, 0xa9, 0x6f, - 0xa3, 0x06, 0xd3, 0x04, 0xb9, 0x0e, 0xb5, 0x07, 0x6c, 0x6f, 0x4b, 0x39, 0x58, 0x98, 0xe7, 0xc0, - 0x72, 0xe9, 0x5d, 0x68, 0xab, 0xe8, 0x50, 0xcb, 0x36, 0xd2, 0x2a, 0x2c, 0x2a, 0x2c, 0x8f, 0xd6, - 0x50, 0x85, 0xab, 0x4a, 0xc9, 0x15, 0xfd, 0x5a, 0x5b, 0xd8, 0x3e, 0x16, 0x91, 0x2c, 0xb5, 0x22, - 0xd2, 0x68, 0xa0, 0xc5, 0x34, 0x41, 0xa8, 0xae, 0x84, 0x49, 0x79, 0xb9, 0x88, 0x48, 0xa1, 0x0c, - 0x79, 0xf4, 0x47, 0x07, 0xc0, 0x06, 0x94, 0xa5, 0xb9, 0x8a, 0x73, 0xb6, 0x0a, 0xe9, 0xda, 0x96, - 0x32, 0x63, 0xd8, 0x2e, 0xa4, 0x34, 0xce, 0x6c, 0xcb, 0x7d, 0x58, 0xb4, 0x9c, 0x3e, 0xfc, 0xcb, - 0x33, 0xad, 0xa2, 0xbd, 0x16, 0x8d, 0xb7, 0x07, 0xcd, 0x12, 0x7e, 0x46, 0xfb, 0xd9, 0x7e, 0xaa, - 0xcc, 0x9a, 0x44, 0xdc, 0x98, 0xb4, 0x5d, 0xf5, 0x10, 0x9a, 0x25, 0x78, 0xae, 0xc5, 0x2e, 0x5c, - 0x98, 0x1e, 0x70, 0x7b, 0x71, 0xcc, 0xc2, 0x34, 0x80, 0xd6, 0x56, 0x98, 0xa5, 0x52, 0x24, 0xc6, - 0x9c, 0xba, 0x6d, 0x34, 0x90, 0x1f, 0x5e, 0x01, 0xcc, 0x3f, 0x3f, 0x72, 0x0d, 0x16, 0x54, 0x19, - 0xf5, 0x9c, 0x9e, 0xae, 0xb1, 0x66, 0xd2, 0xa7, 0x50, 0xdf, 0x1c, 0xf4, 0x1f, 0x24, 0x71, 0x36, - 0x9e, 0x1b, 0xb4, 0x7d, 0x4a, 0x55, 0x4a, 0x4f, 0xa9, 0xb6, 0x7e, 0x16, 0xb8, 0xf8, 0x9c, 0xc0, - 0x37, 0x40, 0x5b, 0xbf, 0x01, 0xaa, 0x06, 0xe1, 0x6a, 0xb1, 0xaf, 0xe8, 0x1d, 0xac, 0xd6, 0xc3, - 0xeb, 0x6c, 0x32, 0x7b, 0x9b, 0xbb, 0xc5, 0x6d, 0xae, 0x8c, 0xea, 0x45, 0xf9, 0x5f, 0x1a, 0xfd, - 0xab, 0x02, 0x2b, 0x4c, 0xa4, 0xc1, 0x0b, 0xd1, 0x8f, 0x52, 0x99, 0x64, 0xbe, 0xda, 0x27, 0x4a, - 0xff, 0xab, 0xf8, 0xc0, 0x54, 0xdb, 0x65, 0x9a, 0x78, 0x95, 0x4e, 0x27, 0x37, 0xa1, 0x39, 0x3b, - 0xdc, 0xa7, 0x45, 0xcb, 0x22, 0xe4, 0x26, 0xd4, 0x06, 0x71, 0x96, 0xf8, 0x79, 0xfb, 0x96, 0x16, - 0xb0, 0x8e, 0x4c, 0xb3, 0x99, 0x15, 0x23, 0x8f, 0x81, 0xec, 0x27, 0x3c, 0x4a, 0x43, 0xae, 0x82, - 0xb5, 0xca, 0xf5, 0xd9, 0x07, 0x44, 0x49, 0x66, 0xca, 0xce, 0x1c, 0x65, 0xf2, 0x51, 0x79, 0x3e, - 0xbd, 0x1a, 0x46, 0x7d, 0x69, 0x3a, 0x6a, 0xd3, 0xf2, 0xe5, 0x39, 0xbe, 0x33, 0xd3, 0xa9, 0xde, - 0x22, 0x2a, 0x5e, 0x29, 0x14, 0xa7, 0xd8, 0x6c, 0x5a, 0x9a, 0xfe, 0xe0, 0xc0, 0x52, 0x39, 0xb2, - 0x57, 0xda, 0x0b, 0xf9, 0x81, 0x57, 0xce, 0x7f, 0xa1, 0xd8, 0x03, 0xaf, 0xce, 0x7b, 0x13, 0x2e, - 0x94, 0x5f, 0x2d, 0x19, 0x5c, 0x39, 0xa3, 0x5c, 0x6f, 0x10, 0x54, 0x07, 0x9a, 0x7b, 0x3c, 0x91, - 0x81, 0x32, 0x69, 0xae, 0xe4, 0x05, 0x56, 0x86, 0xe8, 0x11, 0x5c, 0x3d, 0xd5, 0x7c, 0x5b, 0xf1, - 0x68, 0xac, 0xba, 0xfc, 0x0d, 0x9a, 0x50, 0x2d, 0xea, 0x24, 0x31, 0xed, 0xd7, 0x60, 0x9a, 0xa0, - 0x9f, 0xc2, 0xe5, 0x81, 0x90, 0xa5, 0xd6, 0xb3, 0x33, 0xd4, 0x01, 0x77, 0x57, 0x3c, 0x3f, 0x23, - 0x41, 0xc5, 0xa2, 0x5f, 0x80, 0xf7, 0x64, 0x3c, 0xe4, 0x52, 0xbc, 0x96, 0xf6, 0x26, 0xd4, 0xf7, - 0xe3, 0x71, 0x1c, 0xc6, 0xcf, 0x4e, 0xce, 0xd9, 0x65, 0x1e, 0xd4, 0xf4, 0xad, 0xa4, 0x97, 0x63, - 0x83, 0x59, 0x92, 0x5e, 0x54, 0x63, 0xea, 0xf3, 0xd0, 0xcf, 0x42, 0x15, 0x86, 0x7a, 0x5e, 0xa7, - 0x54, 0x98, 0x41, 0xe0, 0x58, 0xb8, 0xd2, 0x45, 0x77, 0x0f, 0x01, 0x7b, 0xd1, 0x69, 0x8a, 0x7c, - 0x0c, 0xcd, 0x92, 0xb4, 0x29, 0xe0, 0xe5, 0x99, 0x79, 0xd1, 0x4c, 0x56, 0x96, 0xa4, 0xbf, 0x3a, - 0x53, 0x9a, 0xa7, 0xee, 0x7c, 0xe3, 0xf0, 0x58, 0x1f, 0x4a, 0x9d, 0x19, 0x4a, 0xe5, 0xba, 0x3d, - 0xf1, 0xc3, 0x2c, 0x55, 0x2c, 0x7d, 0xcd, 0x17, 0x80, 0xca, 0x55, 0xfd, 0x43, 0xc6, 0x99, 0x34, - 0x9b, 0xd3, 0x92, 0xea, 0x77, 0xae, 0x27, 0xf8, 0x30, 0x0c, 0x22, 0x81, 0x5d, 0xea, 0xb2, 0x9c, - 0x26, 0x37, 0xf5, 0xb6, 0xb7, 0xa3, 0xb6, 0x36, 0x37, 0x7c, 0x94, 0xd0, 0x37, 0x41, 0x4a, 0x09, - 0xb4, 0x67, 0x59, 0x9b, 0xed, 0xdf, 0x5e, 0xae, 0x3b, 0xbf, 0xbf, 0x5c, 0x77, 0xfe, 0x78, 0xb9, - 0xee, 0xfc, 0xf4, 0xe7, 0xfa, 0xff, 0x0e, 0x16, 0xf1, 0xaf, 0xfc, 0xf6, 0xdf, 0x01, 0x00, 0x00, - 0xff, 0xff, 0x8a, 0x80, 0x2a, 0x36, 0xbe, 0x0f, 0x00, 0x00, + // 1448 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdd, 0x6e, 0x1b, 0xc5, + 0x17, 0xff, 0xaf, 0xd7, 0x4e, 0xec, 0xe3, 0x38, 0x75, 0xa6, 0x6d, 0xba, 0xcd, 0x1f, 0x05, 0x33, + 0x54, 0xd4, 0x54, 0x6a, 0xa8, 0x5a, 0x24, 0x3e, 0x2b, 0xb5, 0x89, 0xdd, 0x62, 0x20, 0x69, 0x3b, + 0x4e, 0x7b, 0x8b, 0x26, 0xf6, 0x28, 0x59, 0x65, 0xbd, 0xeb, 0xee, 0xce, 0xa6, 0x4e, 0x2f, 0x10, + 0x77, 0x20, 0xf1, 0x02, 0x5c, 0x70, 0xc1, 0x7b, 0xf0, 0x02, 0x5c, 0xf2, 0x08, 0xa8, 0x3c, 0x05, + 0x77, 0x68, 0xce, 0xcc, 0xec, 0xae, 0x1d, 0x87, 0x84, 0x96, 0xbb, 0x39, 0x5f, 0x73, 0x7e, 0xe7, + 0x73, 0xc7, 0x86, 0xc6, 0x38, 0xf6, 0x8f, 0xb8, 0x14, 0x1b, 0xe3, 0x38, 0x92, 0x11, 0xa9, 0xfa, + 0xa1, 0x14, 0x71, 0xc8, 0x83, 0xb5, 0xa5, 0x71, 0xba, 0x17, 0xf8, 0x03, 0xcd, 0xa7, 0x0f, 0xa1, + 0xd6, 0x0b, 0x87, 0x62, 0xb2, 0x2d, 0x24, 0x27, 0x04, 0xca, 0x5f, 0x89, 0xe3, 0xc4, 0x73, 0x5b, + 0x4e, 0xbb, 0xca, 0xf0, 0x4c, 0xde, 0x83, 0xe5, 0xdd, 0x98, 0x0f, 0x0e, 0xbb, 0x13, 0x3f, 0x91, + 0x22, 0x1c, 0x08, 0xaf, 0x8c, 0xd2, 0x19, 0x2e, 0xfd, 0xc5, 0x85, 0xa5, 0x07, 0xbe, 0x08, 0x86, + 0x8f, 0xc6, 0xd2, 0x8f, 0xc2, 0x44, 0x5d, 0xb6, 0x7b, 0x3c, 0x16, 0x5e, 0xb5, 0xe5, 0xb4, 0x6b, + 0x0c, 0xcf, 0xe4, 0x2d, 0xa8, 0x6d, 0xf1, 0xc1, 0x81, 0x40, 0x81, 0x8b, 0x82, 0x9c, 0x91, 0x49, + 0xfb, 0xfe, 0x4b, 0xed, 0xa5, 0xc1, 0x72, 0x06, 0x69, 0x41, 0x7d, 0xd7, 0x1f, 0x89, 0x27, 0x29, + 0x0f, 0x65, 0x3a, 0xf2, 0x2a, 0x68, 0x5d, 0x64, 0x91, 0x55, 0x58, 0x78, 0x14, 0x0c, 0xb7, 0xfd, + 0xd0, 0xab, 0xb5, 0x9c, 0xb6, 0xcb, 0x0c, 0x65, 0xf9, 0x7c, 0xe2, 0x41, 0xce, 0xe7, 0x93, 0x2c, + 0xdc, 0xfa, 0x74, 0xb8, 0x3b, 0x51, 0x5f, 0xf2, 0x70, 0xc8, 0xe3, 0xe1, 0x33, 0x5f, 0xbc, 0xf0, + 0x96, 0x74, 0xb8, 0xd3, 0x5c, 0x65, 0xbb, 0xc9, 0x13, 0xe1, 0x35, 0xf0, 0x46, 0x3c, 0x93, 0x35, + 0xa8, 0x6e, 0xfa, 0xb2, 0x23, 0xc6, 0xf2, 0xc0, 0x5b, 0x6e, 0x39, 0xed, 0x32, 0xcb, 0x68, 0x72, + 0x09, 0x2a, 0xfd, 0x01, 0x0f, 0x84, 0x77, 0x01, 0x0d, 0x34, 0x41, 0x28, 0x2c, 0x3d, 0x88, 0x62, + 0xe1, 0xef, 0x87, 0x58, 0x04, 0xaf, 0x89, 0x41, 0x4d, 0xf1, 0xc8, 0xbb, 0xe0, 0xaa, 0x90, 0x56, + 0x5a, 0x4e, 0xbb, 0x7e, 0x7b, 0x65, 0xc3, 0xd6, 0x71, 0xa3, 0x23, 0x06, 0xfe, 0x88, 0x07, 0x4c, + 0x49, 0x51, 0x89, 0x4f, 0x3c, 0x72, 0xba, 0x12, 0x9f, 0x50, 0x0a, 0xcb, 0xbd, 0xd1, 0x38, 0x8a, + 0x25, 0x13, 0xc9, 0x38, 0x0a, 0x13, 0x41, 0x9a, 0xe0, 0x76, 0xe3, 0xd8, 0x73, 0xd0, 0xad, 0x3a, + 0xd2, 0x6f, 0xa1, 0xb9, 0x19, 0x44, 0x83, 0xc3, 0x0e, 0x97, 0x9c, 0x89, 0xe7, 0xa9, 0x48, 0xa4, + 0xc2, 0xae, 0xe1, 0x69, 0x3d, 0x4d, 0x28, 0x2e, 0xd6, 0xdb, 0x2b, 0x69, 0x2e, 0x12, 0x2a, 0x2f, + 0x98, 0x35, 0x5d, 0x1e, 0x3c, 0x63, 0xec, 0x07, 0x3c, 0x1e, 0x62, 0x4d, 0xcb, 0x4c, 0x13, 0x8a, + 0x8b, 0x9e, 0xb0, 0x0f, 0xca, 0x4c, 0x13, 0xb4, 0x07, 0x2b, 0x05, 0xff, 0x06, 0xe6, 0x2a, 0x2c, + 0xb0, 0xe8, 0x45, 0xaf, 0x93, 0x78, 0x4e, 0xcb, 0x6d, 0x97, 0x99, 0xa1, 0xb0, 0x61, 0xa2, 0x20, + 0x1d, 0x85, 0x4a, 0x54, 0x42, 0x51, 0xce, 0xa0, 0x57, 0xa1, 0x82, 0xdd, 0xa3, 0xa2, 0xcc, 0x6d, + 0xd5, 0x91, 0x7e, 0xef, 0x40, 0x6d, 0x9b, 0x4f, 0x10, 0x48, 0x42, 0xee, 0x42, 0xd5, 0xd6, 0x16, + 0x95, 0xea, 0xb7, 0xdf, 0xc9, 0x33, 0x98, 0xa9, 0x6d, 0x58, 0x9d, 0x6e, 0x28, 0xe3, 0x63, 0x96, + 0x99, 0xac, 0x7d, 0x06, 0x8d, 0x29, 0x91, 0xf2, 0x77, 0x28, 0x8e, 0x6d, 0x56, 0x0f, 0xc5, 0xb1, + 0x8a, 0xf5, 0x88, 0x07, 0xa9, 0xc0, 0x5c, 0x95, 0x99, 0x26, 0x3e, 0x2d, 0x7d, 0xec, 0xd0, 0x67, + 0x40, 0xb6, 0x62, 0xc1, 0xa5, 0x40, 0x27, 0xdb, 0x22, 0x49, 0xf8, 0xbe, 0x38, 0x2b, 0xe3, 0x6e, + 0x31, 0xe3, 0x59, 0x76, 0x4b, 0x85, 0xec, 0xd2, 0x1b, 0x40, 0x3a, 0x22, 0x10, 0x52, 0x98, 0xe9, + 0xfe, 0x87, 0x7b, 0xe9, 0xbe, 0xc5, 0x70, 0xb6, 0x2e, 0xb9, 0x0e, 0x65, 0xb5, 0x2a, 0xd0, 0x59, + 0xfd, 0xf6, 0xc5, 0x3c, 0x4f, 0xd9, 0x16, 0x61, 0x65, 0xbb, 0x4b, 0xba, 0xbb, 0x7c, 0x1f, 0xb1, + 0xba, 0x0c, 0xcf, 0xf4, 0x3b, 0xc7, 0x7a, 0x42, 0xe8, 0xe7, 0x8c, 0x76, 0xaa, 0xbf, 0x6e, 0x18, + 0xff, 0x2e, 0xfa, 0x5f, 0xcd, 0xfd, 0x17, 0x77, 0xcf, 0x0c, 0x84, 0x72, 0x01, 0xc2, 0x3d, 0x9b, + 0x97, 0xd7, 0x45, 0x40, 0x07, 0xf0, 0x7f, 0x7d, 0xc3, 0xfd, 0x23, 0xee, 0x07, 0x7c, 0x2f, 0xf8, + 0x57, 0xa5, 0x9b, 0x0a, 0xc6, 0x83, 0x45, 0xb4, 0xed, 0x75, 0xcc, 0x10, 0x58, 0x92, 0x3e, 0x87, + 0x7c, 0x9e, 0x76, 0xf8, 0x48, 0x98, 0xdb, 0xf0, 0x9c, 0xe5, 0xa0, 0x74, 0x8e, 0x1c, 0x5c, 0x82, + 0x8a, 0x9a, 0x41, 0xb5, 0xd3, 0x5d, 0xe5, 0x18, 0x89, 0xb9, 0x99, 0xb9, 0x03, 0x0b, 0xfd, 0xc1, + 0x81, 0x18, 0x71, 0xf2, 0x3e, 0x2c, 0x22, 0x6a, 0x91, 0x98, 0x71, 0xb8, 0x30, 0x53, 0x66, 0x66, + 0xe5, 0x6a, 0x90, 0x4c, 0x84, 0xf3, 0x80, 0x5a, 0x37, 0xa5, 0xdc, 0x0d, 0xb9, 0x09, 0x8b, 0x06, + 0x21, 0xee, 0x88, 0x53, 0x7a, 0xc8, 0xea, 0x90, 0xeb, 0xb0, 0x80, 0x51, 0x25, 0x5e, 0x79, 0x16, + 0x0a, 0xf2, 0x99, 0x11, 0xd3, 0x2e, 0xb8, 0x4f, 0x59, 0x4f, 0xad, 0x0a, 0x8c, 0xc2, 0x02, 0x31, + 0x94, 0x82, 0xf2, 0x45, 0x94, 0x48, 0x93, 0x7f, 0x3c, 0x2b, 0xde, 0xe3, 0x28, 0x96, 0x98, 0xfb, + 0x06, 0xc3, 0x33, 0xfd, 0xd9, 0x81, 0xf2, 0x4e, 0x34, 0x14, 0x64, 0x19, 0x4a, 0xbd, 0x8e, 0xb9, + 0xa4, 0xd4, 0xeb, 0x90, 0xb7, 0xf1, 0x7e, 0x93, 0xf3, 0x46, 0x8e, 0xe2, 0x29, 0xeb, 0x31, 0xf4, + 0x7c, 0x0d, 0x1a, 0xbd, 0x64, 0x2b, 0x8a, 0xe2, 0xa1, 0x1f, 0x72, 0x19, 0xc5, 0xe6, 0x2b, 0x3a, + 0xcd, 0xc4, 0x69, 0x95, 0x5c, 0xea, 0xef, 0x5b, 0x8d, 0x69, 0x82, 0x5c, 0x87, 0xc5, 0x87, 0xec, + 0xf1, 0x96, 0x72, 0x50, 0x99, 0xe7, 0xc0, 0x4a, 0xe9, 0x3d, 0x68, 0x2a, 0x74, 0x68, 0x65, 0x3b, + 0x6e, 0x15, 0x16, 0x14, 0x2f, 0x43, 0x6b, 0xa8, 0xdc, 0x55, 0xa9, 0xe0, 0x8a, 0x7e, 0xad, 0x6f, + 0xe8, 0x1e, 0x89, 0x50, 0x16, 0x7a, 0x16, 0x69, 0xbc, 0xa0, 0xc1, 0x34, 0x41, 0xa8, 0xce, 0x84, + 0x09, 0x79, 0x39, 0x47, 0xa4, 0xb8, 0x0c, 0x65, 0xf4, 0x47, 0x07, 0xc0, 0x02, 0x4a, 0x93, 0xcc, + 0xc4, 0x39, 0xdd, 0x84, 0xb4, 0x6d, 0x9f, 0x99, 0x19, 0x6e, 0xe6, 0x5a, 0x9a, 0xcf, 0x6c, 0x1f, + 0x7e, 0x90, 0xf7, 0xa1, 0x2e, 0xfe, 0xe5, 0x99, 0x56, 0xd1, 0x5e, 0xf3, 0x6e, 0x1c, 0x42, 0xbd, + 0xc0, 0x9f, 0xdb, 0x92, 0x37, 0xb3, 0x7e, 0x2a, 0xcd, 0x5e, 0x89, 0x7c, 0x73, 0xa5, 0x51, 0x9a, + 0xbb, 0xc5, 0xbe, 0x81, 0x7a, 0x41, 0x75, 0xae, 0x97, 0x36, 0x5c, 0x98, 0xde, 0x0e, 0xf6, 0xf3, + 0x34, 0xcb, 0x9e, 0xeb, 0xc0, 0x87, 0xc6, 0x56, 0x90, 0x26, 0x52, 0xc4, 0xc6, 0x85, 0xfa, 0xce, + 0x69, 0x46, 0x56, 0xe4, 0x9c, 0x31, 0xbf, 0xce, 0xe4, 0x1a, 0x54, 0x54, 0xba, 0xf5, 0xe0, 0x9f, + 0xac, 0x85, 0x16, 0xd2, 0x67, 0x50, 0xdd, 0xec, 0xf7, 0x1e, 0xc6, 0x51, 0x3a, 0x3e, 0x6d, 0x82, + 0xf1, 0xad, 0x56, 0x2a, 0x3c, 0xe2, 0x9a, 0xfa, 0x41, 0xa2, 0x11, 0xe3, 0xeb, 0xa3, 0xa9, 0x5f, + 0x1f, 0x65, 0xc3, 0xe1, 0x13, 0xda, 0x87, 0x15, 0xbd, 0xe8, 0xd5, 0xbe, 0x79, 0x9d, 0xd5, 0x68, + 0xdf, 0x11, 0x6e, 0xfe, 0x8e, 0x50, 0x97, 0xea, 0xcd, 0xfb, 0x5f, 0x5e, 0xfa, 0x57, 0x09, 0x56, + 0x98, 0x48, 0xfc, 0x97, 0xa2, 0x17, 0x26, 0x32, 0x4e, 0x07, 0x6a, 0xef, 0x28, 0xfb, 0x2f, 0xa3, + 0x3d, 0x93, 0x6d, 0x97, 0x69, 0xe2, 0x3c, 0x13, 0x41, 0x6e, 0x41, 0x7d, 0x76, 0x09, 0x9c, 0x54, + 0x2d, 0xaa, 0x90, 0x5b, 0xb0, 0xd8, 0x8f, 0xd2, 0x78, 0x90, 0xb5, 0x79, 0x61, 0xa3, 0x6b, 0x64, + 0x5a, 0xcc, 0xac, 0x1a, 0x79, 0x02, 0x64, 0x37, 0xe6, 0x61, 0x12, 0x70, 0x05, 0xd6, 0x1a, 0x57, + 0x67, 0x9f, 0x2e, 0x05, 0x9d, 0xa9, 0x7b, 0xe6, 0x18, 0x93, 0x0f, 0x8b, 0x73, 0xec, 0x2d, 0x22, + 0xea, 0x4b, 0xd3, 0xa8, 0xcd, 0x68, 0x14, 0xe7, 0xfd, 0xee, 0x4c, 0xa7, 0x7a, 0x0b, 0x68, 0x78, + 0x25, 0x37, 0x9c, 0x12, 0xb3, 0x69, 0x6d, 0xfa, 0x83, 0x03, 0x4b, 0x45, 0x64, 0xe7, 0xda, 0x1f, + 0x59, 0xc1, 0x4b, 0x67, 0xbf, 0x8d, 0x6c, 0xc1, 0xcb, 0xf3, 0x5e, 0xa3, 0x95, 0xe2, 0x7b, 0x29, + 0x85, 0x2b, 0xa7, 0xa4, 0xeb, 0x0d, 0x40, 0xb5, 0xa0, 0xfe, 0x98, 0xc7, 0xd2, 0x57, 0x57, 0x9a, + 0x6f, 0x7c, 0x85, 0x15, 0x59, 0xf4, 0x10, 0xae, 0x9e, 0x68, 0xbe, 0xad, 0x68, 0x34, 0x56, 0x5d, + 0xfe, 0x06, 0x4d, 0xa8, 0x16, 0x7a, 0x1c, 0x9b, 0xf6, 0xab, 0x31, 0x4d, 0xd0, 0x4f, 0xe0, 0x72, + 0x5f, 0xc8, 0x42, 0xeb, 0xd9, 0x19, 0x6a, 0x81, 0xbb, 0x23, 0x5e, 0x9c, 0x12, 0xa0, 0x12, 0xd1, + 0xcf, 0xc1, 0x7b, 0x3a, 0x1e, 0x72, 0x29, 0x5e, 0xcb, 0x7a, 0x13, 0xaa, 0xbb, 0xd1, 0x38, 0x0a, + 0xa2, 0xfd, 0xe3, 0x33, 0x76, 0x99, 0x07, 0x8b, 0xfa, 0xeb, 0xa5, 0x17, 0x66, 0x8d, 0x59, 0x92, + 0x5e, 0x54, 0x63, 0x3a, 0xe0, 0xc1, 0x20, 0x0d, 0x14, 0x0c, 0xf5, 0xb0, 0x4f, 0xa8, 0x30, 0x83, + 0xc0, 0x31, 0x71, 0x85, 0x0f, 0xe2, 0x7d, 0x64, 0xd8, 0x0f, 0xa2, 0xa6, 0xc8, 0x47, 0x50, 0x2f, + 0x68, 0x9b, 0x04, 0x5e, 0x9e, 0x99, 0x17, 0x2d, 0x64, 0x45, 0x4d, 0xfa, 0xab, 0x33, 0x65, 0x79, + 0xe2, 0x6d, 0x60, 0x1c, 0x1e, 0xe9, 0xa2, 0x54, 0x99, 0xa1, 0x54, 0xac, 0xdd, 0xc9, 0x20, 0x48, + 0x13, 0x25, 0xd2, 0xcf, 0x81, 0x9c, 0xa1, 0x62, 0x55, 0xbf, 0x5e, 0xa3, 0x54, 0x9a, 0xcd, 0x69, + 0x49, 0xf5, 0x43, 0xb2, 0x23, 0xf8, 0x30, 0xf0, 0x43, 0x81, 0x5d, 0xea, 0xb2, 0x8c, 0x26, 0xb7, + 0xf4, 0xb6, 0xb7, 0xa3, 0xb6, 0x36, 0x17, 0x3e, 0x6a, 0xe8, 0x2f, 0x41, 0x42, 0x09, 0x34, 0x67, + 0x45, 0x9b, 0xcd, 0xdf, 0x5e, 0xad, 0x3b, 0xbf, 0xbf, 0x5a, 0x77, 0xfe, 0x78, 0xb5, 0xee, 0xfc, + 0xf4, 0xe7, 0xfa, 0xff, 0xf6, 0x16, 0xf0, 0xff, 0x80, 0x3b, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, + 0x4f, 0xdc, 0x5a, 0x3d, 0x38, 0x10, 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -3043,6 +3093,11 @@ func (m *CreateIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ETag != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + i-- + dAtA[i] = 0x18 + } if m.Meta != nil { { size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) @@ -3089,6 +3144,11 @@ func (m *CreateFieldMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ETag != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + i-- + dAtA[i] = 0x20 + } if m.Meta != nil { { size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) @@ -3229,6 +3289,11 @@ func (m *Field) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ETag != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + i-- + dAtA[i] = 0x20 + } if len(m.Views) > 0 { for iNdEx := len(m.Views) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Views[iNdEx]) @@ -3351,6 +3416,11 @@ func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x22 } } + if m.ETag != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + i-- + dAtA[i] = 0x10 + } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) @@ -3656,6 +3726,11 @@ func (m *IndexStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ETag != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + i-- + dAtA[i] = 0x18 + } if len(m.Fields) > 0 { for iNdEx := len(m.Fields) - 1; iNdEx >= 0; iNdEx-- { { @@ -3704,6 +3779,11 @@ func (m *FieldStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ETag != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + i-- + dAtA[i] = 0x18 + } if len(m.AvailableShards) > 0 { dAtA19 := make([]byte, len(m.AvailableShards)*10) var j18 int @@ -4759,6 +4839,9 @@ func (m *CreateIndexMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } + if m.ETag != 0 { + n += 1 + sovPrivate(uint64(m.ETag)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4783,6 +4866,9 @@ func (m *CreateFieldMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } + if m.ETag != 0 { + n += 1 + sovPrivate(uint64(m.ETag)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4852,6 +4938,9 @@ func (m *Field) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if m.ETag != 0 { + n += 1 + sovPrivate(uint64(m.ETag)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4886,6 +4975,9 @@ func (m *Index) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.ETag != 0 { + n += 1 + sovPrivate(uint64(m.ETag)) + } if len(m.Fields) > 0 { for _, e := range m.Fields { l = e.Size() @@ -5037,6 +5129,9 @@ func (m *IndexStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if m.ETag != 0 { + n += 1 + sovPrivate(uint64(m.ETag)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5060,6 +5155,9 @@ func (m *FieldStatus) Size() (n int) { } n += 1 + sovPrivate(uint64(l)) + l } + if m.ETag != 0 { + n += 1 + sovPrivate(uint64(m.ETag)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7021,6 +7119,25 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + } + m.ETag = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ETag |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7175,6 +7292,25 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + } + m.ETag = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ETag |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7584,6 +7720,25 @@ func (m *Field) Unmarshal(dAtA []byte) error { } m.Views = append(m.Views, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + } + m.ETag = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ETag |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7758,6 +7913,25 @@ func (m *Index) Unmarshal(dAtA []byte) error { } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + } + m.ETag = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ETag |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) @@ -8682,6 +8856,25 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + } + m.ETag = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ETag |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8844,6 +9037,25 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field AvailableShards", wireType) } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + } + m.ETag = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ETag |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) diff --git a/internal/private.proto b/internal/private.proto index 849fff04a..502d3b7bd 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -64,12 +64,14 @@ message DeleteIndexMessage { message CreateIndexMessage { string Index = 1; IndexMeta Meta = 2; + int64 ETag = 3; } message CreateFieldMessage { string Index = 1; string Field = 2; FieldOptions Meta = 3; + int64 ETag = 4; } message DeleteFieldMessage { @@ -87,6 +89,7 @@ message Field { string Name = 1; FieldOptions Meta = 2; repeated string Views = 3; + int64 ETag = 4; } message Schema { @@ -95,6 +98,7 @@ message Schema { message Index { string Name = 1; + int64 ETag = 2; IndexMeta Options = 5; repeated Field Fields = 4; } @@ -132,11 +136,13 @@ message NodeStatus { message IndexStatus { string Name = 1; repeated FieldStatus Fields = 2; + int64 ETag = 3; } message FieldStatus { string Name = 1; repeated uint64 AvailableShards = 2; + int64 ETag = 3; } message ClusterStatus { diff --git a/pilosa.go b/pilosa.go index d7eb2f3c8..e49af5406 100644 --- a/pilosa.go +++ b/pilosa.go @@ -17,6 +17,7 @@ package pilosa import ( "encoding/json" "regexp" + "time" "github.com/pkg/errors" ) @@ -191,6 +192,10 @@ func stringSlicesAreEqual(a, b []string) bool { return true } +func newETag() int64 { + return time.Now().UTC().UnixNano() +} + // AddressWithDefaults converts addr into a valid address, // using defaults when necessary. func AddressWithDefaults(addr string) (*URI, error) { diff --git a/server.go b/server.go index fd07ec0b2..586d14e73 100644 --- a/server.go +++ b/server.go @@ -356,14 +356,11 @@ func NewServer(opts ...ServerOption) (*Server, error) { } s.executor = newExecutor(executorOpts...) - // s.holder.translateFile.logger = s.logger - path, err := expandDirName(s.dataDir) if err != nil { return nil, err } s.holder.Path = path - // s.holder.translateFile.Path = filepath.Join(path, ".keys") s.holder.Logger = s.logger s.holder.Stats.SetLogger(s.logger) @@ -716,10 +713,13 @@ func (s *Server) receiveMessage(m Message) error { } case *CreateIndexMessage: opt := obj.Meta - _, err := s.holder.CreateIndex(obj.Index, *opt) + idx, err := s.holder.CreateIndex(obj.Index, *opt) if err != nil { return err } + idx.mu.Lock() + idx.etag = obj.ETag + idx.mu.Unlock() case *DeleteIndexMessage: if err := s.holder.DeleteIndex(obj.Index); err != nil { return err @@ -730,10 +730,13 @@ func (s *Server) receiveMessage(m Message) error { return fmt.Errorf("local index not found: %s", obj.Index) } opt := obj.Meta - _, err := idx.createFieldIfNotExists(obj.Field, opt) + fld, err := idx.createFieldIfNotExists(obj.Field, opt) if err != nil { return err } + fld.mu.Lock() + fld.etag = obj.ETag + fld.mu.Unlock() case *DeleteFieldMessage: idx := s.holder.Index(obj.Index) if err := idx.DeleteField(obj.Field); err != nil { diff --git a/server/handler_test.go b/server/handler_test.go index b01b15c59..9cd25f50e 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -203,6 +203,53 @@ func TestHandler_Endpoints(t *testing.T) { } }) + t.Run("Import", func(t *testing.T) { + indexInfo := cmd.API.Schema(context.Background()) + err := cmd.API.ApplySchema(context.Background(), &pilosa.Schema{Indexes: indexInfo}, false) + if err != nil { + t.Fatalf("applying schema: %v", err) + } + + idx := indexInfo[0] + fld := indexInfo[0].Fields[0] + msg := pilosa.ImportRequest{ + Index: idx.Name, + Field: fld.Name, + Shard: 0, + } + ser := proto.Serializer{} + data, err := ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + path := fmt.Sprintf("/index/%s/field/%s/import", idx.Name, fld.Name) + etag := fmt.Sprintf("%d, %d", idx.ETag, fld.ETag) + + httpReq := test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data)) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + httpReq.Header.Set("If-Match", etag) + + w := httptest.NewRecorder() + h.ServeHTTP(w, httpReq) + if w.Body.String() != "" { + t.Fatalf(w.Body.String()) + } + + etag = "invalid-index-etag, invalid-field-etag" + + httpReq = test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data)) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + httpReq.Header.Set("If-Match", etag) + + h.ServeHTTP(w, httpReq) + + if strings.TrimSpace(w.Body.String()) != "Precondition Failed" { + t.Fatal("expected: Precondition Failed, got:" + w.Body.String()) + } + }) + t.Run("ImportRoaring", func(t *testing.T) { w := httptest.NewRecorder() roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") @@ -217,10 +264,23 @@ func TestHandler_Endpoints(t *testing.T) { if err != nil { t.Fatal(err) } + idx, err := cmd.API.Index(context.Background(), "i0") + if err != nil { + t.Fatal(err) + } + fld, err := cmd.API.Field(context.Background(), "i0", "f1") + if err != nil { + t.Fatal(err) + } + httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(data)) httpReq.Header.Set("Content-Type", "application/x-protobuf") httpReq.Header.Set("Accept", "application/x-protobuf") + httpReq.Header.Set("If-Match", fmt.Sprintf("%d, %d", idx.ETag(), fld.ETag())) h.ServeHTTP(w, httpReq) + if w.Body.String() != "" { + t.Fatalf("Unexpected response body: %s", w.Body.String()) + } resp, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i0", Query: "TopN(f1)"}) if err != nil { t.Fatalf("querying: %v", err) From ba7f039dd1a0369b3b58c407bfd2f5b09cbc7dcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Tue, 26 May 2020 00:33:52 +0200 Subject: [PATCH 15/45] Rename etag to createdAt --- api.go | 64 ++++--- api_test.go | 22 +-- cluster.go | 26 +-- encoding/proto/proto.go | 102 ++++++----- field.go | 24 +-- gossip/gossip.go | 4 +- handler.go | 64 +++++-- holder.go | 32 ++-- http/handler.go | 83 +++------ index.go | 32 ++-- internal/private.pb.go | 304 ++++++++++++++++----------------- internal/private.proto | 12 +- internal/public.pb.go | 365 +++++++++++++++++++++++++++++++--------- internal/public.proto | 8 + pilosa.go | 14 +- server.go | 4 +- server/handler_test.go | 183 ++++++++++++++------ 17 files changed, 838 insertions(+), 505 deletions(-) diff --git a/api.go b/api.go index 43dba98f0..0a29ad30a 100644 --- a/api.go +++ b/api.go @@ -182,15 +182,15 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index return nil, errors.Wrap(err, "creating index") } index.mu.Lock() - index.etag = newETag() + index.createdAt = timestamp() index.mu.Unlock() // Send the create index message to all nodes. err = api.server.SendSync( &CreateIndexMessage{ - Index: indexName, - ETag: index.ETag(), - Meta: &options, + Index: indexName, + CreatedAt: index.CreatedAt(), + Meta: &options, }) if err != nil { return nil, errors.Wrap(err, "sending CreateIndex message") @@ -275,15 +275,15 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str return nil, errors.Wrap(err, "creating field") } field.mu.Lock() - field.etag = newETag() + field.createdAt = timestamp() field.mu.Unlock() // Send the create field message to all nodes. err = api.server.SendSync(&CreateFieldMessage{ - Index: indexName, - Field: fieldName, - ETag: field.ETag(), - Meta: &fo, + Index: indexName, + Field: fieldName, + CreatedAt: field.CreatedAt(), + Meta: &fo, }) if err != nil { api.server.logger.Printf("problem sending CreateField message: %s", err) @@ -421,14 +421,17 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, return errors.Wrap(err, "validating api method") } - nodes := api.cluster.shardNodes(indexName, shard) - - field := api.holder.Field(indexName, fieldName) - if field == nil { + index, field, err := api.indexField(indexName, fieldName, shard) + if index == nil || field == nil { return newNotFoundError(ErrFieldNotFound) } - errCh := make(chan error, len(nodes)) + if err = req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { + return newPreconditionFailedError(err) + } + + nodes := api.cluster.shardNodes(indexName, shard) + errCh := make(chan error, len(nodes)) for _, node := range nodes { node := node if node.ID == api.server.nodeID { @@ -811,14 +814,14 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { return errors.Wrap(err, "validating api method") } - // set etags for indexes and fields (if empty), and then apply schema. + // set CreatedAt for indexes and fields (if empty), and then apply schema. for _, index := range s.Indexes { - if index.ETag == 0 { - index.ETag = newETag() + if index.CreatedAt == 0 { + index.CreatedAt = timestamp() } for _, field := range index.Fields { - if field.ETag == 0 { - field.ETag = newETag() + if field.CreatedAt == 0 { + field.CreatedAt = timestamp() } } } @@ -1023,16 +1026,20 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp return errors.Wrap(err, "validating api method") } + index, field, err := api.indexField(req.Index, req.Field, req.Shard) + if err != nil { + return errors.Wrap(err, "getting index and field") + } + + if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { + return errors.Wrap(err, "validating import value request") + } + // Set up import options. options, err := setUpImportOptions(opts...) if err != nil { return errors.Wrap(err, "setting up import options") } - - index, field, err := api.indexField(req.Index, req.Field, req.Shard) - if err != nil { - return errors.Wrap(err, "getting index and field") - } span.LogKV( "index", req.Index, "field", req.Field) @@ -1139,7 +1146,12 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . return errors.Wrap(err, "validating api method") } - if err := req.Validate(); err != nil { + index, field, err := api.indexField(req.Index, req.Field, req.Shard) + if err != nil { + return errors.Wrap(err, "getting index and field") + } + + if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { return errors.Wrap(err, "validating import value request") } @@ -1149,7 +1161,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . return errors.Wrap(err, "setting up import options") } - index, field, err := api.indexField(req.Index, req.Field, req.Shard) + index, field, err = api.indexField(req.Index, req.Field, req.Shard) if err != nil { return errors.Wrap(err, "getting index and field") } diff --git a/api_test.go b/api_test.go index c11ac4f8a..e8b4f28f5 100644 --- a/api_test.go +++ b/api_test.go @@ -193,16 +193,16 @@ func TestAPI_Import(t *testing.T) { if err != nil { t.Fatalf("creating index: %v", err) } - if index.ETag() == 0 { - t.Fatal("index etag is empty") + if index.CreatedAt() == 0 { + t.Fatal("index createdAt is empty") } field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) if err != nil { t.Fatalf("creating field: %v", err) } - if field.ETag() == 0 { - t.Fatal("field etag is empty") + if field.CreatedAt() == 0 { + t.Fatal("field createdAt is empty") } rowID := uint64(1) @@ -222,12 +222,14 @@ func TestAPI_Import(t *testing.T) { // Import data with keys to the coordinator (node0) and verify that it gets // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) req := &pilosa.ImportRequest{ - Index: indexName, - Field: fieldName, - Shard: 0, - RowIDs: rowIDs, - ColumnKeys: colKeys, - Timestamps: timestamps, + Index: indexName, + IndexCreatedAt: index.CreatedAt(), + Field: fieldName, + FieldCreatedAt: field.CreatedAt(), + Shard: 0, + RowIDs: rowIDs, + ColumnKeys: colKeys, + Timestamps: timestamps, } if err := m0.API.Import(ctx, req); err != nil { t.Fatal(err) diff --git a/cluster.go b/cluster.go index 72ce09b17..d283593a1 100644 --- a/cluster.go +++ b/cluster.go @@ -2144,7 +2144,7 @@ func (c *cluster) nodeStatus() *NodeStatus { } var availableShards *roaring.Bitmap for _, idx := range ns.Schema.Indexes { - is := &IndexStatus{Name: idx.Name, ETag: idx.ETag} + is := &IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt} for _, f := range idx.Fields { if field := c.holder.Field(idx.Name, f.Name); field != nil { availableShards = field.AvailableShards() @@ -2153,7 +2153,7 @@ func (c *cluster) nodeStatus() *NodeStatus { } is.Fields = append(is.Fields, &FieldStatus{ Name: f.Name, - ETag: f.ETag, + CreatedAt: f.CreatedAt, AvailableShards: availableShards, }) } @@ -2528,9 +2528,9 @@ type CreateShardMessage struct { // CreateIndexMessage is an internal message indicating index creation. type CreateIndexMessage struct { - Index string - ETag int64 - Meta *IndexOptions + Index string + CreatedAt int64 + Meta *IndexOptions } // DeleteIndexMessage is an internal message indicating index deletion. @@ -2540,10 +2540,10 @@ type DeleteIndexMessage struct { // CreateFieldMessage is an internal message indicating field creation. type CreateFieldMessage struct { - Index string - Field string - ETag int64 - Meta *FieldOptions + Index string + Field string + CreatedAt int64 + Meta *FieldOptions } // DeleteFieldMessage is an internal message indicating field deletion. @@ -2606,15 +2606,15 @@ type NodeStatus struct { // IndexStatus is an internal message representing the contents of an index. type IndexStatus struct { - Name string - ETag int64 - Fields []*FieldStatus + Name string + CreatedAt int64 + Fields []*FieldStatus } // FieldStatus is an internal message representing the contents of a field. type FieldStatus struct { Name string - ETag int64 + CreatedAt int64 AvailableShards *roaring.Bitmap } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 6eb389302..7d647ded0 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -403,27 +403,31 @@ func (s Serializer) encodeImportResponse(m *pilosa.ImportResponse) *internal.Imp func (s Serializer) encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest { return &internal.ImportRequest{ - Index: m.Index, - Field: m.Field, - Shard: m.Shard, - RowIDs: m.RowIDs, - ColumnIDs: m.ColumnIDs, - RowKeys: m.RowKeys, - ColumnKeys: m.ColumnKeys, - Timestamps: m.Timestamps, + Index: m.Index, + Field: m.Field, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Shard: m.Shard, + RowIDs: m.RowIDs, + ColumnIDs: m.ColumnIDs, + RowKeys: m.RowKeys, + ColumnKeys: m.ColumnKeys, + Timestamps: m.Timestamps, } } func (s Serializer) encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest { return &internal.ImportValueRequest{ - Index: m.Index, - Field: m.Field, - Shard: m.Shard, - ColumnIDs: m.ColumnIDs, - ColumnKeys: m.ColumnKeys, - Values: m.Values, - FloatValues: m.FloatValues, - StringValues: m.StringValues, + Index: m.Index, + Field: m.Field, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Shard: m.Shard, + ColumnIDs: m.ColumnIDs, + ColumnKeys: m.ColumnKeys, + Values: m.Values, + FloatValues: m.FloatValues, + StringValues: m.StringValues, } } @@ -438,10 +442,12 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) * i++ } return &internal.ImportRoaringRequest{ - Clear: m.Clear, - Action: m.Action, - Block: uint64(m.Block), - Views: views, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Clear: m.Clear, + Action: m.Action, + Block: uint64(m.Block), + Views: views, } } @@ -593,10 +599,10 @@ func (s Serializer) encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index func (s Serializer) encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index { return &internal.Index{ - Name: idx.Name, - ETag: idx.ETag, - Options: s.encodeIndexMeta(&idx.Options), - Fields: s.encodeFieldInfos(idx.Fields), + Name: idx.Name, + CreatedAt: idx.CreatedAt, + Options: s.encodeIndexMeta(&idx.Options), + Fields: s.encodeFieldInfos(idx.Fields), } } @@ -610,10 +616,10 @@ func (s Serializer) encodeFieldInfos(fs []*pilosa.FieldInfo) []*internal.Field { func (s Serializer) encodeFieldInfo(f *pilosa.FieldInfo) *internal.Field { ifield := &internal.Field{ - Name: f.Name, - ETag: f.ETag, - Meta: s.encodeFieldOptions(&f.Options), - Views: make([]string, 0, len(f.Views)), + Name: f.Name, + CreatedAt: f.CreatedAt, + Meta: s.encodeFieldOptions(&f.Options), + Views: make([]string, 0, len(f.Views)), } for _, viewinfo := range f.Views { @@ -687,9 +693,9 @@ func (s Serializer) encodeCreateShardMessage(m *pilosa.CreateShardMessage) *inte func (s Serializer) encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateIndexMessage { return &internal.CreateIndexMessage{ - Index: m.Index, - ETag: m.ETag, - Meta: s.encodeIndexMeta(m.Meta), + Index: m.Index, + CreatedAt: m.CreatedAt, + Meta: s.encodeIndexMeta(m.Meta), } } @@ -708,10 +714,10 @@ func (s Serializer) encodeDeleteIndexMessage(m *pilosa.DeleteIndexMessage) *inte func (s Serializer) encodeCreateFieldMessage(m *pilosa.CreateFieldMessage) *internal.CreateFieldMessage { return &internal.CreateFieldMessage{ - Index: m.Index, - Field: m.Field, - ETag: m.ETag, - Meta: s.encodeFieldOptions(m.Meta), + Index: m.Index, + Field: m.Field, + CreatedAt: m.CreatedAt, + Meta: s.encodeFieldOptions(m.Meta), } } @@ -790,9 +796,9 @@ func (s Serializer) encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus func (s Serializer) encodeIndexStatus(m *pilosa.IndexStatus) *internal.IndexStatus { return &internal.IndexStatus{ - Name: m.Name, - ETag: m.ETag, - Fields: s.encodeFieldStatuses(m.Fields), + Name: m.Name, + CreatedAt: m.CreatedAt, + Fields: s.encodeFieldStatuses(m.Fields), } } @@ -807,7 +813,7 @@ func (s Serializer) encodeIndexStatuses(a []*pilosa.IndexStatus) []*internal.Ind func (s Serializer) encodeFieldStatus(m *pilosa.FieldStatus) *internal.FieldStatus { return &internal.FieldStatus{ Name: m.Name, - ETag: m.ETag, + CreatedAt: m.CreatedAt, AvailableShards: m.AvailableShards.Slice(), } } @@ -944,7 +950,7 @@ func (s Serializer) decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo) func (s Serializer) decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) { m.Name = idx.Name - m.ETag = idx.ETag + m.CreatedAt = idx.CreatedAt m.Options = pilosa.IndexOptions{} s.decodeIndexMeta(idx.Options, &m.Options) m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields)) @@ -960,7 +966,7 @@ func (s Serializer) decodeFields(fs []*internal.Field, m []*pilosa.FieldInfo) { func (s Serializer) decodeField(f *internal.Field, m *pilosa.FieldInfo) { m.Name = f.Name - m.ETag = f.ETag + m.CreatedAt = f.CreatedAt m.Options = pilosa.FieldOptions{} s.decodeFieldOptions(f.Meta, &m.Options) m.Views = make([]*pilosa.ViewInfo, 0, len(f.Views)) @@ -1024,7 +1030,7 @@ func (s Serializer) decodeCreateShardMessage(pb *internal.CreateShardMessage, m func (s Serializer) decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) { m.Index = pb.Index - m.ETag = pb.ETag + m.CreatedAt = pb.CreatedAt m.Meta = &pilosa.IndexOptions{} s.decodeIndexMeta(pb.Meta, m.Meta) } @@ -1043,7 +1049,7 @@ func (s Serializer) decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m func (s Serializer) decodeCreateFieldMessage(pb *internal.CreateFieldMessage, m *pilosa.CreateFieldMessage) { m.Index = pb.Index m.Field = pb.Field - m.ETag = pb.ETag + m.CreatedAt = pb.CreatedAt m.Meta = &pilosa.FieldOptions{} s.decodeFieldOptions(pb.Meta, m.Meta) } @@ -1117,7 +1123,7 @@ func (s Serializer) decodeIndexStatuses(a []*internal.IndexStatus) []*pilosa.Ind func (s Serializer) decodeIndexStatus(pb *internal.IndexStatus, m *pilosa.IndexStatus) { m.Name = pb.Name - m.ETag = pb.ETag + m.CreatedAt = pb.CreatedAt m.Fields = s.decodeFieldStatuses(pb.Fields) } @@ -1132,7 +1138,7 @@ func (s Serializer) decodeFieldStatuses(a []*internal.FieldStatus) []*pilosa.Fie func (s Serializer) decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldStatus) { m.Name = pb.Name - m.ETag = pb.ETag + m.CreatedAt = pb.CreatedAt m.AvailableShards = roaring.NewBitmap(pb.AvailableShards...) } @@ -1161,6 +1167,8 @@ func (s Serializer) decodeImportRequest(pb *internal.ImportRequest, m *pilosa.Im m.RowKeys = pb.RowKeys m.ColumnKeys = pb.ColumnKeys m.Timestamps = pb.Timestamps + m.IndexCreatedAt = pb.IndexCreatedAt + m.FieldCreatedAt = pb.FieldCreatedAt } func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportValueRequest) { @@ -1172,6 +1180,8 @@ func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m m.Values = pb.Values m.FloatValues = pb.FloatValues m.StringValues = pb.StringValues + m.IndexCreatedAt = pb.IndexCreatedAt + m.FieldCreatedAt = pb.FieldCreatedAt } func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) { @@ -1183,6 +1193,8 @@ func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest m.Action = pb.Action m.Block = int(pb.Block) m.Views = views + m.IndexCreatedAt = pb.IndexCreatedAt + m.FieldCreatedAt = pb.FieldCreatedAt } func (s Serializer) decodeImportColumnAttrsRequest(pb *internal.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) { diff --git a/field.go b/field.go index 272717565..84bdbec73 100644 --- a/field.go +++ b/field.go @@ -86,11 +86,11 @@ var availableShardFileFlushDuration = &protected{ // Field represents a container for views. type Field struct { - mu sync.RWMutex - etag int64 - path string - index string - name string + mu sync.RWMutex + createdAt int64 + path string + index string + name string viewMap map[string]*view @@ -383,12 +383,12 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) { // Name returns the name the field was initialized with. func (f *Field) Name() string { return f.name } -// ETag is an identifier for a specific version of field. -func (f *Field) ETag() int64 { +// CreatedAt is an timestamp for a specific version of field. +func (f *Field) CreatedAt() int64 { f.mu.RLock() defer f.mu.RUnlock() - return f.etag + return f.createdAt } // Index returns the index name the field was initialized with. @@ -1980,10 +1980,10 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FieldInfo represents schema information for a field. type FieldInfo struct { - Name string `json:"name"` - ETag int64 `json:"etag,omitempty"` - Options FieldOptions `json:"options"` - Views []*ViewInfo `json:"views,omitempty"` + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options FieldOptions `json:"options"` + Views []*ViewInfo `json:"views,omitempty"` } type fieldInfoSlice []*FieldInfo diff --git a/gossip/gossip.go b/gossip/gossip.go index 115c4274b..975c57a19 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -324,7 +324,7 @@ func (g *memberSet) LocalState(join bool) []byte { Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())}, } for _, idx := range m.Schema.Indexes { - is := &pilosa.IndexStatus{Name: idx.Name, ETag: idx.ETag} + is := &pilosa.IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt} for _, f := range idx.Fields { availableShards := roaring.NewBitmap() @@ -334,7 +334,7 @@ func (g *memberSet) LocalState(join bool) []byte { fs := &pilosa.FieldStatus{ Name: f.Name, - ETag: f.ETag, + CreatedAt: f.CreatedAt, AvailableShards: availableShards, } is.Fields = append(is.Fields, fs) diff --git a/handler.go b/handler.go index 3c40d20ad..79c60e285 100644 --- a/handler.go +++ b/handler.go @@ -114,8 +114,10 @@ var NopHandler Handler = nopHandler{} // ImportValueRequest describes the import request structure // for a value (BSI) import. type ImportValueRequest struct { - Index string - Field string + Index string + IndexCreatedAt int64 + Field string + FieldCreatedAt int64 // if Shard is MaxUint64 (an impossible shard value), this // indicates that the column IDs may come from multiple shards. Shard uint64 @@ -141,6 +143,11 @@ func (ivr *ImportValueRequest) Swap(i, j int) { // Validate ensures that the payload of the request is valid. func (ivr *ImportValueRequest) Validate() error { + return ivr.ValidateWithTimestamp(ivr.IndexCreatedAt, ivr.FieldCreatedAt) +} + +// ValidateWithTimestamp ensures that the payload of the request is valid. +func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { if ivr.Index == "" || ivr.Field == "" { return errors.Errorf("index and field required, but got '%s' and '%s'", ivr.Index, ivr.Field) } @@ -160,6 +167,11 @@ func (ivr *ImportValueRequest) Validate() error { if valueSetCount > 1 { return errors.Errorf("must pass ints, floats, or strings but not multiple") } + if ivr.IndexCreatedAt != 0 && ivr.FieldCreatedAt != 0 { + if ivr.IndexCreatedAt != indexCreatedAt || ivr.FieldCreatedAt != fieldCreatedAt { + return ErrPreconditionFailed + } + } return nil } @@ -176,14 +188,26 @@ type ImportColumnAttrsRequest struct { // ImportRequest describes the import request structure // for an import. type ImportRequest struct { - Index string - Field string - Shard uint64 - RowIDs []uint64 - ColumnIDs []uint64 - RowKeys []string - ColumnKeys []string - Timestamps []int64 + Index string + IndexCreatedAt int64 + Field string + FieldCreatedAt int64 + Shard uint64 + RowIDs []uint64 + ColumnIDs []uint64 + RowKeys []string + ColumnKeys []string + Timestamps []int64 +} + +// ValidateWithTimestamp ensures that the payload of the request is valid. +func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { + if ir.IndexCreatedAt != 0 && ir.FieldCreatedAt != 0 { + if ir.IndexCreatedAt != indexCreatedAt || ir.FieldCreatedAt != fieldCreatedAt { + return ErrPreconditionFailed + } + } + return nil } const ( @@ -195,10 +219,22 @@ const ( // ImportRoaringRequest describes the import request structure // for an import containing roaring-encoded data. type ImportRoaringRequest struct { - Clear bool - Action string // [set, clear, overwrite] - Block int - Views map[string][]byte + IndexCreatedAt int64 + FieldCreatedAt int64 + Clear bool + Action string // [set, clear, overwrite] + Block int + Views map[string][]byte +} + +// ValidateWithTimestamp ensures that the payload of the request is valid. +func (irr *ImportRoaringRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { + if irr.IndexCreatedAt != 0 && irr.FieldCreatedAt != 0 { + if irr.IndexCreatedAt != indexCreatedAt || irr.FieldCreatedAt != fieldCreatedAt { + return ErrPreconditionFailed + } + } + return nil } // ImportResponse is the structured response of an import. diff --git a/holder.go b/holder.go index 7434a0d0c..dd995566a 100644 --- a/holder.go +++ b/holder.go @@ -235,8 +235,8 @@ func (h *Holder) Open() error { } if h.isCoordinator() { - index.etag = newETag() - err = index.OpenWithETag() + index.createdAt = timestamp() + err = index.OpenWithTimestamp() } else { err = index.Open() } @@ -386,15 +386,15 @@ func (h *Holder) Schema() []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { di := &IndexInfo{ - Name: index.Name(), - ETag: index.ETag(), - Options: index.Options(), + Name: index.Name(), + CreatedAt: index.CreatedAt(), + Options: index.Options(), } for _, field := range index.Fields() { fi := &FieldInfo{ - Name: field.Name(), - ETag: field.ETag(), - Options: field.Options(), + Name: field.Name(), + CreatedAt: field.CreatedAt(), + Options: field.Options(), } for _, view := range field.views() { fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) @@ -415,7 +415,7 @@ func (h *Holder) limitedSchema() []*IndexInfo { for _, index := range h.Indexes() { di := &IndexInfo{ Name: index.Name(), - ETag: index.ETag(), + CreatedAt: index.CreatedAt(), Options: index.Options(), ShardWidth: ShardWidth, } @@ -424,9 +424,9 @@ func (h *Holder) limitedSchema() []*IndexInfo { continue } fi := &FieldInfo{ - Name: field.Name(), - ETag: field.ETag(), - Options: field.Options(), + Name: field.Name(), + CreatedAt: field.CreatedAt(), + Options: field.Options(), } di.Fields = append(di.Fields, fi) } @@ -445,9 +445,9 @@ func (h *Holder) applySchema(schema *Schema) error { if err != nil { return errors.Wrap(err, "creating index") } - if i.ETag != 0 { + if i.CreatedAt != 0 { idx.mu.Lock() - idx.etag = i.ETag + idx.createdAt = i.CreatedAt idx.mu.Unlock() } @@ -457,9 +457,9 @@ func (h *Holder) applySchema(schema *Schema) error { if err != nil { return errors.Wrap(err, "creating field") } - if f.ETag != 0 { + if f.CreatedAt != 0 { fld.mu.Lock() - fld.etag = f.ETag + fld.createdAt = f.CreatedAt fld.mu.Unlock() } diff --git a/http/handler.go b/http/handler.go index 555c789c9..e2787d77c 100644 --- a/http/handler.go +++ b/http/handler.go @@ -401,9 +401,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // successResponse is a general success/error struct for http responses. type successResponse struct { - h *Handler - Success bool `json:"success"` - Error *Error `json:"error,omitempty"` + h *Handler + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + Error *Error `json:"error,omitempty"` } // check determines success or failure based on the error. @@ -773,7 +775,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { return } - resp := successResponse{h: h} + resp := successResponse{h: h, Name: indexName} // Decode request. req := postIndexRequest{ @@ -788,11 +790,12 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { return } index, err := h.api.CreateIndex(r.Context(), indexName, req.Options) + if index != nil { - w.Header().Add("ETag", strconv.FormatInt(index.ETag(), 10)) - } else if _, ok = err.(pilosa.ConflictError); ok { + resp.CreatedAt = index.CreatedAt() + } else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok { if index, _ = h.api.Index(r.Context(), indexName); index != nil { - w.Header().Add("ETag", strconv.FormatInt(index.ETag(), 10)) + resp.CreatedAt = index.CreatedAt() } } resp.write(w, err) @@ -859,7 +862,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { return } - resp := successResponse{h: h} + resp := successResponse{h: h, Name: fieldName} // Decode request. var req postFieldRequest @@ -937,10 +940,10 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { return } if field != nil { - w.Header().Add("ETag", strconv.FormatInt(field.ETag(), 10)) - } else if _, ok = err.(pilosa.ConflictError); ok { + resp.CreatedAt = field.CreatedAt() + } else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok { if field, _ = h.api.Field(r.Context(), indexName, fieldName); field != nil { - w.Header().Add("ETag", strconv.FormatInt(field.ETag(), 10)) + resp.CreatedAt = field.CreatedAt() } } resp.write(w, err) @@ -1342,30 +1345,6 @@ func validateProtobufHeader(r *http.Request) (error string, code int) { return } -func validateETagHeader(r *http.Request, index *pilosa.Index, field *pilosa.Field) (error string, code int) { - etags := strings.Split(r.Header.Get("If-Match"), ",") - netags := len(etags) - for i := 0; i < netags && i < 2; i++ { - etags[i] = strings.TrimLeft(strings.TrimSpace(etags[i]), "W/") - } - if netags == 1 && etags[0] != "" && etags[0] != "*" { - return "Precondition Failed", http.StatusPreconditionFailed - } - if netags > 1 { - indexETag := etags[0] - if indexETag != "" && strconv.FormatInt(index.ETag(), 10) != indexETag { - return "Precondition Failed", http.StatusPreconditionFailed - } - - fieldETag := etags[1] - if fieldETag != "" && strconv.FormatInt(field.ETag(), 10) != fieldETag { - return "Precondition Failed", http.StatusPreconditionFailed - } - } - - return -} - // handleGetExport handles /export requests. func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { switch r.Header.Get("Accept") { @@ -1868,12 +1847,6 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { return } - // Verify if request matches etag - if error, code := validateETagHeader(r, index, field); error != "" { - http.Error(w, error, code) - return - } - // If the clear flag is true, treat the import as clear bits. q := r.URL.Query() doClear := q.Get("clear") == "true" @@ -1902,7 +1875,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { if err := h.api.ImportValue(r.Context(), req, opts...); err != nil { switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard: + case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1920,7 +1893,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { if err := h.api.Import(r.Context(), req, opts...); err != nil { switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard: + case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1984,27 +1957,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request // Get index and field type to determine how to handle the // import data. indexName := mux.Vars(r)["index"] - index, err := h.api.Index(r.Context(), indexName) - if err != nil { - if errors.Cause(err) == pilosa.ErrIndexNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } fieldName := mux.Vars(r)["field"] - field := index.Field(fieldName) - if field == nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - // Verify if request matches etag - if error, code := validateETagHeader(r, index, field); error != "" { - http.Error(w, error, code) - return - } q := r.URL.Query() remoteStr := q.Get("remote") @@ -2047,6 +2000,10 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request resp.Err = err.Error() if _, ok := err.(pilosa.BadRequestError); ok { w.WriteHeader(http.StatusBadRequest) + } else if _, ok := err.(pilosa.NotFoundError); ok { + w.WriteHeader(http.StatusNotFound) + } else if _, ok := err.(pilosa.PreconditionFailedError); ok { + w.WriteHeader(http.StatusPreconditionFailed) } else { w.WriteHeader(http.StatusInternalServerError) } diff --git a/index.go b/index.go index aaf61e70f..481d22cf4 100644 --- a/index.go +++ b/index.go @@ -36,11 +36,11 @@ import ( // Index represents a container for fields. type Index struct { - mu sync.RWMutex - etag int64 - path string - name string - keys bool // use string keys + mu sync.RWMutex + createdAt int64 + path string + name string + keys bool // use string keys // Existence tracking. trackExistence bool @@ -104,11 +104,11 @@ func NewIndex(path, name string, partitionN int) (*Index, error) { }, nil } -// ETag is an identifier for a specific version of an index. -func (i *Index) ETag() int64 { +// CreatedAt is an timestamp for a specific version of an index. +func (i *Index) CreatedAt() int64 { i.mu.RLock() defer i.mu.RUnlock() - return i.etag + return i.createdAt } // Name returns name of the index. @@ -150,10 +150,10 @@ func (i *Index) options() IndexOptions { // Open opens and initializes the index. func (i *Index) Open() error { return i.open(false) } -// OpenWithETag opens and initializes the index and set a new ETag for fields. -func (i *Index) OpenWithETag() error { return i.open(true) } +// OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields. +func (i *Index) OpenWithTimestamp() error { return i.open(true) } -func (i *Index) open(withETag bool) (err error) { +func (i *Index) open(withTimestamp bool) (err error) { // Ensure the path exists. i.logger.Debugf("ensure index path exists: %s", i.path) if err := os.MkdirAll(i.path, 0777); err != nil { @@ -167,7 +167,7 @@ func (i *Index) open(withETag bool) (err error) { } i.logger.Debugf("open fields for index: %s", i.name) - if err := i.openFields(withETag); err != nil { + if err := i.openFields(withTimestamp); err != nil { return errors.Wrap(err, "opening fields") } @@ -210,7 +210,7 @@ func (i *Index) open(withETag bool) (err error) { var indexQueue = make(chan struct{}, 8) // openFields opens and initializes the fields inside the index. -func (i *Index) openFields(withETag bool) error { +func (i *Index) openFields(withTimestamp bool) error { f, err := os.Open(i.path) if err != nil { return errors.Wrap(err, "opening directory") @@ -242,8 +242,8 @@ fileLoop: i.logger.Debugf("open field: %s", fi.Name()) mu.Lock() fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) - if withETag { - fld.etag = newETag() + if withTimestamp { + fld.createdAt = timestamp() } mu.Unlock() if err != nil { @@ -575,7 +575,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { Name string `json:"name"` - ETag int64 `json:"etag,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` Options IndexOptions `json:"options"` Fields []*FieldInfo `json:"fields"` ShardWidth uint64 `json:"shardWidth"` diff --git a/internal/private.pb.go b/internal/private.pb.go index 694935fb0..ea281b409 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -616,7 +616,7 @@ func (m *DeleteIndexMessage) GetIndex() string { type CreateIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` - ETag int64 `protobuf:"varint,3,opt,name=ETag,proto3" json:"ETag,omitempty"` + CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -669,9 +669,9 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { return nil } -func (m *CreateIndexMessage) GetETag() int64 { +func (m *CreateIndexMessage) GetCreatedAt() int64 { if m != nil { - return m.ETag + return m.CreatedAt } return 0 } @@ -680,7 +680,7 @@ type CreateFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta,proto3" json:"Meta,omitempty"` - ETag int64 `protobuf:"varint,4,opt,name=ETag,proto3" json:"ETag,omitempty"` + CreatedAt int64 `protobuf:"varint,4,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -740,9 +740,9 @@ func (m *CreateFieldMessage) GetMeta() *FieldOptions { return nil } -func (m *CreateFieldMessage) GetETag() int64 { +func (m *CreateFieldMessage) GetCreatedAt() int64 { if m != nil { - return m.ETag + return m.CreatedAt } return 0 } @@ -869,7 +869,7 @@ type Field struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` Views []string `protobuf:"bytes,3,rep,name=Views,proto3" json:"Views,omitempty"` - ETag int64 `protobuf:"varint,4,opt,name=ETag,proto3" json:"ETag,omitempty"` + CreatedAt int64 `protobuf:"varint,4,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -929,9 +929,9 @@ func (m *Field) GetViews() []string { return nil } -func (m *Field) GetETag() int64 { +func (m *Field) GetCreatedAt() int64 { if m != nil { - return m.ETag + return m.CreatedAt } return 0 } @@ -985,7 +985,7 @@ func (m *Schema) GetIndexes() []*Index { type Index struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - ETag int64 `protobuf:"varint,2,opt,name=ETag,proto3" json:"ETag,omitempty"` + CreatedAt int64 `protobuf:"varint,2,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` Options *IndexMeta `protobuf:"bytes,5,opt,name=Options,proto3" json:"Options,omitempty"` Fields []*Field `protobuf:"bytes,4,rep,name=Fields,proto3" json:"Fields,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1033,9 +1033,9 @@ func (m *Index) GetName() string { return "" } -func (m *Index) GetETag() int64 { +func (m *Index) GetCreatedAt() int64 { if m != nil { - return m.ETag + return m.CreatedAt } return 0 } @@ -1372,7 +1372,7 @@ func (m *NodeStatus) GetIndexes() []*IndexStatus { type IndexStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields,proto3" json:"Fields,omitempty"` - ETag int64 `protobuf:"varint,3,opt,name=ETag,proto3" json:"ETag,omitempty"` + CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1425,9 +1425,9 @@ func (m *IndexStatus) GetFields() []*FieldStatus { return nil } -func (m *IndexStatus) GetETag() int64 { +func (m *IndexStatus) GetCreatedAt() int64 { if m != nil { - return m.ETag + return m.CreatedAt } return 0 } @@ -1435,7 +1435,7 @@ func (m *IndexStatus) GetETag() int64 { type FieldStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards,proto3" json:"AvailableShards,omitempty"` - ETag int64 `protobuf:"varint,3,opt,name=ETag,proto3" json:"ETag,omitempty"` + CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1488,9 +1488,9 @@ func (m *FieldStatus) GetAvailableShards() []uint64 { return nil } -func (m *FieldStatus) GetETag() int64 { +func (m *FieldStatus) GetCreatedAt() int64 { if m != nil { - return m.ETag + return m.CreatedAt } return 0 } @@ -2469,98 +2469,98 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1448 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdd, 0x6e, 0x1b, 0xc5, - 0x17, 0xff, 0xaf, 0xd7, 0x4e, 0xec, 0xe3, 0x38, 0x75, 0xa6, 0x6d, 0xba, 0xcd, 0x1f, 0x05, 0x33, - 0x54, 0xd4, 0x54, 0x6a, 0xa8, 0x5a, 0x24, 0x3e, 0x2b, 0xb5, 0x89, 0xdd, 0x62, 0x20, 0x69, 0x3b, - 0x4e, 0x7b, 0x8b, 0x26, 0xf6, 0x28, 0x59, 0x65, 0xbd, 0xeb, 0xee, 0xce, 0xa6, 0x4e, 0x2f, 0x10, - 0x77, 0x20, 0xf1, 0x02, 0x5c, 0x70, 0xc1, 0x7b, 0xf0, 0x02, 0x5c, 0xf2, 0x08, 0xa8, 0x3c, 0x05, - 0x77, 0x68, 0xce, 0xcc, 0xec, 0xae, 0x1d, 0x87, 0x84, 0x96, 0xbb, 0x39, 0x5f, 0x73, 0x7e, 0xe7, - 0x73, 0xc7, 0x86, 0xc6, 0x38, 0xf6, 0x8f, 0xb8, 0x14, 0x1b, 0xe3, 0x38, 0x92, 0x11, 0xa9, 0xfa, - 0xa1, 0x14, 0x71, 0xc8, 0x83, 0xb5, 0xa5, 0x71, 0xba, 0x17, 0xf8, 0x03, 0xcd, 0xa7, 0x0f, 0xa1, - 0xd6, 0x0b, 0x87, 0x62, 0xb2, 0x2d, 0x24, 0x27, 0x04, 0xca, 0x5f, 0x89, 0xe3, 0xc4, 0x73, 0x5b, - 0x4e, 0xbb, 0xca, 0xf0, 0x4c, 0xde, 0x83, 0xe5, 0xdd, 0x98, 0x0f, 0x0e, 0xbb, 0x13, 0x3f, 0x91, - 0x22, 0x1c, 0x08, 0xaf, 0x8c, 0xd2, 0x19, 0x2e, 0xfd, 0xc5, 0x85, 0xa5, 0x07, 0xbe, 0x08, 0x86, - 0x8f, 0xc6, 0xd2, 0x8f, 0xc2, 0x44, 0x5d, 0xb6, 0x7b, 0x3c, 0x16, 0x5e, 0xb5, 0xe5, 0xb4, 0x6b, - 0x0c, 0xcf, 0xe4, 0x2d, 0xa8, 0x6d, 0xf1, 0xc1, 0x81, 0x40, 0x81, 0x8b, 0x82, 0x9c, 0x91, 0x49, - 0xfb, 0xfe, 0x4b, 0xed, 0xa5, 0xc1, 0x72, 0x06, 0x69, 0x41, 0x7d, 0xd7, 0x1f, 0x89, 0x27, 0x29, - 0x0f, 0x65, 0x3a, 0xf2, 0x2a, 0x68, 0x5d, 0x64, 0x91, 0x55, 0x58, 0x78, 0x14, 0x0c, 0xb7, 0xfd, - 0xd0, 0xab, 0xb5, 0x9c, 0xb6, 0xcb, 0x0c, 0x65, 0xf9, 0x7c, 0xe2, 0x41, 0xce, 0xe7, 0x93, 0x2c, - 0xdc, 0xfa, 0x74, 0xb8, 0x3b, 0x51, 0x5f, 0xf2, 0x70, 0xc8, 0xe3, 0xe1, 0x33, 0x5f, 0xbc, 0xf0, - 0x96, 0x74, 0xb8, 0xd3, 0x5c, 0x65, 0xbb, 0xc9, 0x13, 0xe1, 0x35, 0xf0, 0x46, 0x3c, 0x93, 0x35, - 0xa8, 0x6e, 0xfa, 0xb2, 0x23, 0xc6, 0xf2, 0xc0, 0x5b, 0x6e, 0x39, 0xed, 0x32, 0xcb, 0x68, 0x72, - 0x09, 0x2a, 0xfd, 0x01, 0x0f, 0x84, 0x77, 0x01, 0x0d, 0x34, 0x41, 0x28, 0x2c, 0x3d, 0x88, 0x62, - 0xe1, 0xef, 0x87, 0x58, 0x04, 0xaf, 0x89, 0x41, 0x4d, 0xf1, 0xc8, 0xbb, 0xe0, 0xaa, 0x90, 0x56, - 0x5a, 0x4e, 0xbb, 0x7e, 0x7b, 0x65, 0xc3, 0xd6, 0x71, 0xa3, 0x23, 0x06, 0xfe, 0x88, 0x07, 0x4c, - 0x49, 0x51, 0x89, 0x4f, 0x3c, 0x72, 0xba, 0x12, 0x9f, 0x50, 0x0a, 0xcb, 0xbd, 0xd1, 0x38, 0x8a, - 0x25, 0x13, 0xc9, 0x38, 0x0a, 0x13, 0x41, 0x9a, 0xe0, 0x76, 0xe3, 0xd8, 0x73, 0xd0, 0xad, 0x3a, - 0xd2, 0x6f, 0xa1, 0xb9, 0x19, 0x44, 0x83, 0xc3, 0x0e, 0x97, 0x9c, 0x89, 0xe7, 0xa9, 0x48, 0xa4, - 0xc2, 0xae, 0xe1, 0x69, 0x3d, 0x4d, 0x28, 0x2e, 0xd6, 0xdb, 0x2b, 0x69, 0x2e, 0x12, 0x2a, 0x2f, - 0x98, 0x35, 0x5d, 0x1e, 0x3c, 0x63, 0xec, 0x07, 0x3c, 0x1e, 0x62, 0x4d, 0xcb, 0x4c, 0x13, 0x8a, - 0x8b, 0x9e, 0xb0, 0x0f, 0xca, 0x4c, 0x13, 0xb4, 0x07, 0x2b, 0x05, 0xff, 0x06, 0xe6, 0x2a, 0x2c, - 0xb0, 0xe8, 0x45, 0xaf, 0x93, 0x78, 0x4e, 0xcb, 0x6d, 0x97, 0x99, 0xa1, 0xb0, 0x61, 0xa2, 0x20, - 0x1d, 0x85, 0x4a, 0x54, 0x42, 0x51, 0xce, 0xa0, 0x57, 0xa1, 0x82, 0xdd, 0xa3, 0xa2, 0xcc, 0x6d, - 0xd5, 0x91, 0x7e, 0xef, 0x40, 0x6d, 0x9b, 0x4f, 0x10, 0x48, 0x42, 0xee, 0x42, 0xd5, 0xd6, 0x16, - 0x95, 0xea, 0xb7, 0xdf, 0xc9, 0x33, 0x98, 0xa9, 0x6d, 0x58, 0x9d, 0x6e, 0x28, 0xe3, 0x63, 0x96, - 0x99, 0xac, 0x7d, 0x06, 0x8d, 0x29, 0x91, 0xf2, 0x77, 0x28, 0x8e, 0x6d, 0x56, 0x0f, 0xc5, 0xb1, - 0x8a, 0xf5, 0x88, 0x07, 0xa9, 0xc0, 0x5c, 0x95, 0x99, 0x26, 0x3e, 0x2d, 0x7d, 0xec, 0xd0, 0x67, - 0x40, 0xb6, 0x62, 0xc1, 0xa5, 0x40, 0x27, 0xdb, 0x22, 0x49, 0xf8, 0xbe, 0x38, 0x2b, 0xe3, 0x6e, - 0x31, 0xe3, 0x59, 0x76, 0x4b, 0x85, 0xec, 0xd2, 0x1b, 0x40, 0x3a, 0x22, 0x10, 0x52, 0x98, 0xe9, - 0xfe, 0x87, 0x7b, 0xe9, 0xbe, 0xc5, 0x70, 0xb6, 0x2e, 0xb9, 0x0e, 0x65, 0xb5, 0x2a, 0xd0, 0x59, - 0xfd, 0xf6, 0xc5, 0x3c, 0x4f, 0xd9, 0x16, 0x61, 0x65, 0xbb, 0x4b, 0xba, 0xbb, 0x7c, 0x1f, 0xb1, - 0xba, 0x0c, 0xcf, 0xf4, 0x3b, 0xc7, 0x7a, 0x42, 0xe8, 0xe7, 0x8c, 0x76, 0xaa, 0xbf, 0x6e, 0x18, - 0xff, 0x2e, 0xfa, 0x5f, 0xcd, 0xfd, 0x17, 0x77, 0xcf, 0x0c, 0x84, 0x72, 0x01, 0xc2, 0x3d, 0x9b, - 0x97, 0xd7, 0x45, 0x40, 0x07, 0xf0, 0x7f, 0x7d, 0xc3, 0xfd, 0x23, 0xee, 0x07, 0x7c, 0x2f, 0xf8, - 0x57, 0xa5, 0x9b, 0x0a, 0xc6, 0x83, 0x45, 0xb4, 0xed, 0x75, 0xcc, 0x10, 0x58, 0x92, 0x3e, 0x87, - 0x7c, 0x9e, 0x76, 0xf8, 0x48, 0x98, 0xdb, 0xf0, 0x9c, 0xe5, 0xa0, 0x74, 0x8e, 0x1c, 0x5c, 0x82, - 0x8a, 0x9a, 0x41, 0xb5, 0xd3, 0x5d, 0xe5, 0x18, 0x89, 0xb9, 0x99, 0xb9, 0x03, 0x0b, 0xfd, 0xc1, - 0x81, 0x18, 0x71, 0xf2, 0x3e, 0x2c, 0x22, 0x6a, 0x91, 0x98, 0x71, 0xb8, 0x30, 0x53, 0x66, 0x66, - 0xe5, 0x6a, 0x90, 0x4c, 0x84, 0xf3, 0x80, 0x5a, 0x37, 0xa5, 0xdc, 0x0d, 0xb9, 0x09, 0x8b, 0x06, - 0x21, 0xee, 0x88, 0x53, 0x7a, 0xc8, 0xea, 0x90, 0xeb, 0xb0, 0x80, 0x51, 0x25, 0x5e, 0x79, 0x16, - 0x0a, 0xf2, 0x99, 0x11, 0xd3, 0x2e, 0xb8, 0x4f, 0x59, 0x4f, 0xad, 0x0a, 0x8c, 0xc2, 0x02, 0x31, - 0x94, 0x82, 0xf2, 0x45, 0x94, 0x48, 0x93, 0x7f, 0x3c, 0x2b, 0xde, 0xe3, 0x28, 0x96, 0x98, 0xfb, - 0x06, 0xc3, 0x33, 0xfd, 0xd9, 0x81, 0xf2, 0x4e, 0x34, 0x14, 0x64, 0x19, 0x4a, 0xbd, 0x8e, 0xb9, - 0xa4, 0xd4, 0xeb, 0x90, 0xb7, 0xf1, 0x7e, 0x93, 0xf3, 0x46, 0x8e, 0xe2, 0x29, 0xeb, 0x31, 0xf4, - 0x7c, 0x0d, 0x1a, 0xbd, 0x64, 0x2b, 0x8a, 0xe2, 0xa1, 0x1f, 0x72, 0x19, 0xc5, 0xe6, 0x2b, 0x3a, - 0xcd, 0xc4, 0x69, 0x95, 0x5c, 0xea, 0xef, 0x5b, 0x8d, 0x69, 0x82, 0x5c, 0x87, 0xc5, 0x87, 0xec, - 0xf1, 0x96, 0x72, 0x50, 0x99, 0xe7, 0xc0, 0x4a, 0xe9, 0x3d, 0x68, 0x2a, 0x74, 0x68, 0x65, 0x3b, - 0x6e, 0x15, 0x16, 0x14, 0x2f, 0x43, 0x6b, 0xa8, 0xdc, 0x55, 0xa9, 0xe0, 0x8a, 0x7e, 0xad, 0x6f, - 0xe8, 0x1e, 0x89, 0x50, 0x16, 0x7a, 0x16, 0x69, 0xbc, 0xa0, 0xc1, 0x34, 0x41, 0xa8, 0xce, 0x84, - 0x09, 0x79, 0x39, 0x47, 0xa4, 0xb8, 0x0c, 0x65, 0xf4, 0x47, 0x07, 0xc0, 0x02, 0x4a, 0x93, 0xcc, - 0xc4, 0x39, 0xdd, 0x84, 0xb4, 0x6d, 0x9f, 0x99, 0x19, 0x6e, 0xe6, 0x5a, 0x9a, 0xcf, 0x6c, 0x1f, - 0x7e, 0x90, 0xf7, 0xa1, 0x2e, 0xfe, 0xe5, 0x99, 0x56, 0xd1, 0x5e, 0xf3, 0x6e, 0x1c, 0x42, 0xbd, - 0xc0, 0x9f, 0xdb, 0x92, 0x37, 0xb3, 0x7e, 0x2a, 0xcd, 0x5e, 0x89, 0x7c, 0x73, 0xa5, 0x51, 0x9a, - 0xbb, 0xc5, 0xbe, 0x81, 0x7a, 0x41, 0x75, 0xae, 0x97, 0x36, 0x5c, 0x98, 0xde, 0x0e, 0xf6, 0xf3, - 0x34, 0xcb, 0x9e, 0xeb, 0xc0, 0x87, 0xc6, 0x56, 0x90, 0x26, 0x52, 0xc4, 0xc6, 0x85, 0xfa, 0xce, - 0x69, 0x46, 0x56, 0xe4, 0x9c, 0x31, 0xbf, 0xce, 0xe4, 0x1a, 0x54, 0x54, 0xba, 0xf5, 0xe0, 0x9f, - 0xac, 0x85, 0x16, 0xd2, 0x67, 0x50, 0xdd, 0xec, 0xf7, 0x1e, 0xc6, 0x51, 0x3a, 0x3e, 0x6d, 0x82, - 0xf1, 0xad, 0x56, 0x2a, 0x3c, 0xe2, 0x9a, 0xfa, 0x41, 0xa2, 0x11, 0xe3, 0xeb, 0xa3, 0xa9, 0x5f, - 0x1f, 0x65, 0xc3, 0xe1, 0x13, 0xda, 0x87, 0x15, 0xbd, 0xe8, 0xd5, 0xbe, 0x79, 0x9d, 0xd5, 0x68, - 0xdf, 0x11, 0x6e, 0xfe, 0x8e, 0x50, 0x97, 0xea, 0xcd, 0xfb, 0x5f, 0x5e, 0xfa, 0x57, 0x09, 0x56, - 0x98, 0x48, 0xfc, 0x97, 0xa2, 0x17, 0x26, 0x32, 0x4e, 0x07, 0x6a, 0xef, 0x28, 0xfb, 0x2f, 0xa3, - 0x3d, 0x93, 0x6d, 0x97, 0x69, 0xe2, 0x3c, 0x13, 0x41, 0x6e, 0x41, 0x7d, 0x76, 0x09, 0x9c, 0x54, - 0x2d, 0xaa, 0x90, 0x5b, 0xb0, 0xd8, 0x8f, 0xd2, 0x78, 0x90, 0xb5, 0x79, 0x61, 0xa3, 0x6b, 0x64, - 0x5a, 0xcc, 0xac, 0x1a, 0x79, 0x02, 0x64, 0x37, 0xe6, 0x61, 0x12, 0x70, 0x05, 0xd6, 0x1a, 0x57, - 0x67, 0x9f, 0x2e, 0x05, 0x9d, 0xa9, 0x7b, 0xe6, 0x18, 0x93, 0x0f, 0x8b, 0x73, 0xec, 0x2d, 0x22, - 0xea, 0x4b, 0xd3, 0xa8, 0xcd, 0x68, 0x14, 0xe7, 0xfd, 0xee, 0x4c, 0xa7, 0x7a, 0x0b, 0x68, 0x78, - 0x25, 0x37, 0x9c, 0x12, 0xb3, 0x69, 0x6d, 0xfa, 0x83, 0x03, 0x4b, 0x45, 0x64, 0xe7, 0xda, 0x1f, - 0x59, 0xc1, 0x4b, 0x67, 0xbf, 0x8d, 0x6c, 0xc1, 0xcb, 0xf3, 0x5e, 0xa3, 0x95, 0xe2, 0x7b, 0x29, - 0x85, 0x2b, 0xa7, 0xa4, 0xeb, 0x0d, 0x40, 0xb5, 0xa0, 0xfe, 0x98, 0xc7, 0xd2, 0x57, 0x57, 0x9a, - 0x6f, 0x7c, 0x85, 0x15, 0x59, 0xf4, 0x10, 0xae, 0x9e, 0x68, 0xbe, 0xad, 0x68, 0x34, 0x56, 0x5d, - 0xfe, 0x06, 0x4d, 0xa8, 0x16, 0x7a, 0x1c, 0x9b, 0xf6, 0xab, 0x31, 0x4d, 0xd0, 0x4f, 0xe0, 0x72, - 0x5f, 0xc8, 0x42, 0xeb, 0xd9, 0x19, 0x6a, 0x81, 0xbb, 0x23, 0x5e, 0x9c, 0x12, 0xa0, 0x12, 0xd1, - 0xcf, 0xc1, 0x7b, 0x3a, 0x1e, 0x72, 0x29, 0x5e, 0xcb, 0x7a, 0x13, 0xaa, 0xbb, 0xd1, 0x38, 0x0a, - 0xa2, 0xfd, 0xe3, 0x33, 0x76, 0x99, 0x07, 0x8b, 0xfa, 0xeb, 0xa5, 0x17, 0x66, 0x8d, 0x59, 0x92, - 0x5e, 0x54, 0x63, 0x3a, 0xe0, 0xc1, 0x20, 0x0d, 0x14, 0x0c, 0xf5, 0xb0, 0x4f, 0xa8, 0x30, 0x83, - 0xc0, 0x31, 0x71, 0x85, 0x0f, 0xe2, 0x7d, 0x64, 0xd8, 0x0f, 0xa2, 0xa6, 0xc8, 0x47, 0x50, 0x2f, - 0x68, 0x9b, 0x04, 0x5e, 0x9e, 0x99, 0x17, 0x2d, 0x64, 0x45, 0x4d, 0xfa, 0xab, 0x33, 0x65, 0x79, - 0xe2, 0x6d, 0x60, 0x1c, 0x1e, 0xe9, 0xa2, 0x54, 0x99, 0xa1, 0x54, 0xac, 0xdd, 0xc9, 0x20, 0x48, - 0x13, 0x25, 0xd2, 0xcf, 0x81, 0x9c, 0xa1, 0x62, 0x55, 0xbf, 0x5e, 0xa3, 0x54, 0x9a, 0xcd, 0x69, - 0x49, 0xf5, 0x43, 0xb2, 0x23, 0xf8, 0x30, 0xf0, 0x43, 0x81, 0x5d, 0xea, 0xb2, 0x8c, 0x26, 0xb7, - 0xf4, 0xb6, 0xb7, 0xa3, 0xb6, 0x36, 0x17, 0x3e, 0x6a, 0xe8, 0x2f, 0x41, 0x42, 0x09, 0x34, 0x67, - 0x45, 0x9b, 0xcd, 0xdf, 0x5e, 0xad, 0x3b, 0xbf, 0xbf, 0x5a, 0x77, 0xfe, 0x78, 0xb5, 0xee, 0xfc, - 0xf4, 0xe7, 0xfa, 0xff, 0xf6, 0x16, 0xf0, 0xff, 0x80, 0x3b, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, - 0x4f, 0xdc, 0x5a, 0x3d, 0x38, 0x10, 0x00, 0x00, + // 1445 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x6f, 0x1b, 0x45, + 0x14, 0x66, 0xbd, 0x76, 0x62, 0x1f, 0xc7, 0xa9, 0x33, 0x6d, 0xd3, 0x6d, 0x40, 0xc1, 0x0c, 0x15, + 0x35, 0x95, 0x1a, 0xaa, 0x16, 0x89, 0x6b, 0xa5, 0x36, 0x71, 0x5a, 0x0c, 0x24, 0x6d, 0xc7, 0x69, + 0xdf, 0x27, 0xeb, 0x51, 0xb3, 0xca, 0x7a, 0xd7, 0xdd, 0x4b, 0xea, 0x14, 0x89, 0x57, 0x10, 0x3c, + 0xf1, 0xc6, 0x03, 0x0f, 0xfc, 0x0f, 0xfe, 0x00, 0x8f, 0xfc, 0x04, 0x54, 0x7e, 0x05, 0x6f, 0x68, + 0xce, 0xcc, 0xec, 0xc5, 0x71, 0x70, 0x68, 0x79, 0x9b, 0x73, 0xff, 0xce, 0x65, 0xce, 0x8e, 0x0d, + 0xad, 0x71, 0xe4, 0x1d, 0xf1, 0x44, 0x6c, 0x8c, 0xa3, 0x30, 0x09, 0x49, 0xdd, 0x0b, 0x12, 0x11, + 0x05, 0xdc, 0x5f, 0x5b, 0x1a, 0xa7, 0xfb, 0xbe, 0xe7, 0x2a, 0x3e, 0xbd, 0x0f, 0x8d, 0x7e, 0x30, + 0x14, 0x93, 0x1d, 0x91, 0x70, 0x42, 0xa0, 0xfa, 0x95, 0x38, 0x8e, 0x1d, 0xbb, 0x63, 0x75, 0xeb, + 0x0c, 0xcf, 0xe4, 0x3d, 0x58, 0xde, 0x8b, 0xb8, 0x7b, 0xb8, 0x3d, 0xf1, 0xe2, 0x44, 0x04, 0xae, + 0x70, 0xaa, 0x28, 0x9d, 0xe2, 0xd2, 0x5f, 0x6d, 0x58, 0xba, 0xe7, 0x09, 0x7f, 0xf8, 0x60, 0x9c, + 0x78, 0x61, 0x10, 0x4b, 0x67, 0x7b, 0xc7, 0x63, 0xe1, 0xd4, 0x3b, 0x56, 0xb7, 0xc1, 0xf0, 0x4c, + 0xde, 0x82, 0xc6, 0x16, 0x77, 0x0f, 0x04, 0x0a, 0x6c, 0x14, 0xe4, 0x8c, 0x4c, 0x3a, 0xf0, 0x5e, + 0xa8, 0x28, 0x2d, 0x96, 0x33, 0x48, 0x07, 0x9a, 0x7b, 0xde, 0x48, 0x3c, 0x4a, 0x79, 0x90, 0xa4, + 0x23, 0xa7, 0x86, 0xd6, 0x45, 0x16, 0x59, 0x85, 0x85, 0x07, 0xfe, 0x70, 0xc7, 0x0b, 0x9c, 0x46, + 0xc7, 0xea, 0xda, 0x4c, 0x53, 0x86, 0xcf, 0x27, 0x0e, 0xe4, 0x7c, 0x3e, 0xc9, 0xd2, 0x6d, 0x96, + 0xd3, 0xdd, 0x0d, 0x07, 0x09, 0x0f, 0x86, 0x3c, 0x1a, 0x3e, 0xf1, 0xc4, 0x73, 0x67, 0x49, 0xa5, + 0x5b, 0xe6, 0x4a, 0xdb, 0x4d, 0x1e, 0x0b, 0xa7, 0x85, 0x1e, 0xf1, 0x4c, 0xd6, 0xa0, 0xbe, 0xe9, + 0x25, 0x3d, 0x31, 0x4e, 0x0e, 0x9c, 0xe5, 0x8e, 0xd5, 0xad, 0xb2, 0x8c, 0x26, 0x17, 0xa0, 0x36, + 0x70, 0xb9, 0x2f, 0x9c, 0x73, 0x68, 0xa0, 0x08, 0x42, 0x61, 0xe9, 0x5e, 0x18, 0x09, 0xef, 0x69, + 0x80, 0x4d, 0x70, 0xda, 0x98, 0x54, 0x89, 0x47, 0xde, 0x05, 0x5b, 0xa6, 0xb4, 0xd2, 0xb1, 0xba, + 0xcd, 0x9b, 0x2b, 0x1b, 0xa6, 0x8f, 0x1b, 0x3d, 0xe1, 0x7a, 0x23, 0xee, 0x33, 0x29, 0x45, 0x25, + 0x3e, 0x71, 0xc8, 0xe9, 0x4a, 0x7c, 0x42, 0x29, 0x2c, 0xf7, 0x47, 0xe3, 0x30, 0x4a, 0x98, 0x88, + 0xc7, 0x61, 0x10, 0x0b, 0xd2, 0x06, 0x7b, 0x3b, 0x8a, 0x1c, 0x0b, 0xc3, 0xca, 0x23, 0xfd, 0x16, + 0xda, 0x9b, 0x7e, 0xe8, 0x1e, 0xf6, 0x78, 0xc2, 0x99, 0x78, 0x96, 0x8a, 0x38, 0x91, 0xd8, 0x15, + 0x3c, 0xa5, 0xa7, 0x08, 0xc9, 0xc5, 0x7e, 0x3b, 0x15, 0xc5, 0x45, 0x42, 0xd6, 0x05, 0xab, 0xa6, + 0xda, 0x83, 0x67, 0xcc, 0xfd, 0x80, 0x47, 0x43, 0xec, 0x69, 0x95, 0x29, 0x42, 0x72, 0x31, 0x12, + 0xce, 0x41, 0x95, 0x29, 0x82, 0xf6, 0x61, 0xa5, 0x10, 0x5f, 0xc3, 0x5c, 0x85, 0x05, 0x16, 0x3e, + 0xef, 0xf7, 0x62, 0xc7, 0xea, 0xd8, 0xdd, 0x2a, 0xd3, 0x14, 0x0e, 0x4c, 0xe8, 0xa7, 0xa3, 0x40, + 0x8a, 0x2a, 0x28, 0xca, 0x19, 0xf4, 0x32, 0xd4, 0x70, 0x7a, 0x64, 0x96, 0xb9, 0xad, 0x3c, 0xd2, + 0xef, 0x2c, 0x68, 0xec, 0xf0, 0x09, 0x02, 0x89, 0xc9, 0x6d, 0xa8, 0x9b, 0xde, 0xa2, 0x52, 0xf3, + 0xe6, 0x3b, 0x79, 0x05, 0x33, 0xb5, 0x0d, 0xa3, 0xb3, 0x1d, 0x24, 0xd1, 0x31, 0xcb, 0x4c, 0xd6, + 0x3e, 0x83, 0x56, 0x49, 0x24, 0xe3, 0x1d, 0x8a, 0x63, 0x53, 0xd5, 0x43, 0x71, 0x2c, 0x73, 0x3d, + 0xe2, 0x7e, 0x2a, 0xb0, 0x56, 0x55, 0xa6, 0x88, 0x4f, 0x2b, 0x1f, 0x5b, 0xf4, 0x09, 0x90, 0xad, + 0x48, 0xf0, 0x44, 0x60, 0x90, 0x1d, 0x11, 0xc7, 0xfc, 0xa9, 0x98, 0x57, 0x71, 0xbb, 0x58, 0xf1, + 0xac, 0xba, 0x95, 0x42, 0x75, 0xe9, 0x35, 0x20, 0x3d, 0xe1, 0x8b, 0x44, 0xe8, 0xdb, 0xfd, 0x2f, + 0x7e, 0xe9, 0x33, 0x83, 0x61, 0xbe, 0x2e, 0xb9, 0x0a, 0x55, 0xb9, 0x2a, 0x30, 0x58, 0xf3, 0xe6, + 0xf9, 0xbc, 0x4e, 0xd9, 0x16, 0x61, 0xa8, 0x80, 0xbd, 0x41, 0xa7, 0xc3, 0xbb, 0x09, 0x02, 0xb6, + 0x59, 0xce, 0xa0, 0x3f, 0x58, 0x26, 0x26, 0x26, 0x71, 0xc6, 0xbc, 0x4b, 0x93, 0x76, 0x4d, 0x23, + 0xb1, 0x11, 0xc9, 0x6a, 0x8e, 0xa4, 0xb8, 0x85, 0x66, 0x81, 0xa9, 0x4e, 0x83, 0xb9, 0x63, 0x6a, + 0xf5, 0xaa, 0x58, 0xa8, 0x0b, 0x6f, 0x2a, 0x0f, 0x77, 0x8f, 0xb8, 0xe7, 0xf3, 0x7d, 0xff, 0x3f, + 0xb5, 0xb3, 0x94, 0x96, 0x03, 0x8b, 0x68, 0xdb, 0xef, 0xe9, 0x8b, 0x61, 0x48, 0xfa, 0x0d, 0xe4, + 0x77, 0x6c, 0x97, 0x8f, 0x84, 0xf6, 0x86, 0xe7, 0xac, 0x1a, 0x95, 0x33, 0x54, 0xe3, 0x02, 0xd4, + 0xe4, 0xbd, 0x94, 0x7b, 0xde, 0x96, 0x81, 0x91, 0x98, 0x53, 0xa3, 0x5b, 0xb0, 0x30, 0x70, 0x0f, + 0xc4, 0x88, 0x93, 0xf7, 0x61, 0x11, 0xf1, 0x8b, 0x58, 0x5f, 0x96, 0x73, 0x53, 0x43, 0xc0, 0x8c, + 0x9c, 0xfe, 0x64, 0xe9, 0xc4, 0x67, 0x42, 0x2e, 0x05, 0xac, 0x4c, 0x05, 0x24, 0xd7, 0x61, 0x51, + 0xa3, 0xc6, 0x5d, 0x72, 0xca, 0xac, 0x19, 0x1d, 0x72, 0x15, 0x16, 0x30, 0xd3, 0xd8, 0xa9, 0x4e, + 0x83, 0x42, 0x3e, 0xd3, 0x62, 0xba, 0x0d, 0xf6, 0x63, 0xd6, 0x97, 0x2b, 0x05, 0xf3, 0x31, 0x90, + 0x34, 0x25, 0x81, 0x7e, 0x11, 0xc6, 0x89, 0xee, 0x09, 0x9e, 0x25, 0xef, 0x61, 0x18, 0xa9, 0x29, + 0x6e, 0x31, 0x3c, 0xd3, 0x5f, 0x2c, 0xa8, 0xee, 0x86, 0x43, 0x41, 0x96, 0xa1, 0xd2, 0xef, 0x69, + 0x27, 0x95, 0x7e, 0x8f, 0xbc, 0x8d, 0xfe, 0x75, 0x1f, 0x5a, 0x39, 0x8a, 0xc7, 0xac, 0xcf, 0x30, + 0xf2, 0x15, 0x68, 0xf5, 0xe3, 0xad, 0x30, 0x8c, 0x86, 0x5e, 0xc0, 0x93, 0x30, 0xd2, 0x5f, 0xdb, + 0x32, 0x13, 0x6f, 0x75, 0xc2, 0x13, 0xf5, 0x1d, 0x6c, 0x30, 0x45, 0x90, 0xab, 0xb0, 0x78, 0x9f, + 0x3d, 0xdc, 0x92, 0x01, 0x6a, 0xb3, 0x02, 0x18, 0x29, 0xbd, 0x03, 0x6d, 0x89, 0x0e, 0xad, 0xcc, + 0x14, 0xae, 0xc2, 0x82, 0xe4, 0x65, 0x68, 0x35, 0x95, 0x87, 0xaa, 0x14, 0x42, 0xd1, 0xaf, 0x95, + 0x87, 0xed, 0x23, 0x11, 0x24, 0x85, 0x39, 0x46, 0x1a, 0x1d, 0xb4, 0x98, 0x22, 0x08, 0x55, 0x95, + 0xd0, 0x29, 0x2f, 0xe7, 0x88, 0x24, 0x97, 0xa1, 0x8c, 0xfe, 0x68, 0x01, 0x18, 0x40, 0x69, 0x9c, + 0x99, 0x58, 0xa7, 0x9b, 0x90, 0xae, 0x99, 0x38, 0x7d, 0xc3, 0xdb, 0xb9, 0x96, 0xe2, 0x33, 0x33, + 0x91, 0x1f, 0xe4, 0x13, 0xa9, 0x9a, 0x7f, 0x71, 0x6a, 0x54, 0x54, 0xd4, 0x7c, 0x2e, 0x03, 0x68, + 0x16, 0xf8, 0x33, 0x87, 0xf3, 0x7a, 0x36, 0x4f, 0x95, 0x69, 0x97, 0xc8, 0xd7, 0x2e, 0xb5, 0xd2, + 0x9c, 0x6d, 0xe7, 0x41, 0xb3, 0x60, 0x34, 0x33, 0x5e, 0x17, 0xce, 0x95, 0x77, 0x87, 0xf9, 0xa0, + 0x4d, 0xb3, 0xe7, 0x86, 0x6a, 0x6d, 0xf9, 0x69, 0x9c, 0x88, 0x48, 0x07, 0x93, 0xea, 0x8a, 0x91, + 0x35, 0x3e, 0x67, 0xcc, 0xee, 0x3d, 0xb9, 0x02, 0x35, 0xd9, 0x02, 0xb5, 0x20, 0x4e, 0xf6, 0x47, + 0x09, 0xe9, 0x13, 0xa8, 0x6f, 0x0e, 0xfa, 0xf7, 0xa3, 0x30, 0x1d, 0xcf, 0x4c, 0xc9, 0x3c, 0x00, + 0x2b, 0x85, 0x07, 0x60, 0x5b, 0x3d, 0x66, 0x14, 0x6c, 0x7c, 0xb9, 0xb4, 0xd5, 0xcb, 0xa5, 0xaa, + 0x39, 0x7c, 0x42, 0x07, 0xb0, 0xa2, 0xf2, 0x91, 0x7b, 0xe9, 0x55, 0x56, 0xa8, 0x79, 0x83, 0xd8, + 0xf9, 0x1b, 0x44, 0x3a, 0x55, 0x1b, 0xfa, 0xff, 0x74, 0xfa, 0x77, 0x05, 0x56, 0x98, 0x88, 0xbd, + 0x17, 0xa2, 0x1f, 0xc4, 0x49, 0x94, 0xba, 0x72, 0x17, 0x49, 0xfb, 0x2f, 0xc3, 0x7d, 0x5d, 0x6d, + 0x9b, 0x29, 0xe2, 0x2c, 0xb7, 0x84, 0xdc, 0x80, 0xe6, 0xf4, 0x62, 0x38, 0xa9, 0x5a, 0x54, 0x21, + 0x37, 0x60, 0x71, 0x10, 0xa6, 0x91, 0x9b, 0x8d, 0x7e, 0x61, 0xf3, 0x2b, 0x64, 0x4a, 0xcc, 0x8c, + 0x1a, 0x79, 0x04, 0x64, 0x2f, 0xe2, 0x41, 0xec, 0x73, 0x09, 0xd6, 0x18, 0xd7, 0xa7, 0x9f, 0x3d, + 0x05, 0x9d, 0x92, 0x9f, 0x19, 0xc6, 0xe4, 0xc3, 0xe2, 0xdd, 0x76, 0x16, 0x11, 0xf5, 0x85, 0x32, + 0x6a, 0x7d, 0x5d, 0x8a, 0x3b, 0xe0, 0xf6, 0xd4, 0xa4, 0x3a, 0x0b, 0x68, 0x78, 0x29, 0x37, 0x2c, + 0x89, 0x59, 0x59, 0x9b, 0x7e, 0x6f, 0xc1, 0x52, 0x11, 0xd9, 0x99, 0x76, 0x4a, 0xd6, 0xf0, 0xca, + 0xfc, 0x77, 0x95, 0x69, 0x78, 0x75, 0xd6, 0x4b, 0xb6, 0x56, 0x7c, 0x6b, 0xa5, 0x70, 0xe9, 0x94, + 0x72, 0xbd, 0x06, 0xa8, 0x0e, 0x34, 0x1f, 0xf2, 0x28, 0xf1, 0xa4, 0x4b, 0xfd, 0x16, 0xa8, 0xb1, + 0x22, 0x8b, 0x1e, 0xc2, 0xe5, 0x13, 0xc3, 0xb7, 0x15, 0x8e, 0xc6, 0x72, 0xca, 0x5f, 0x63, 0x08, + 0xe5, 0x92, 0x8f, 0x22, 0x3d, 0x7e, 0x0d, 0xa6, 0x08, 0xfa, 0x09, 0x5c, 0x1c, 0x88, 0xa4, 0x30, + 0x7a, 0xe6, 0x0e, 0x75, 0xc0, 0xde, 0x15, 0xcf, 0x4f, 0x49, 0x50, 0x8a, 0xe8, 0xe7, 0xe0, 0x3c, + 0x1e, 0x0f, 0x79, 0x22, 0x5e, 0xc9, 0x7a, 0x13, 0xea, 0x7b, 0xe1, 0x38, 0xf4, 0xc3, 0xa7, 0xc7, + 0x73, 0x76, 0x99, 0x03, 0x8b, 0xea, 0x8b, 0xa6, 0x56, 0x67, 0x83, 0x19, 0x92, 0x9e, 0x97, 0xd7, + 0xd4, 0xe5, 0xbe, 0x9b, 0xfa, 0x12, 0x86, 0xfc, 0x51, 0x10, 0x53, 0xa1, 0x2f, 0x02, 0xc7, 0xc2, + 0x15, 0x3e, 0x92, 0x77, 0x91, 0x61, 0x3e, 0x92, 0x8a, 0x22, 0x1f, 0x41, 0xb3, 0xa0, 0xad, 0x0b, + 0x78, 0x71, 0xea, 0xbe, 0x28, 0x21, 0x2b, 0x6a, 0xd2, 0xdf, 0xac, 0x92, 0xe5, 0x89, 0xf7, 0x82, + 0x0e, 0x78, 0xa4, 0x9a, 0x52, 0x67, 0x9a, 0x92, 0xb9, 0x6e, 0x4f, 0x5c, 0x3f, 0x8d, 0xa5, 0x48, + 0x3d, 0x11, 0x72, 0x86, 0xcc, 0x55, 0xfe, 0xf2, 0x0d, 0x53, 0xf3, 0x54, 0x33, 0xa4, 0xfc, 0x11, + 0xda, 0x13, 0x7c, 0xe8, 0x7b, 0x81, 0xc0, 0x29, 0xb5, 0x59, 0x46, 0x93, 0x1b, 0x6a, 0xdb, 0x9b, + 0xab, 0xb6, 0x36, 0x13, 0x3e, 0x6a, 0xa8, 0x2f, 0x41, 0x4c, 0x09, 0xb4, 0xa7, 0x45, 0x9b, 0xed, + 0xdf, 0x5f, 0xae, 0x5b, 0x7f, 0xbc, 0x5c, 0xb7, 0xfe, 0x7c, 0xb9, 0x6e, 0xfd, 0xfc, 0xd7, 0xfa, + 0x1b, 0xfb, 0x0b, 0xf8, 0x5f, 0xc2, 0xad, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x10, 0xd8, 0x2a, + 0xb6, 0x74, 0x10, 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -3093,8 +3093,8 @@ func (m *CreateIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.ETag != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) i-- dAtA[i] = 0x18 } @@ -3144,8 +3144,8 @@ func (m *CreateFieldMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.ETag != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) i-- dAtA[i] = 0x20 } @@ -3289,8 +3289,8 @@ func (m *Field) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.ETag != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) i-- dAtA[i] = 0x20 } @@ -3416,8 +3416,8 @@ func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x22 } } - if m.ETag != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) i-- dAtA[i] = 0x10 } @@ -3726,8 +3726,8 @@ func (m *IndexStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.ETag != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) i-- dAtA[i] = 0x18 } @@ -3779,8 +3779,8 @@ func (m *FieldStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.ETag != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) i-- dAtA[i] = 0x18 } @@ -4839,8 +4839,8 @@ func (m *CreateIndexMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.ETag != 0 { - n += 1 + sovPrivate(uint64(m.ETag)) + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -4866,8 +4866,8 @@ func (m *CreateFieldMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.ETag != 0 { - n += 1 + sovPrivate(uint64(m.ETag)) + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -4938,8 +4938,8 @@ func (m *Field) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.ETag != 0 { - n += 1 + sovPrivate(uint64(m.ETag)) + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -4975,8 +4975,8 @@ func (m *Index) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.ETag != 0 { - n += 1 + sovPrivate(uint64(m.ETag)) + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) } if len(m.Fields) > 0 { for _, e := range m.Fields { @@ -5129,8 +5129,8 @@ func (m *IndexStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.ETag != 0 { - n += 1 + sovPrivate(uint64(m.ETag)) + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -5155,8 +5155,8 @@ func (m *FieldStatus) Size() (n int) { } n += 1 + sovPrivate(uint64(l)) + l } - if m.ETag != 0 { - n += 1 + sovPrivate(uint64(m.ETag)) + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -7121,9 +7121,9 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) } - m.ETag = 0 + m.CreatedAt = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -7133,7 +7133,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ETag |= int64(b&0x7F) << shift + m.CreatedAt |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -7294,9 +7294,9 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) } - m.ETag = 0 + m.CreatedAt = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -7306,7 +7306,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ETag |= int64(b&0x7F) << shift + m.CreatedAt |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -7722,9 +7722,9 @@ func (m *Field) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) } - m.ETag = 0 + m.CreatedAt = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -7734,7 +7734,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ETag |= int64(b&0x7F) << shift + m.CreatedAt |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -7915,9 +7915,9 @@ func (m *Index) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) } - m.ETag = 0 + m.CreatedAt = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -7927,7 +7927,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ETag |= int64(b&0x7F) << shift + m.CreatedAt |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -8858,9 +8858,9 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) } - m.ETag = 0 + m.CreatedAt = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -8870,7 +8870,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ETag |= int64(b&0x7F) << shift + m.CreatedAt |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -9039,9 +9039,9 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) } - m.ETag = 0 + m.CreatedAt = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -9051,7 +9051,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ETag |= int64(b&0x7F) << shift + m.CreatedAt |= int64(b&0x7F) << shift if b < 0x80 { break } diff --git a/internal/private.proto b/internal/private.proto index 502d3b7bd..18494a87f 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -64,14 +64,14 @@ message DeleteIndexMessage { message CreateIndexMessage { string Index = 1; IndexMeta Meta = 2; - int64 ETag = 3; + int64 CreatedAt = 3; } message CreateFieldMessage { string Index = 1; string Field = 2; FieldOptions Meta = 3; - int64 ETag = 4; + int64 CreatedAt = 4; } message DeleteFieldMessage { @@ -89,7 +89,7 @@ message Field { string Name = 1; FieldOptions Meta = 2; repeated string Views = 3; - int64 ETag = 4; + int64 CreatedAt = 4; } message Schema { @@ -98,7 +98,7 @@ message Schema { message Index { string Name = 1; - int64 ETag = 2; + int64 CreatedAt = 2; IndexMeta Options = 5; repeated Field Fields = 4; } @@ -136,13 +136,13 @@ message NodeStatus { message IndexStatus { string Name = 1; repeated FieldStatus Fields = 2; - int64 ETag = 3; + int64 CreatedAt = 3; } message FieldStatus { string Name = 1; repeated uint64 AvailableShards = 2; - int64 ETag = 3; + int64 CreatedAt = 3; } message ClusterStatus { diff --git a/internal/public.pb.go b/internal/public.pb.go index bf04b2d90..826c6fc62 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1183,6 +1183,8 @@ type ImportRequest struct { RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys,proto3" json:"RowKeys,omitempty"` ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys,proto3" json:"ColumnKeys,omitempty"` Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps,proto3" json:"Timestamps,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,9,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` + FieldCreatedAt int64 `protobuf:"varint,10,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1277,6 +1279,20 @@ func (m *ImportRequest) GetTimestamps() []int64 { return nil } +func (m *ImportRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + +func (m *ImportRequest) GetFieldCreatedAt() int64 { + if m != nil { + return m.FieldCreatedAt + } + return 0 +} + type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -1286,6 +1302,8 @@ type ImportValueRequest struct { Values []int64 `protobuf:"varint,6,rep,packed,name=Values,proto3" json:"Values,omitempty"` FloatValues []float64 `protobuf:"fixed64,8,rep,packed,name=FloatValues,proto3" json:"FloatValues,omitempty"` StringValues []string `protobuf:"bytes,9,rep,name=StringValues,proto3" json:"StringValues,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,10,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` + FieldCreatedAt int64 `protobuf:"varint,11,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1380,6 +1398,20 @@ func (m *ImportValueRequest) GetStringValues() []string { return nil } +func (m *ImportValueRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + +func (m *ImportValueRequest) GetFieldCreatedAt() int64 { + if m != nil { + return m.FieldCreatedAt + } + return 0 +} + type TranslateKeysRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -1660,6 +1692,8 @@ type ImportRoaringRequest struct { Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views,proto3" json:"views,omitempty"` Action string `protobuf:"bytes,3,opt,name=Action,proto3" json:"Action,omitempty"` Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,5,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` + FieldCreatedAt int64 `protobuf:"varint,6,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1726,6 +1760,20 @@ func (m *ImportRoaringRequest) GetBlock() uint64 { return 0 } +func (m *ImportRoaringRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + +func (m *ImportRoaringRequest) GetFieldCreatedAt() int64 { + if m != nil { + return m.FieldCreatedAt + } + return 0 +} + type ImportColumnAttrsRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Shard int64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` @@ -1837,83 +1885,86 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1207 bytes of a gzipped FileDescriptorProto + // 1253 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x8e, 0x1b, 0x45, - 0x10, 0xa6, 0x3d, 0xe3, 0xb5, 0x5d, 0xf6, 0x6e, 0x42, 0xc7, 0x09, 0x23, 0x14, 0x36, 0x56, 0x2b, - 0x20, 0xc3, 0x61, 0xa3, 0x0d, 0x21, 0xca, 0x09, 0xc8, 0xc6, 0x1b, 0xb0, 0xa2, 0xac, 0x42, 0x3b, - 0x32, 0x37, 0xa4, 0x59, 0xbb, 0xd9, 0x8c, 0x18, 0xcf, 0x98, 0xf9, 0xc1, 0xd9, 0x23, 0xcf, 0x00, - 0x07, 0xc4, 0x13, 0xf0, 0x28, 0x1c, 0x79, 0x04, 0x58, 0xee, 0x1c, 0xb8, 0x72, 0x41, 0x55, 0x3d, - 0xed, 0x6e, 0x7b, 0xbd, 0x4b, 0x14, 0x71, 0xeb, 0xaf, 0xaa, 0xa6, 0xba, 0xbe, 0xea, 0xea, 0xaa, - 0x1e, 0xe8, 0xcc, 0xcb, 0xe3, 0x38, 0x9a, 0xec, 0xcd, 0xb3, 0xb4, 0x48, 0x79, 0x33, 0x4a, 0x0a, - 0x95, 0x25, 0x61, 0x2c, 0x72, 0xf0, 0x64, 0xba, 0xe0, 0x01, 0x34, 0x1e, 0xa5, 0x71, 0x39, 0x4b, - 0xf2, 0x80, 0xf5, 0xbc, 0xbe, 0x2f, 0x0d, 0xe4, 0x1c, 0xfc, 0x27, 0xea, 0x34, 0x0f, 0xbc, 0x9e, - 0xd7, 0x6f, 0x49, 0x5a, 0xf3, 0xdb, 0x50, 0x7f, 0x58, 0x14, 0x59, 0x1e, 0xd4, 0x7a, 0x5e, 0xbf, - 0x7d, 0x77, 0x67, 0xcf, 0xb8, 0xdb, 0x43, 0xb1, 0xd4, 0x4a, 0xf4, 0x29, 0xd3, 0x30, 0x8b, 0x92, - 0x93, 0xc0, 0xef, 0xb1, 0x7e, 0x47, 0x1a, 0x28, 0x9e, 0x42, 0x6b, 0x14, 0x9d, 0x24, 0x6a, 0x8a, - 0x5b, 0xdf, 0x02, 0xef, 0x59, 0x8a, 0xdb, 0xb2, 0x7e, 0xfb, 0xee, 0xb6, 0x75, 0x25, 0xd3, 0x85, - 0x44, 0x0d, 0x1a, 0x1c, 0xa9, 0x93, 0xa0, 0xb6, 0xd1, 0xe0, 0x48, 0x9d, 0x88, 0x07, 0xb0, 0x23, - 0xd3, 0xc5, 0x70, 0xaa, 0x92, 0x22, 0xfa, 0x3a, 0x52, 0x19, 0x05, 0x2d, 0xd3, 0x85, 0xe1, 0x42, - 0xeb, 0x25, 0x91, 0x9a, 0x25, 0x22, 0x3e, 0x06, 0xff, 0x59, 0x18, 0x65, 0x7c, 0x07, 0x6a, 0xc3, - 0x01, 0x85, 0xe0, 0xcb, 0xda, 0x70, 0xc0, 0xaf, 0x82, 0xf7, 0x44, 0x9d, 0x06, 0x5e, 0x8f, 0xf5, - 0x5b, 0x12, 0x97, 0xbc, 0x0b, 0xf5, 0x47, 0x69, 0x99, 0x14, 0x14, 0x86, 0x2f, 0x35, 0x10, 0x87, - 0xd0, 0xc2, 0xef, 0x1f, 0x47, 0x2a, 0x9e, 0x72, 0xa1, 0x9d, 0x55, 0x4c, 0x9c, 0xa4, 0xa0, 0x54, - 0xea, 0x8d, 0xba, 0x50, 0x27, 0x63, 0x72, 0xd3, 0x92, 0x1a, 0x88, 0xcf, 0x01, 0x50, 0x9b, 0x6b, - 0x3f, 0xb7, 0xa1, 0x4e, 0x88, 0xa2, 0x3f, 0xef, 0x48, 0x2b, 0x2f, 0xf0, 0xf4, 0x0e, 0xd4, 0x87, - 0x49, 0x71, 0xff, 0x1e, 0xaa, 0xc7, 0x61, 0x5c, 0x2a, 0x8a, 0xc6, 0x93, 0x1a, 0x88, 0x12, 0x9a, - 0x64, 0x87, 0x79, 0x5f, 0x3a, 0x60, 0x8e, 0x03, 0x94, 0x62, 0x2e, 0x07, 0x86, 0x27, 0x01, 0x7e, - 0x03, 0xb6, 0x64, 0xba, 0xb0, 0x29, 0xa9, 0x10, 0x7f, 0xd7, 0xec, 0xe2, 0x13, 0xe7, 0x2b, 0x36, - 0x54, 0x8a, 0xc2, 0x6c, 0xfb, 0x15, 0xc0, 0x67, 0x59, 0x5a, 0xce, 0x29, 0x69, 0xbc, 0x0f, 0x75, - 0x42, 0x15, 0x3f, 0x6e, 0x3f, 0x32, 0xb1, 0x49, 0x6d, 0xb0, 0x39, 0xe9, 0x78, 0x38, 0xa3, 0x72, - 0x46, 0x91, 0x78, 0x12, 0x97, 0xe2, 0x7b, 0x06, 0xcd, 0x71, 0x18, 0x2f, 0xd5, 0xe3, 0x30, 0xae, - 0x78, 0xe3, 0x72, 0xd5, 0x8d, 0x67, 0xdc, 0xbc, 0x0d, 0xcd, 0xc7, 0x71, 0x1a, 0x16, 0x68, 0x8c, - 0xbe, 0x98, 0x5c, 0x62, 0xbe, 0x0f, 0x30, 0x50, 0x93, 0x68, 0x16, 0xc6, 0xa8, 0xd5, 0xe4, 0xde, - 0xb4, 0x71, 0x56, 0x3a, 0xe9, 0x18, 0x89, 0x8f, 0xa0, 0x51, 0xa1, 0xcd, 0xb9, 0x47, 0xe9, 0x68, - 0x12, 0xc6, 0xca, 0x44, 0x41, 0x40, 0x7c, 0x09, 0xdb, 0xfa, 0xa6, 0xe1, 0x9d, 0x19, 0xa9, 0xe2, - 0x15, 0x4a, 0xf1, 0x95, 0x6e, 0x9f, 0xf8, 0x85, 0x81, 0x8f, 0x2b, 0xe3, 0x80, 0x59, 0x07, 0x1c, - 0xfc, 0xe7, 0xa7, 0x73, 0x55, 0x65, 0x95, 0xd6, 0xbc, 0x07, 0xed, 0x51, 0x81, 0x97, 0x53, 0x47, - 0xae, 0xb7, 0x73, 0x45, 0x98, 0xaf, 0x61, 0x52, 0xd8, 0xe3, 0xf6, 0xe4, 0x12, 0xf3, 0x9b, 0xd0, - 0x3a, 0x48, 0xd3, 0x58, 0x2b, 0xeb, 0x3d, 0xd6, 0x6f, 0x4a, 0x2b, 0xe0, 0xbb, 0x00, 0x26, 0xb3, - 0xa5, 0x0a, 0xb6, 0x28, 0xd7, 0x8e, 0x44, 0xdc, 0x81, 0x06, 0x46, 0xfa, 0x34, 0x9c, 0x5b, 0x6e, - 0xec, 0x32, 0x6e, 0xff, 0x30, 0xe8, 0x7c, 0x51, 0xaa, 0xec, 0x54, 0xaa, 0x6f, 0x4b, 0x95, 0x17, - 0x98, 0x5b, 0xc2, 0xa6, 0x96, 0x09, 0x60, 0xd5, 0x8e, 0x5e, 0x84, 0xd9, 0x54, 0x67, 0xca, 0x97, - 0x15, 0x42, 0xae, 0x36, 0xe7, 0x39, 0x71, 0x6d, 0x4a, 0x57, 0x44, 0xf5, 0xae, 0x66, 0x69, 0x61, - 0xc8, 0x54, 0x88, 0xf7, 0xe1, 0xca, 0xe1, 0xcb, 0x49, 0x5c, 0x4e, 0x95, 0x4c, 0x17, 0xfa, 0xeb, - 0x2d, 0x32, 0x58, 0x17, 0xf3, 0xf7, 0x60, 0xa7, 0x12, 0x99, 0xbe, 0xda, 0x20, 0xc3, 0x35, 0x29, - 0xdf, 0x87, 0xce, 0xe1, 0xec, 0x58, 0x4d, 0xa7, 0x6a, 0x3a, 0x08, 0x8b, 0x30, 0x68, 0x12, 0xef, - 0xb5, 0x2e, 0xb7, 0x62, 0x22, 0x7e, 0x60, 0xb0, 0x5d, 0xb1, 0xcf, 0xe7, 0x69, 0x92, 0x2b, 0x3c, - 0xe2, 0xc3, 0x2c, 0x33, 0x47, 0x7c, 0x98, 0x65, 0xfc, 0x0e, 0x34, 0xa4, 0xca, 0xcb, 0xb8, 0x30, - 0x55, 0x72, 0xdd, 0x7a, 0x34, 0xdf, 0x96, 0x71, 0x21, 0x8d, 0x15, 0xff, 0x04, 0x76, 0x56, 0xea, - 0x50, 0x37, 0xfc, 0xf6, 0xdd, 0xb7, 0xec, 0x77, 0x2b, 0x7a, 0xb9, 0x66, 0x2e, 0xfe, 0xf2, 0xa0, - 0xed, 0x78, 0x5e, 0x16, 0x19, 0xe6, 0x67, 0xbb, 0x2a, 0xb2, 0x5b, 0x34, 0x6c, 0x2e, 0x68, 0xf5, - 0xd8, 0x93, 0x3a, 0xc0, 0x8e, 0xaa, 0xb2, 0x64, 0x47, 0xb6, 0x11, 0x7a, 0x97, 0x35, 0x42, 0x1c, - 0x5d, 0x2f, 0xc2, 0xe4, 0x44, 0x4d, 0xa9, 0x2c, 0x9b, 0xd2, 0x40, 0xbe, 0x67, 0xbb, 0x02, 0x9d, - 0xe3, 0x4a, 0xaf, 0x31, 0x1a, 0x69, 0x3b, 0x87, 0xee, 0x72, 0xc3, 0x01, 0x9e, 0x15, 0xd5, 0x8b, - 0x46, 0xfc, 0x3e, 0xb4, 0x6d, 0xfb, 0xca, 0xab, 0x23, 0xea, 0x5a, 0x57, 0x56, 0x29, 0x5d, 0x43, - 0xfe, 0xe9, 0xfa, 0x5c, 0x0a, 0x5a, 0x14, 0x45, 0xb0, 0xc2, 0xdc, 0xd1, 0xcb, 0xf5, 0x39, 0xb6, - 0xef, 0x0c, 0xca, 0x00, 0xe8, 0xe3, 0x6b, 0xf6, 0xe3, 0xa5, 0x4a, 0x3a, 0xe3, 0xf4, 0x9e, 0x3b, - 0x4b, 0x82, 0x36, 0x7d, 0xd3, 0x5d, 0xcd, 0x9c, 0xd6, 0x49, 0x77, 0xe6, 0xec, 0x3b, 0x83, 0x2c, - 0xe8, 0xac, 0x6f, 0xb4, 0x54, 0x49, 0x6b, 0x25, 0xfe, 0x60, 0xb0, 0x3d, 0x9c, 0xcd, 0xd3, 0xac, - 0x70, 0x6e, 0xe1, 0x30, 0x99, 0xaa, 0x97, 0xe6, 0x16, 0x12, 0xd8, 0x3c, 0xa8, 0xa8, 0x1b, 0xe2, - 0x6d, 0xa4, 0xdb, 0xe7, 0x4b, 0x0d, 0x9c, 0x13, 0xf0, 0x57, 0x4e, 0xe0, 0x26, 0xb4, 0x74, 0xb9, - 0xa1, 0xaa, 0x4e, 0x2a, 0x2b, 0xd0, 0x0f, 0x8d, 0x05, 0x0d, 0xf7, 0x06, 0x0d, 0x77, 0x03, 0xb1, - 0xf3, 0x68, 0x33, 0x52, 0x36, 0x49, 0xe9, 0x48, 0x50, 0xff, 0x3c, 0x9a, 0xa9, 0xbc, 0x08, 0x67, - 0x73, 0xbc, 0xca, 0x5e, 0xdf, 0x93, 0x8e, 0x44, 0xfc, 0xcd, 0x80, 0x6b, 0x8e, 0xd4, 0xa9, 0xfe, - 0x3f, 0xa2, 0x97, 0x13, 0x5a, 0x0d, 0xbb, 0x71, 0x2e, 0xec, 0x1b, 0xb0, 0x45, 0xf1, 0x98, 0x90, - 0x2b, 0x84, 0x8d, 0xcd, 0xb6, 0x55, 0xcd, 0x97, 0x49, 0x57, 0xc4, 0x05, 0x74, 0x9c, 0x9e, 0x8e, - 0x05, 0x89, 0xbe, 0x57, 0x64, 0x62, 0x0c, 0xdd, 0xe7, 0x59, 0x98, 0xe4, 0x71, 0x58, 0x28, 0xdc, - 0xee, 0x75, 0x58, 0x6f, 0x78, 0x35, 0x8a, 0xf7, 0xe1, 0xfa, 0x9a, 0x5f, 0xdb, 0xbe, 0x30, 0x0d, - 0x1e, 0xa5, 0x01, 0x97, 0x62, 0x04, 0xd7, 0x96, 0xa6, 0xc3, 0xc1, 0x6b, 0x45, 0x70, 0xde, 0xe9, - 0x07, 0x0e, 0x2f, 0x72, 0x5a, 0x6d, 0xbf, 0x29, 0xd6, 0x03, 0x08, 0xaa, 0xda, 0xd6, 0x4f, 0xd6, - 0x2a, 0x82, 0x71, 0xa4, 0x16, 0x68, 0x7f, 0x14, 0xce, 0x54, 0x15, 0x04, 0xad, 0x51, 0x46, 0xed, - 0xbb, 0x46, 0x0f, 0x5d, 0x5a, 0x8b, 0x1f, 0x19, 0x74, 0x37, 0x39, 0xa1, 0xf7, 0x48, 0xac, 0x42, - 0xdd, 0xb0, 0x9b, 0x52, 0x03, 0xfe, 0x00, 0xea, 0xdf, 0x45, 0x6a, 0x61, 0x1a, 0xb6, 0x70, 0xde, - 0x52, 0x17, 0x44, 0x22, 0xf5, 0x07, 0x58, 0x0e, 0x0f, 0x27, 0x45, 0x94, 0x26, 0xe6, 0x75, 0xa6, - 0x11, 0xee, 0x73, 0x10, 0xa7, 0x93, 0x6f, 0xa8, 0x2f, 0xfa, 0x52, 0x03, 0xf1, 0x33, 0x33, 0xdc, - 0x9c, 0x89, 0xf7, 0x9f, 0x19, 0xd6, 0x35, 0x6c, 0x9e, 0x2e, 0x54, 0xc3, 0x81, 0x1e, 0xdb, 0xf6, - 0x75, 0x62, 0x20, 0x3e, 0x15, 0x70, 0x39, 0x0e, 0x63, 0x7d, 0x91, 0x5b, 0x72, 0x89, 0x2f, 0xaf, - 0xfc, 0x83, 0xab, 0xbf, 0x9e, 0xed, 0xb2, 0xdf, 0xce, 0x76, 0xd9, 0xef, 0x67, 0xbb, 0xec, 0xa7, - 0x3f, 0x77, 0xdf, 0x38, 0xde, 0xa2, 0x3f, 0x96, 0x0f, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xfc, - 0x97, 0x67, 0xec, 0xc1, 0x0c, 0x00, 0x00, + 0x10, 0x66, 0x3c, 0xe3, 0xbf, 0xb2, 0x77, 0x13, 0x3a, 0x4e, 0x18, 0xa1, 0xb0, 0xb1, 0x46, 0x01, + 0x19, 0x0e, 0x1b, 0x6d, 0x08, 0x51, 0x4e, 0x40, 0x36, 0xde, 0x80, 0x15, 0x65, 0x15, 0xca, 0x91, + 0xb9, 0x21, 0x4d, 0xec, 0x66, 0x33, 0x62, 0x3c, 0x63, 0xe6, 0x07, 0x67, 0x8f, 0x3c, 0x03, 0x17, + 0xc4, 0x13, 0x70, 0xe5, 0x15, 0x38, 0x71, 0xe4, 0x11, 0x50, 0xe0, 0xcc, 0x0b, 0x70, 0x41, 0x55, + 0x3d, 0xed, 0x6e, 0x3b, 0xde, 0x65, 0x15, 0x71, 0xeb, 0xaf, 0xaa, 0xa6, 0xba, 0xea, 0xeb, 0xea, + 0xaa, 0x1e, 0xe8, 0x2e, 0xca, 0x67, 0x71, 0x34, 0xdd, 0x5f, 0x64, 0x69, 0x91, 0x8a, 0x56, 0x94, + 0x14, 0x32, 0x4b, 0xc2, 0x38, 0xc8, 0xc1, 0xc5, 0x74, 0x29, 0x7c, 0x68, 0x3e, 0x48, 0xe3, 0x72, + 0x9e, 0xe4, 0xbe, 0xd3, 0x77, 0x07, 0x1e, 0x6a, 0x28, 0x04, 0x78, 0x8f, 0xe4, 0x69, 0xee, 0xbb, + 0x7d, 0x77, 0xd0, 0x46, 0x5e, 0x8b, 0x9b, 0x50, 0xbf, 0x5f, 0x14, 0x59, 0xee, 0xd7, 0xfa, 0xee, + 0xa0, 0x73, 0x7b, 0x77, 0x5f, 0xbb, 0xdb, 0x27, 0x31, 0x2a, 0x25, 0xf9, 0xc4, 0x34, 0xcc, 0xa2, + 0xe4, 0xc4, 0xf7, 0xfa, 0xce, 0xa0, 0x8b, 0x1a, 0x06, 0x8f, 0xa1, 0x3d, 0x8e, 0x4e, 0x12, 0x39, + 0xa3, 0xad, 0x6f, 0x80, 0xfb, 0x24, 0xa5, 0x6d, 0x9d, 0x41, 0xe7, 0xf6, 0x8e, 0x71, 0x85, 0xe9, + 0x12, 0x49, 0x43, 0x06, 0xc7, 0xf2, 0xc4, 0xaf, 0x6d, 0x35, 0x38, 0x96, 0x27, 0xc1, 0x3d, 0xd8, + 0xc5, 0x74, 0x39, 0x9a, 0xc9, 0xa4, 0x88, 0xbe, 0x8e, 0x64, 0xc6, 0x41, 0x63, 0xba, 0xd4, 0xb9, + 0xf0, 0x7a, 0x95, 0x48, 0xcd, 0x24, 0x12, 0x7c, 0x0c, 0xde, 0x93, 0x30, 0xca, 0xc4, 0x2e, 0xd4, + 0x46, 0x43, 0x0e, 0xc1, 0xc3, 0xda, 0x68, 0x28, 0x2e, 0x83, 0xfb, 0x48, 0x9e, 0xfa, 0x6e, 0xdf, + 0x19, 0xb4, 0x91, 0x96, 0xa2, 0x07, 0xf5, 0x07, 0x69, 0x99, 0x14, 0x1c, 0x86, 0x87, 0x0a, 0x04, + 0x47, 0xd0, 0xa6, 0xef, 0x1f, 0x46, 0x32, 0x9e, 0x89, 0x40, 0x39, 0xab, 0x32, 0xb1, 0x48, 0x21, + 0x29, 0xaa, 0x8d, 0x7a, 0x50, 0x67, 0x63, 0x76, 0xd3, 0x46, 0x05, 0x82, 0xcf, 0x01, 0x48, 0x9b, + 0x2b, 0x3f, 0x37, 0xa1, 0xce, 0x88, 0xa3, 0x7f, 0xd5, 0x91, 0x52, 0x9e, 0xe1, 0xe9, 0x1d, 0xa8, + 0x8f, 0x92, 0xe2, 0xee, 0x1d, 0x52, 0x4f, 0xc2, 0xb8, 0x94, 0x1c, 0x8d, 0x8b, 0x0a, 0x04, 0x25, + 0xb4, 0xd8, 0x8e, 0x78, 0x5f, 0x39, 0x70, 0x2c, 0x07, 0x24, 0x25, 0x2e, 0x87, 0x3a, 0x4f, 0x06, + 0xe2, 0x1a, 0x34, 0x30, 0x5d, 0x1a, 0x4a, 0x2a, 0x24, 0xde, 0xd5, 0xbb, 0x78, 0x9c, 0xf3, 0x25, + 0x13, 0x2a, 0x47, 0xa1, 0xb7, 0xfd, 0x0a, 0xe0, 0xb3, 0x2c, 0x2d, 0x17, 0x4c, 0x9a, 0x18, 0x40, + 0x9d, 0x51, 0x95, 0x9f, 0x30, 0x1f, 0xe9, 0xd8, 0x50, 0x19, 0x6c, 0x27, 0x9d, 0x0e, 0x67, 0x5c, + 0xce, 0x39, 0x12, 0x17, 0x69, 0x19, 0x7c, 0xef, 0x40, 0x6b, 0x12, 0xc6, 0x2b, 0xf5, 0x24, 0x8c, + 0xab, 0xbc, 0x69, 0xb9, 0xee, 0xc6, 0xd5, 0x6e, 0xde, 0x86, 0xd6, 0xc3, 0x38, 0x0d, 0x0b, 0x32, + 0x26, 0x5f, 0x0e, 0xae, 0xb0, 0x38, 0x00, 0x18, 0xca, 0x69, 0x34, 0x0f, 0x63, 0xd2, 0xaa, 0xe4, + 0xde, 0x34, 0x71, 0x56, 0x3a, 0xb4, 0x8c, 0x82, 0x8f, 0xa0, 0x59, 0xa1, 0xed, 0xdc, 0x93, 0x74, + 0x3c, 0x0d, 0x63, 0xa9, 0xa3, 0x60, 0x10, 0x7c, 0x09, 0x3b, 0xea, 0xa6, 0xd1, 0x9d, 0x19, 0xcb, + 0xe2, 0x02, 0xa5, 0x78, 0xa1, 0xdb, 0x17, 0xfc, 0xec, 0x80, 0x47, 0x2b, 0xed, 0xc0, 0x31, 0x0e, + 0x04, 0x78, 0x4f, 0x4f, 0x17, 0xb2, 0x62, 0x95, 0xd7, 0xa2, 0x0f, 0x9d, 0x71, 0x41, 0x97, 0x53, + 0x45, 0xae, 0xb6, 0xb3, 0x45, 0xc4, 0xd7, 0x28, 0x29, 0xcc, 0x71, 0xbb, 0xb8, 0xc2, 0xe2, 0x3a, + 0xb4, 0x0f, 0xd3, 0x34, 0x56, 0xca, 0x7a, 0xdf, 0x19, 0xb4, 0xd0, 0x08, 0xc4, 0x1e, 0x80, 0x66, + 0xb6, 0x94, 0x7e, 0x83, 0xb9, 0xb6, 0x24, 0xc1, 0x2d, 0x68, 0x52, 0xa4, 0x8f, 0xc3, 0x85, 0xc9, + 0xcd, 0x39, 0x2f, 0xb7, 0x7f, 0x1c, 0xe8, 0x7e, 0x51, 0xca, 0xec, 0x14, 0xe5, 0xb7, 0xa5, 0xcc, + 0x0b, 0xe2, 0x96, 0xb1, 0xae, 0x65, 0x06, 0x54, 0xb5, 0xe3, 0xe7, 0x61, 0x36, 0x53, 0x4c, 0x79, + 0x58, 0x21, 0xca, 0xd5, 0x70, 0x9e, 0x73, 0xae, 0x2d, 0xb4, 0x45, 0x5c, 0xef, 0x72, 0x9e, 0x16, + 0x3a, 0x99, 0x0a, 0x89, 0x01, 0x5c, 0x3a, 0x7a, 0x31, 0x8d, 0xcb, 0x99, 0xc4, 0x74, 0xa9, 0xbe, + 0x6e, 0xb0, 0xc1, 0xa6, 0x58, 0xbc, 0x07, 0xbb, 0x95, 0x48, 0xf7, 0xd5, 0x26, 0x1b, 0x6e, 0x48, + 0xc5, 0x01, 0x74, 0x8f, 0xe6, 0xcf, 0xe4, 0x6c, 0x26, 0x67, 0xc3, 0xb0, 0x08, 0xfd, 0x16, 0xe7, + 0xbd, 0xd1, 0xe5, 0xd6, 0x4c, 0x82, 0x1f, 0x1c, 0xd8, 0xa9, 0xb2, 0xcf, 0x17, 0x69, 0x92, 0x4b, + 0x3a, 0xe2, 0xa3, 0x2c, 0xd3, 0x47, 0x7c, 0x94, 0x65, 0xe2, 0x16, 0x34, 0x51, 0xe6, 0x65, 0x5c, + 0xe8, 0x2a, 0xb9, 0x6a, 0x3c, 0xea, 0x6f, 0xcb, 0xb8, 0x40, 0x6d, 0x25, 0x3e, 0x81, 0xdd, 0xb5, + 0x3a, 0x54, 0x0d, 0xbf, 0x73, 0xfb, 0x2d, 0xf3, 0xdd, 0x9a, 0x1e, 0x37, 0xcc, 0x83, 0xbf, 0x5d, + 0xe8, 0x58, 0x9e, 0x57, 0x45, 0x46, 0xfc, 0xec, 0x54, 0x45, 0x76, 0x83, 0x87, 0xcd, 0x19, 0xad, + 0x9e, 0x7a, 0x52, 0x17, 0x9c, 0xe3, 0xaa, 0x2c, 0x9d, 0x63, 0xd3, 0x08, 0xdd, 0xf3, 0x1a, 0x21, + 0x8d, 0xae, 0xe7, 0x61, 0x72, 0x22, 0x67, 0x5c, 0x96, 0x2d, 0xd4, 0x50, 0xec, 0x9b, 0xae, 0xc0, + 0xe7, 0xb8, 0xd6, 0x6b, 0xb4, 0x06, 0x4d, 0xe7, 0x50, 0x5d, 0x6e, 0x34, 0xa4, 0xb3, 0xe2, 0x7a, + 0x51, 0x48, 0xdc, 0x85, 0x8e, 0x69, 0x5f, 0x79, 0x75, 0x44, 0x3d, 0xe3, 0xca, 0x28, 0xd1, 0x36, + 0x14, 0x9f, 0x6e, 0xce, 0x25, 0xbf, 0xcd, 0x51, 0xf8, 0x6b, 0x99, 0x5b, 0x7a, 0xdc, 0x9c, 0x63, + 0x07, 0xd6, 0xa0, 0xf4, 0x81, 0x3f, 0xbe, 0x62, 0x3e, 0x5e, 0xa9, 0xd0, 0x1a, 0xa7, 0x77, 0xec, + 0x59, 0xe2, 0x77, 0xf8, 0x9b, 0xde, 0x3a, 0x73, 0x4a, 0x87, 0xf6, 0xcc, 0x39, 0xb0, 0x06, 0x99, + 0xdf, 0xdd, 0xdc, 0x68, 0xa5, 0x42, 0x63, 0x15, 0xfc, 0x52, 0x83, 0x9d, 0xd1, 0x7c, 0x91, 0x66, + 0x85, 0x75, 0x0b, 0x47, 0xc9, 0x4c, 0xbe, 0xd0, 0xb7, 0x90, 0xc1, 0xf6, 0x41, 0xc5, 0xdd, 0x90, + 0x6e, 0x23, 0xdf, 0x3e, 0x0f, 0x15, 0xb0, 0x4e, 0xc0, 0x5b, 0x3b, 0x81, 0xeb, 0xd0, 0x56, 0xe5, + 0x46, 0xaa, 0x3a, 0xab, 0x8c, 0x40, 0x3d, 0x34, 0x96, 0x3c, 0xdc, 0x9b, 0x3c, 0xdc, 0x35, 0xa4, + 0xce, 0xa3, 0xcc, 0x58, 0xd9, 0x62, 0xa5, 0x25, 0x21, 0xfd, 0xd3, 0x68, 0x2e, 0xf3, 0x22, 0x9c, + 0x2f, 0xe8, 0x2a, 0xbb, 0x03, 0x17, 0x2d, 0x09, 0xdd, 0x62, 0x4e, 0xe2, 0x41, 0x26, 0xc3, 0x42, + 0xce, 0xee, 0x17, 0x7c, 0x82, 0x2e, 0x6e, 0x48, 0xc9, 0x8e, 0xd3, 0x32, 0x76, 0xa0, 0xec, 0xd6, + 0xa5, 0xc1, 0xaf, 0x35, 0x10, 0x8a, 0x33, 0xee, 0x7c, 0xff, 0x1f, 0x71, 0xe7, 0x13, 0xb4, 0x4e, + 0x43, 0xf3, 0x15, 0x1a, 0xae, 0x41, 0x83, 0xe3, 0xd1, 0x14, 0x54, 0x88, 0x1a, 0xa5, 0x69, 0xd3, + 0x8a, 0x3f, 0x07, 0x6d, 0x91, 0x08, 0xa0, 0x6b, 0xcd, 0x08, 0x2a, 0x70, 0xf2, 0xbd, 0x26, 0xdb, + 0x42, 0x22, 0x5c, 0x90, 0xc4, 0xce, 0x56, 0x12, 0x27, 0xd0, 0x7b, 0x9a, 0x85, 0x49, 0x1e, 0x87, + 0x85, 0xa4, 0xf0, 0x5f, 0x87, 0xc5, 0x2d, 0xaf, 0xda, 0xe0, 0x7d, 0xb8, 0xba, 0xe1, 0xd7, 0xb4, + 0x57, 0xa2, 0xd5, 0x65, 0x5a, 0x69, 0x19, 0x8c, 0xe1, 0xca, 0xca, 0x74, 0x34, 0x7c, 0xad, 0x08, + 0x5e, 0x75, 0xfa, 0x81, 0x95, 0x17, 0x3b, 0xad, 0xb6, 0xdf, 0x16, 0xeb, 0x21, 0xf8, 0xd5, 0xdd, + 0x53, 0x4f, 0xea, 0x2a, 0x82, 0x49, 0x24, 0x97, 0x64, 0x7f, 0x1c, 0xce, 0x65, 0x15, 0x04, 0xaf, + 0x49, 0xc6, 0xe3, 0xa5, 0xc6, 0x0f, 0x71, 0x5e, 0x07, 0x7f, 0x39, 0xd0, 0xdb, 0xe6, 0x84, 0xdf, + 0x4b, 0xb1, 0x0c, 0xd5, 0x40, 0x69, 0xa1, 0x02, 0xe2, 0x1e, 0xd4, 0xbf, 0x8b, 0xe4, 0x52, 0x0f, + 0x94, 0xc0, 0x7a, 0xeb, 0x9d, 0x11, 0x09, 0xaa, 0x0f, 0xa8, 0xbc, 0xee, 0x4f, 0x8b, 0x28, 0x4d, + 0xf4, 0xeb, 0x51, 0x21, 0xda, 0xe7, 0x30, 0x4e, 0xa7, 0xdf, 0x70, 0xdf, 0xf6, 0x50, 0x81, 0x2d, + 0xe5, 0x52, 0xbf, 0x60, 0xb9, 0x34, 0xb6, 0x96, 0xcb, 0x4f, 0x8e, 0xe6, 0xca, 0x9a, 0xf0, 0xff, + 0x79, 0x62, 0xea, 0x8e, 0xe9, 0xa7, 0x1a, 0xdf, 0x31, 0x5f, 0x3d, 0x53, 0xcc, 0x6b, 0x4c, 0x43, + 0x7a, 0x1a, 0xd1, 0x72, 0x12, 0xc6, 0xaa, 0x71, 0xb5, 0x71, 0x85, 0xcf, 0xbf, 0x99, 0x87, 0x97, + 0x7f, 0x7b, 0xb9, 0xe7, 0xfc, 0xfe, 0x72, 0xcf, 0xf9, 0xe3, 0xe5, 0x9e, 0xf3, 0xe3, 0x9f, 0x7b, + 0x6f, 0x3c, 0x6b, 0xf0, 0x1f, 0xda, 0x87, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x91, 0x85, + 0xb9, 0xb1, 0x0d, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -2985,6 +3036,16 @@ func (m *ImportRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FieldCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) + i-- + dAtA[i] = 0x50 + } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x48 + } if len(m.ColumnKeys) > 0 { for iNdEx := len(m.ColumnKeys) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.ColumnKeys[iNdEx]) @@ -3104,6 +3165,16 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FieldCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) + i-- + dAtA[i] = 0x58 + } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x50 + } if len(m.StringValues) > 0 { for iNdEx := len(m.StringValues) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.StringValues[iNdEx]) @@ -3446,6 +3517,16 @@ func (m *ImportRoaringRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FieldCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) + i-- + dAtA[i] = 0x30 + } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x28 + } if m.Block != 0 { i = encodeVarintPublic(dAtA, i, uint64(m.Block)) i-- @@ -4080,6 +4161,12 @@ func (m *ImportRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } + if m.FieldCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.FieldCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4132,6 +4219,12 @@ func (m *ImportValueRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } + if m.FieldCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.FieldCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4270,6 +4363,12 @@ func (m *ImportRoaringRequest) Size() (n int) { if m.Block != 0 { n += 1 + sovPublic(uint64(m.Block)) } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } + if m.FieldCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.FieldCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7525,6 +7624,44 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } m.ColumnKeys = append(m.ColumnKeys, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType) + } + m.FieldCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FieldCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -7932,6 +8069,44 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } m.StringValues = append(m.StringValues, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType) + } + m.FieldCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FieldCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -8771,6 +8946,44 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { break } } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType) + } + m.FieldCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FieldCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/internal/public.proto b/internal/public.proto index 81220fbfe..f1a1fa9fa 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -116,6 +116,7 @@ message QueryResult { message ImportRequest { string Index = 1; + string Field = 2; uint64 Shard = 3; repeated uint64 RowIDs = 4; @@ -123,6 +124,8 @@ message ImportRequest { repeated string RowKeys = 7; repeated string ColumnKeys = 8; repeated int64 Timestamps = 6; + int64 IndexCreatedAt = 9; + int64 FieldCreatedAt = 10; } message ImportValueRequest { @@ -134,6 +137,8 @@ message ImportValueRequest { repeated int64 Values = 6; repeated double FloatValues = 8; repeated string StringValues = 9; + int64 IndexCreatedAt = 10; + int64 FieldCreatedAt = 11; } message TranslateKeysRequest { @@ -166,6 +171,9 @@ message ImportRoaringRequest { repeated ImportRoaringRequestView views = 2; string Action = 3; uint64 Block = 4; + int64 IndexCreatedAt = 5; + int64 FieldCreatedAt = 6; + } message ImportColumnAttrsRequest { diff --git a/pilosa.go b/pilosa.go index e49af5406..060141467 100644 --- a/pilosa.go +++ b/pilosa.go @@ -66,6 +66,9 @@ var ( // we won't need this error at all by 2.0 though. ErrClusterDoesNotOwnShard = errors.New("node does not own shard") + // ErrPreconditionFailed is returned when specified index/field createdAt timestamps don't match + ErrPreconditionFailed = errors.New("precondition failed") + ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") ErrResizeNotRunning = errors.New("no resize job currently running") @@ -125,6 +128,15 @@ func newNotFoundError(err error) NotFoundError { return NotFoundError{err} } +type PreconditionFailedError struct { + error +} + +// newPreconditionFailedError returns err wrapped in a PreconditionFailedError. +func newPreconditionFailedError(err error) PreconditionFailedError { + return PreconditionFailedError{err} +} + // Regular expression to validate index and field names. var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,229}$`) @@ -192,7 +204,7 @@ func stringSlicesAreEqual(a, b []string) bool { return true } -func newETag() int64 { +func timestamp() int64 { return time.Now().UTC().UnixNano() } diff --git a/server.go b/server.go index 586d14e73..8684a6061 100644 --- a/server.go +++ b/server.go @@ -718,7 +718,7 @@ func (s *Server) receiveMessage(m Message) error { return err } idx.mu.Lock() - idx.etag = obj.ETag + idx.createdAt = obj.CreatedAt idx.mu.Unlock() case *DeleteIndexMessage: if err := s.holder.DeleteIndex(obj.Index); err != nil { @@ -735,7 +735,7 @@ func (s *Server) receiveMessage(m Message) error { return err } fld.mu.Lock() - fld.etag = obj.ETag + fld.createdAt = obj.CreatedAt fld.mu.Unlock() case *DeleteFieldMessage: idx := s.holder.Index(obj.Index) diff --git a/server/handler_test.go b/server/handler_test.go index 9cd25f50e..479856fad 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -195,9 +195,9 @@ func TestHandler_Endpoints(t *testing.T) { if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } - body := w.Body.String() - target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]} -`, pilosa.ShardWidth) + + body := strings.TrimSpace(w.Body.String()) + target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) if body != target { t.Fatalf("%s != %s", target, body) } @@ -213,9 +213,11 @@ func TestHandler_Endpoints(t *testing.T) { idx := indexInfo[0] fld := indexInfo[0].Fields[0] msg := pilosa.ImportRequest{ - Index: idx.Name, - Field: fld.Name, - Shard: 0, + Index: idx.Name, + IndexCreatedAt: idx.CreatedAt, + Field: fld.Name, + FieldCreatedAt: fld.CreatedAt, + Shard: 0, } ser := proto.Serializer{} data, err := ser.Marshal(&msg) @@ -223,29 +225,30 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatal(err) } path := fmt.Sprintf("/index/%s/field/%s/import", idx.Name, fld.Name) - etag := fmt.Sprintf("%d, %d", idx.ETag, fld.ETag) - httpReq := test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data)) httpReq.Header.Set("Content-Type", "application/x-protobuf") httpReq.Header.Set("Accept", "application/x-protobuf") - httpReq.Header.Set("If-Match", etag) w := httptest.NewRecorder() h.ServeHTTP(w, httpReq) - if w.Body.String() != "" { + if w.Code != 200 { t.Fatalf(w.Body.String()) } - etag = "invalid-index-etag, invalid-field-etag" - + msg.IndexCreatedAt = -idx.CreatedAt + msg.FieldCreatedAt = -fld.CreatedAt + data, err = ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } httpReq = test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data)) httpReq.Header.Set("Content-Type", "application/x-protobuf") httpReq.Header.Set("Accept", "application/x-protobuf") - httpReq.Header.Set("If-Match", etag) + w = httptest.NewRecorder() h.ServeHTTP(w, httpReq) - if strings.TrimSpace(w.Body.String()) != "Precondition Failed" { + if w.Code != 412 { t.Fatal("expected: Precondition Failed, got:" + w.Body.String()) } }) @@ -253,17 +256,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("ImportRoaring", func(t *testing.T) { w := httptest.NewRecorder() roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - msg := pilosa.ImportRoaringRequest{ - Clear: false, - Views: map[string][]byte{ - "": roaringData, - }, - } - ser := proto.Serializer{} - data, err := ser.Marshal(&msg) - if err != nil { - t.Fatal(err) - } + idx, err := cmd.API.Index(context.Background(), "i0") if err != nil { t.Fatal(err) @@ -273,12 +266,25 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatal(err) } + msg := pilosa.ImportRoaringRequest{ + IndexCreatedAt: idx.CreatedAt(), + FieldCreatedAt: fld.CreatedAt(), + Clear: false, + Views: map[string][]byte{ + "": roaringData, + }, + } + ser := proto.Serializer{} + data, err := ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(data)) httpReq.Header.Set("Content-Type", "application/x-protobuf") httpReq.Header.Set("Accept", "application/x-protobuf") - httpReq.Header.Set("If-Match", fmt.Sprintf("%d, %d", idx.ETag(), fld.ETag())) h.ServeHTTP(w, httpReq) - if w.Body.String() != "" { + if w.Code != 200 { t.Fatalf("Unexpected response body: %s", w.Body.String()) } resp, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i0", Query: "TopN(f1)"}) @@ -821,8 +827,15 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader(""))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected response body: %s", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // Verify index is gone. if hldr.Index("i") != nil { @@ -839,10 +852,18 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f1", strings.NewReader(""))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) - } else if body := w.Body.String(); body != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } else if f := hldr.Index("i").Field("f1"); f != nil { - t.Fatal("expected nil field") + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + if f := hldr.Index("i").Field("f1"); f != nil { + t.Fatal("expected nil field") + } } }) @@ -1020,8 +1041,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // create index again @@ -1030,8 +1057,16 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusConflict { t.Errorf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"creating index: index already exists"}}`+"\n" { - t.Errorf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if resp.Success || resp.Name == "" || resp.CreatedAt == 0 { + t.Errorf("unexpected body: %q", w.Body.String()) + } } // create field @@ -1040,8 +1075,16 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success || resp.Name == "" || resp.CreatedAt == 0 { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // create field again @@ -1050,8 +1093,16 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusConflict { t.Errorf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"creating field: field already exists"}}`+"\n" { - t.Errorf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if resp.Success || resp.Name == "" || resp.CreatedAt == 0 { + t.Errorf("unexpected body: %q", w.Body.String()) + } } // delete field @@ -1060,8 +1111,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // delete field again @@ -1080,8 +1137,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // delete index again @@ -1090,8 +1153,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusNotFound { t.Errorf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"deleting index: index not found"}}`+"\n" { - t.Errorf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } }) @@ -1102,8 +1171,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // create field @@ -1112,8 +1187,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // set some bits From 0c98c887acfd3eb3a1211006186d97bb5d969ad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Thu, 28 May 2020 16:28:30 +0200 Subject: [PATCH 16/45] Pass Schema in ClusterStatus message --- api.go | 6 + api_test.go | 46 ++++---- cluster.go | 5 + encoding/proto/proto.go | 15 ++- handler.go | 11 +- holder.go | 26 +++++ http/handler.go | 7 +- internal/private.pb.go | 245 +++++++++++++++++++++++++--------------- internal/private.proto | 1 + internal/public.pb.go | 191 ++++++++++++++++++------------- internal/public.proto | 1 + 11 files changed, 351 insertions(+), 203 deletions(-) diff --git a/api.go b/api.go index 0a29ad30a..54bb8f3be 100644 --- a/api.go +++ b/api.go @@ -1302,6 +1302,12 @@ func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsReq return errors.Wrap(err, "validating shard ownership") } + if req.IndexCreatedAt != 0 { + if index.CreatedAt() != req.IndexCreatedAt { + return ErrPreconditionFailed + } + } + bulkAttrs := make(map[uint64]map[string]interface{}) for n := 0; n < len(req.ColumnIDs); n++ { bulkAttrs[uint64(req.ColumnIDs[n])] = map[string]interface{}{req.AttrKey: req.AttrVals[n]} diff --git a/api_test.go b/api_test.go index e8b4f28f5..02e8a182f 100644 --- a/api_test.go +++ b/api_test.go @@ -63,15 +63,15 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { m1 := c[1] t.Run("ImportColumnAttrs", func(t *testing.T) { ctx := context.Background() - index := "i" - field := "f" + indexName := "i" + fieldName := "f" attrKey := "k" - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{}) + index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field) + _, err = m0.API.CreateField(ctx, indexName, fieldName) if err != nil { t.Fatalf("creating field: %v", err) } @@ -86,27 +86,28 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { columnIDs0 = append(columnIDs0, uint64(n)) val0 := attrFun(uint64(n)) attrVals0 = append(attrVals0, val0) - setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, field) - if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql0}); err != nil { + setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, fieldName) + if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql0}); err != nil { t.Fatal(err) } columnIDs1 = append(columnIDs1, uint64(n+ShardWidth)) val1 := attrFun(uint64(n + ShardWidth)) attrVals1 = append(attrVals1, val1) - setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, field) - if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql1}); err != nil { + setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, fieldName) + if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql1}); err != nil { t.Fatal(err) } } // send shard0 to node1 req := &pilosa.ImportColumnAttrsRequest{ - AttrKey: attrKey, - ColumnIDs: columnIDs0, - AttrVals: attrVals0, - Shard: 0, - Index: index, + AttrKey: attrKey, + ColumnIDs: columnIDs0, + AttrVals: attrVals0, + Shard: 0, + Index: indexName, + IndexCreatedAt: index.CreatedAt(), } if err := m1.API.ImportColumnAttrs(ctx, req); err != nil { @@ -115,11 +116,12 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { // send shard1 to node0 req = &pilosa.ImportColumnAttrsRequest{ - AttrKey: attrKey, - ColumnIDs: columnIDs1, - AttrVals: attrVals1, - Shard: 1, - Index: index, + AttrKey: attrKey, + ColumnIDs: columnIDs1, + AttrVals: attrVals1, + Shard: 1, + Index: indexName, + IndexCreatedAt: index.CreatedAt(), } if err := m0.API.ImportColumnAttrs(ctx, req); err != nil { @@ -127,8 +129,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { } // Query node0. - pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field) - res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}) + pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName) + res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}) if err != nil { t.Fatal(err) } @@ -143,8 +145,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { } } // Query node1. - pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field) - res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}) + pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName) + res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}) if err != nil { t.Fatal(err) } diff --git a/cluster.go b/cluster.go index d283593a1..ced55e67a 100644 --- a/cluster.go +++ b/cluster.go @@ -610,6 +610,7 @@ func (c *cluster) unprotectedStatus() *ClusterStatus { ClusterID: c.id, State: c.state, Nodes: c.nodes, + Schema: &Schema{Indexes: c.holder.Schema()}, } } @@ -2213,6 +2214,9 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { } } + if cs.Schema != nil { + c.holder.applyCreatedAt(cs.Schema.Indexes) + } c.unprotectedSetState(cs.State) c.markAsJoined() @@ -2449,6 +2453,7 @@ type ClusterStatus struct { ClusterID string State string Nodes []*Node + Schema *Schema } // ResizeInstruction contains the instruction provided to a node diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 7d647ded0..82a31ce05 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -453,11 +453,12 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) * func (s Serializer) encodeImportColumnAttrsRequest(m *pilosa.ImportColumnAttrsRequest) *internal.ImportColumnAttrsRequest { return &internal.ImportColumnAttrsRequest{ - Index: m.Index, - Shard: m.Shard, - AttrKey: m.AttrKey, - AttrVals: m.AttrVals, - ColumnIDs: m.ColumnIDs, + Index: m.Index, + IndexCreatedAt: m.IndexCreatedAt, + Shard: m.Shard, + AttrKey: m.AttrKey, + AttrVals: m.AttrVals, + ColumnIDs: m.ColumnIDs, } } @@ -680,6 +681,7 @@ func (s Serializer) encodeClusterStatus(m *pilosa.ClusterStatus) *internal.Clust State: m.State, ClusterID: m.ClusterID, Nodes: s.encodeNodes(m.Nodes), + Schema: s.encodeSchema(m.Schema), } } @@ -1006,6 +1008,8 @@ func (s Serializer) decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.Cl m.ClusterID = cs.ClusterID m.Nodes = make([]*pilosa.Node, len(cs.Nodes)) s.decodeNodes(cs.Nodes, m.Nodes) + m.Schema = &pilosa.Schema{} + s.decodeSchema(cs.Schema, m.Schema) } func (s Serializer) decodeNode(node *internal.Node, m *pilosa.Node) { @@ -1199,6 +1203,7 @@ func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest func (s Serializer) decodeImportColumnAttrsRequest(pb *internal.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) { m.Index = pb.Index + m.IndexCreatedAt = pb.IndexCreatedAt m.Shard = pb.Shard m.AttrKey = pb.AttrKey m.AttrVals = pb.AttrVals diff --git a/handler.go b/handler.go index 79c60e285..17156245b 100644 --- a/handler.go +++ b/handler.go @@ -178,11 +178,12 @@ func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreate // ImportColumnAttrsRequest describes the import request structure // for a ColumnAttr import type ImportColumnAttrsRequest struct { - AttrKey string - ColumnIDs []uint64 - AttrVals []string - Shard int64 - Index string + AttrKey string + ColumnIDs []uint64 + AttrVals []string + Shard int64 + Index string + IndexCreatedAt int64 } // ImportRequest describes the import request structure diff --git a/holder.go b/holder.go index dd995566a..70501a4b1 100644 --- a/holder.go +++ b/holder.go @@ -475,6 +475,32 @@ func (h *Holder) applySchema(schema *Schema) error { return nil } +func (h *Holder) applyCreatedAt(indexes []*IndexInfo) { + for _, ii := range indexes { + idx := h.Index(ii.Name) + if idx == nil { + continue + } + if ii.CreatedAt != 0 { + idx.mu.Lock() + idx.createdAt = ii.CreatedAt + idx.mu.Unlock() + } + + for _, fi := range ii.Fields { + fld := idx.Field(fi.Name) + if fld == nil { + continue + } + if fi.CreatedAt != 0 { + fld.mu.Lock() + fld.createdAt = fi.CreatedAt + fld.mu.Unlock() + } + } + } +} + // IndexPath returns the path where a given index is stored. func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) } diff --git a/http/handler.go b/http/handler.go index e2787d77c..12773f3ee 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1935,7 +1935,12 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req } if err := h.api.ImportColumnAttrs(r.Context(), req, opts...); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } diff --git a/internal/private.pb.go b/internal/private.pb.go index ea281b409..6dd3eb9bc 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1499,6 +1499,7 @@ type ClusterStatus struct { ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes,proto3" json:"Nodes,omitempty"` + Schema *Schema `protobuf:"bytes,4,opt,name=Schema,proto3" json:"Schema,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1558,6 +1559,13 @@ func (m *ClusterStatus) GetNodes() []*Node { return nil } +func (m *ClusterStatus) GetSchema() *Schema { + if m != nil { + return m.Schema + } + return nil +} + type BSIGroup struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` @@ -2469,98 +2477,99 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1445 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x6f, 0x1b, 0x45, - 0x14, 0x66, 0xbd, 0x76, 0x62, 0x1f, 0xc7, 0xa9, 0x33, 0x6d, 0xd3, 0x6d, 0x40, 0xc1, 0x0c, 0x15, - 0x35, 0x95, 0x1a, 0xaa, 0x16, 0x89, 0x6b, 0xa5, 0x36, 0x71, 0x5a, 0x0c, 0x24, 0x6d, 0xc7, 0x69, - 0xdf, 0x27, 0xeb, 0x51, 0xb3, 0xca, 0x7a, 0xd7, 0xdd, 0x4b, 0xea, 0x14, 0x89, 0x57, 0x10, 0x3c, - 0xf1, 0xc6, 0x03, 0x0f, 0xfc, 0x0f, 0xfe, 0x00, 0x8f, 0xfc, 0x04, 0x54, 0x7e, 0x05, 0x6f, 0x68, - 0xce, 0xcc, 0xec, 0xc5, 0x71, 0x70, 0x68, 0x79, 0x9b, 0x73, 0xff, 0xce, 0x65, 0xce, 0x8e, 0x0d, - 0xad, 0x71, 0xe4, 0x1d, 0xf1, 0x44, 0x6c, 0x8c, 0xa3, 0x30, 0x09, 0x49, 0xdd, 0x0b, 0x12, 0x11, - 0x05, 0xdc, 0x5f, 0x5b, 0x1a, 0xa7, 0xfb, 0xbe, 0xe7, 0x2a, 0x3e, 0xbd, 0x0f, 0x8d, 0x7e, 0x30, - 0x14, 0x93, 0x1d, 0x91, 0x70, 0x42, 0xa0, 0xfa, 0x95, 0x38, 0x8e, 0x1d, 0xbb, 0x63, 0x75, 0xeb, - 0x0c, 0xcf, 0xe4, 0x3d, 0x58, 0xde, 0x8b, 0xb8, 0x7b, 0xb8, 0x3d, 0xf1, 0xe2, 0x44, 0x04, 0xae, - 0x70, 0xaa, 0x28, 0x9d, 0xe2, 0xd2, 0x5f, 0x6d, 0x58, 0xba, 0xe7, 0x09, 0x7f, 0xf8, 0x60, 0x9c, - 0x78, 0x61, 0x10, 0x4b, 0x67, 0x7b, 0xc7, 0x63, 0xe1, 0xd4, 0x3b, 0x56, 0xb7, 0xc1, 0xf0, 0x4c, - 0xde, 0x82, 0xc6, 0x16, 0x77, 0x0f, 0x04, 0x0a, 0x6c, 0x14, 0xe4, 0x8c, 0x4c, 0x3a, 0xf0, 0x5e, - 0xa8, 0x28, 0x2d, 0x96, 0x33, 0x48, 0x07, 0x9a, 0x7b, 0xde, 0x48, 0x3c, 0x4a, 0x79, 0x90, 0xa4, - 0x23, 0xa7, 0x86, 0xd6, 0x45, 0x16, 0x59, 0x85, 0x85, 0x07, 0xfe, 0x70, 0xc7, 0x0b, 0x9c, 0x46, - 0xc7, 0xea, 0xda, 0x4c, 0x53, 0x86, 0xcf, 0x27, 0x0e, 0xe4, 0x7c, 0x3e, 0xc9, 0xd2, 0x6d, 0x96, - 0xd3, 0xdd, 0x0d, 0x07, 0x09, 0x0f, 0x86, 0x3c, 0x1a, 0x3e, 0xf1, 0xc4, 0x73, 0x67, 0x49, 0xa5, - 0x5b, 0xe6, 0x4a, 0xdb, 0x4d, 0x1e, 0x0b, 0xa7, 0x85, 0x1e, 0xf1, 0x4c, 0xd6, 0xa0, 0xbe, 0xe9, - 0x25, 0x3d, 0x31, 0x4e, 0x0e, 0x9c, 0xe5, 0x8e, 0xd5, 0xad, 0xb2, 0x8c, 0x26, 0x17, 0xa0, 0x36, - 0x70, 0xb9, 0x2f, 0x9c, 0x73, 0x68, 0xa0, 0x08, 0x42, 0x61, 0xe9, 0x5e, 0x18, 0x09, 0xef, 0x69, - 0x80, 0x4d, 0x70, 0xda, 0x98, 0x54, 0x89, 0x47, 0xde, 0x05, 0x5b, 0xa6, 0xb4, 0xd2, 0xb1, 0xba, - 0xcd, 0x9b, 0x2b, 0x1b, 0xa6, 0x8f, 0x1b, 0x3d, 0xe1, 0x7a, 0x23, 0xee, 0x33, 0x29, 0x45, 0x25, - 0x3e, 0x71, 0xc8, 0xe9, 0x4a, 0x7c, 0x42, 0x29, 0x2c, 0xf7, 0x47, 0xe3, 0x30, 0x4a, 0x98, 0x88, - 0xc7, 0x61, 0x10, 0x0b, 0xd2, 0x06, 0x7b, 0x3b, 0x8a, 0x1c, 0x0b, 0xc3, 0xca, 0x23, 0xfd, 0x16, - 0xda, 0x9b, 0x7e, 0xe8, 0x1e, 0xf6, 0x78, 0xc2, 0x99, 0x78, 0x96, 0x8a, 0x38, 0x91, 0xd8, 0x15, - 0x3c, 0xa5, 0xa7, 0x08, 0xc9, 0xc5, 0x7e, 0x3b, 0x15, 0xc5, 0x45, 0x42, 0xd6, 0x05, 0xab, 0xa6, - 0xda, 0x83, 0x67, 0xcc, 0xfd, 0x80, 0x47, 0x43, 0xec, 0x69, 0x95, 0x29, 0x42, 0x72, 0x31, 0x12, - 0xce, 0x41, 0x95, 0x29, 0x82, 0xf6, 0x61, 0xa5, 0x10, 0x5f, 0xc3, 0x5c, 0x85, 0x05, 0x16, 0x3e, - 0xef, 0xf7, 0x62, 0xc7, 0xea, 0xd8, 0xdd, 0x2a, 0xd3, 0x14, 0x0e, 0x4c, 0xe8, 0xa7, 0xa3, 0x40, - 0x8a, 0x2a, 0x28, 0xca, 0x19, 0xf4, 0x32, 0xd4, 0x70, 0x7a, 0x64, 0x96, 0xb9, 0xad, 0x3c, 0xd2, - 0xef, 0x2c, 0x68, 0xec, 0xf0, 0x09, 0x02, 0x89, 0xc9, 0x6d, 0xa8, 0x9b, 0xde, 0xa2, 0x52, 0xf3, - 0xe6, 0x3b, 0x79, 0x05, 0x33, 0xb5, 0x0d, 0xa3, 0xb3, 0x1d, 0x24, 0xd1, 0x31, 0xcb, 0x4c, 0xd6, - 0x3e, 0x83, 0x56, 0x49, 0x24, 0xe3, 0x1d, 0x8a, 0x63, 0x53, 0xd5, 0x43, 0x71, 0x2c, 0x73, 0x3d, - 0xe2, 0x7e, 0x2a, 0xb0, 0x56, 0x55, 0xa6, 0x88, 0x4f, 0x2b, 0x1f, 0x5b, 0xf4, 0x09, 0x90, 0xad, - 0x48, 0xf0, 0x44, 0x60, 0x90, 0x1d, 0x11, 0xc7, 0xfc, 0xa9, 0x98, 0x57, 0x71, 0xbb, 0x58, 0xf1, - 0xac, 0xba, 0x95, 0x42, 0x75, 0xe9, 0x35, 0x20, 0x3d, 0xe1, 0x8b, 0x44, 0xe8, 0xdb, 0xfd, 0x2f, - 0x7e, 0xe9, 0x33, 0x83, 0x61, 0xbe, 0x2e, 0xb9, 0x0a, 0x55, 0xb9, 0x2a, 0x30, 0x58, 0xf3, 0xe6, - 0xf9, 0xbc, 0x4e, 0xd9, 0x16, 0x61, 0xa8, 0x80, 0xbd, 0x41, 0xa7, 0xc3, 0xbb, 0x09, 0x02, 0xb6, - 0x59, 0xce, 0xa0, 0x3f, 0x58, 0x26, 0x26, 0x26, 0x71, 0xc6, 0xbc, 0x4b, 0x93, 0x76, 0x4d, 0x23, - 0xb1, 0x11, 0xc9, 0x6a, 0x8e, 0xa4, 0xb8, 0x85, 0x66, 0x81, 0xa9, 0x4e, 0x83, 0xb9, 0x63, 0x6a, - 0xf5, 0xaa, 0x58, 0xa8, 0x0b, 0x6f, 0x2a, 0x0f, 0x77, 0x8f, 0xb8, 0xe7, 0xf3, 0x7d, 0xff, 0x3f, - 0xb5, 0xb3, 0x94, 0x96, 0x03, 0x8b, 0x68, 0xdb, 0xef, 0xe9, 0x8b, 0x61, 0x48, 0xfa, 0x0d, 0xe4, - 0x77, 0x6c, 0x97, 0x8f, 0x84, 0xf6, 0x86, 0xe7, 0xac, 0x1a, 0x95, 0x33, 0x54, 0xe3, 0x02, 0xd4, - 0xe4, 0xbd, 0x94, 0x7b, 0xde, 0x96, 0x81, 0x91, 0x98, 0x53, 0xa3, 0x5b, 0xb0, 0x30, 0x70, 0x0f, - 0xc4, 0x88, 0x93, 0xf7, 0x61, 0x11, 0xf1, 0x8b, 0x58, 0x5f, 0x96, 0x73, 0x53, 0x43, 0xc0, 0x8c, - 0x9c, 0xfe, 0x64, 0xe9, 0xc4, 0x67, 0x42, 0x2e, 0x05, 0xac, 0x4c, 0x05, 0x24, 0xd7, 0x61, 0x51, - 0xa3, 0xc6, 0x5d, 0x72, 0xca, 0xac, 0x19, 0x1d, 0x72, 0x15, 0x16, 0x30, 0xd3, 0xd8, 0xa9, 0x4e, - 0x83, 0x42, 0x3e, 0xd3, 0x62, 0xba, 0x0d, 0xf6, 0x63, 0xd6, 0x97, 0x2b, 0x05, 0xf3, 0x31, 0x90, - 0x34, 0x25, 0x81, 0x7e, 0x11, 0xc6, 0x89, 0xee, 0x09, 0x9e, 0x25, 0xef, 0x61, 0x18, 0xa9, 0x29, - 0x6e, 0x31, 0x3c, 0xd3, 0x5f, 0x2c, 0xa8, 0xee, 0x86, 0x43, 0x41, 0x96, 0xa1, 0xd2, 0xef, 0x69, - 0x27, 0x95, 0x7e, 0x8f, 0xbc, 0x8d, 0xfe, 0x75, 0x1f, 0x5a, 0x39, 0x8a, 0xc7, 0xac, 0xcf, 0x30, - 0xf2, 0x15, 0x68, 0xf5, 0xe3, 0xad, 0x30, 0x8c, 0x86, 0x5e, 0xc0, 0x93, 0x30, 0xd2, 0x5f, 0xdb, - 0x32, 0x13, 0x6f, 0x75, 0xc2, 0x13, 0xf5, 0x1d, 0x6c, 0x30, 0x45, 0x90, 0xab, 0xb0, 0x78, 0x9f, - 0x3d, 0xdc, 0x92, 0x01, 0x6a, 0xb3, 0x02, 0x18, 0x29, 0xbd, 0x03, 0x6d, 0x89, 0x0e, 0xad, 0xcc, - 0x14, 0xae, 0xc2, 0x82, 0xe4, 0x65, 0x68, 0x35, 0x95, 0x87, 0xaa, 0x14, 0x42, 0xd1, 0xaf, 0x95, - 0x87, 0xed, 0x23, 0x11, 0x24, 0x85, 0x39, 0x46, 0x1a, 0x1d, 0xb4, 0x98, 0x22, 0x08, 0x55, 0x95, - 0xd0, 0x29, 0x2f, 0xe7, 0x88, 0x24, 0x97, 0xa1, 0x8c, 0xfe, 0x68, 0x01, 0x18, 0x40, 0x69, 0x9c, - 0x99, 0x58, 0xa7, 0x9b, 0x90, 0xae, 0x99, 0x38, 0x7d, 0xc3, 0xdb, 0xb9, 0x96, 0xe2, 0x33, 0x33, - 0x91, 0x1f, 0xe4, 0x13, 0xa9, 0x9a, 0x7f, 0x71, 0x6a, 0x54, 0x54, 0xd4, 0x7c, 0x2e, 0x03, 0x68, - 0x16, 0xf8, 0x33, 0x87, 0xf3, 0x7a, 0x36, 0x4f, 0x95, 0x69, 0x97, 0xc8, 0xd7, 0x2e, 0xb5, 0xd2, - 0x9c, 0x6d, 0xe7, 0x41, 0xb3, 0x60, 0x34, 0x33, 0x5e, 0x17, 0xce, 0x95, 0x77, 0x87, 0xf9, 0xa0, - 0x4d, 0xb3, 0xe7, 0x86, 0x6a, 0x6d, 0xf9, 0x69, 0x9c, 0x88, 0x48, 0x07, 0x93, 0xea, 0x8a, 0x91, - 0x35, 0x3e, 0x67, 0xcc, 0xee, 0x3d, 0xb9, 0x02, 0x35, 0xd9, 0x02, 0xb5, 0x20, 0x4e, 0xf6, 0x47, - 0x09, 0xe9, 0x13, 0xa8, 0x6f, 0x0e, 0xfa, 0xf7, 0xa3, 0x30, 0x1d, 0xcf, 0x4c, 0xc9, 0x3c, 0x00, - 0x2b, 0x85, 0x07, 0x60, 0x5b, 0x3d, 0x66, 0x14, 0x6c, 0x7c, 0xb9, 0xb4, 0xd5, 0xcb, 0xa5, 0xaa, - 0x39, 0x7c, 0x42, 0x07, 0xb0, 0xa2, 0xf2, 0x91, 0x7b, 0xe9, 0x55, 0x56, 0xa8, 0x79, 0x83, 0xd8, - 0xf9, 0x1b, 0x44, 0x3a, 0x55, 0x1b, 0xfa, 0xff, 0x74, 0xfa, 0x77, 0x05, 0x56, 0x98, 0x88, 0xbd, - 0x17, 0xa2, 0x1f, 0xc4, 0x49, 0x94, 0xba, 0x72, 0x17, 0x49, 0xfb, 0x2f, 0xc3, 0x7d, 0x5d, 0x6d, - 0x9b, 0x29, 0xe2, 0x2c, 0xb7, 0x84, 0xdc, 0x80, 0xe6, 0xf4, 0x62, 0x38, 0xa9, 0x5a, 0x54, 0x21, - 0x37, 0x60, 0x71, 0x10, 0xa6, 0x91, 0x9b, 0x8d, 0x7e, 0x61, 0xf3, 0x2b, 0x64, 0x4a, 0xcc, 0x8c, - 0x1a, 0x79, 0x04, 0x64, 0x2f, 0xe2, 0x41, 0xec, 0x73, 0x09, 0xd6, 0x18, 0xd7, 0xa7, 0x9f, 0x3d, - 0x05, 0x9d, 0x92, 0x9f, 0x19, 0xc6, 0xe4, 0xc3, 0xe2, 0xdd, 0x76, 0x16, 0x11, 0xf5, 0x85, 0x32, - 0x6a, 0x7d, 0x5d, 0x8a, 0x3b, 0xe0, 0xf6, 0xd4, 0xa4, 0x3a, 0x0b, 0x68, 0x78, 0x29, 0x37, 0x2c, - 0x89, 0x59, 0x59, 0x9b, 0x7e, 0x6f, 0xc1, 0x52, 0x11, 0xd9, 0x99, 0x76, 0x4a, 0xd6, 0xf0, 0xca, - 0xfc, 0x77, 0x95, 0x69, 0x78, 0x75, 0xd6, 0x4b, 0xb6, 0x56, 0x7c, 0x6b, 0xa5, 0x70, 0xe9, 0x94, - 0x72, 0xbd, 0x06, 0xa8, 0x0e, 0x34, 0x1f, 0xf2, 0x28, 0xf1, 0xa4, 0x4b, 0xfd, 0x16, 0xa8, 0xb1, - 0x22, 0x8b, 0x1e, 0xc2, 0xe5, 0x13, 0xc3, 0xb7, 0x15, 0x8e, 0xc6, 0x72, 0xca, 0x5f, 0x63, 0x08, - 0xe5, 0x92, 0x8f, 0x22, 0x3d, 0x7e, 0x0d, 0xa6, 0x08, 0xfa, 0x09, 0x5c, 0x1c, 0x88, 0xa4, 0x30, - 0x7a, 0xe6, 0x0e, 0x75, 0xc0, 0xde, 0x15, 0xcf, 0x4f, 0x49, 0x50, 0x8a, 0xe8, 0xe7, 0xe0, 0x3c, - 0x1e, 0x0f, 0x79, 0x22, 0x5e, 0xc9, 0x7a, 0x13, 0xea, 0x7b, 0xe1, 0x38, 0xf4, 0xc3, 0xa7, 0xc7, - 0x73, 0x76, 0x99, 0x03, 0x8b, 0xea, 0x8b, 0xa6, 0x56, 0x67, 0x83, 0x19, 0x92, 0x9e, 0x97, 0xd7, - 0xd4, 0xe5, 0xbe, 0x9b, 0xfa, 0x12, 0x86, 0xfc, 0x51, 0x10, 0x53, 0xa1, 0x2f, 0x02, 0xc7, 0xc2, - 0x15, 0x3e, 0x92, 0x77, 0x91, 0x61, 0x3e, 0x92, 0x8a, 0x22, 0x1f, 0x41, 0xb3, 0xa0, 0xad, 0x0b, - 0x78, 0x71, 0xea, 0xbe, 0x28, 0x21, 0x2b, 0x6a, 0xd2, 0xdf, 0xac, 0x92, 0xe5, 0x89, 0xf7, 0x82, - 0x0e, 0x78, 0xa4, 0x9a, 0x52, 0x67, 0x9a, 0x92, 0xb9, 0x6e, 0x4f, 0x5c, 0x3f, 0x8d, 0xa5, 0x48, - 0x3d, 0x11, 0x72, 0x86, 0xcc, 0x55, 0xfe, 0xf2, 0x0d, 0x53, 0xf3, 0x54, 0x33, 0xa4, 0xfc, 0x11, - 0xda, 0x13, 0x7c, 0xe8, 0x7b, 0x81, 0xc0, 0x29, 0xb5, 0x59, 0x46, 0x93, 0x1b, 0x6a, 0xdb, 0x9b, - 0xab, 0xb6, 0x36, 0x13, 0x3e, 0x6a, 0xa8, 0x2f, 0x41, 0x4c, 0x09, 0xb4, 0xa7, 0x45, 0x9b, 0xed, - 0xdf, 0x5f, 0xae, 0x5b, 0x7f, 0xbc, 0x5c, 0xb7, 0xfe, 0x7c, 0xb9, 0x6e, 0xfd, 0xfc, 0xd7, 0xfa, - 0x1b, 0xfb, 0x0b, 0xf8, 0x5f, 0xc2, 0xad, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x10, 0xd8, 0x2a, - 0xb6, 0x74, 0x10, 0x00, 0x00, + // 1458 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcb, 0x72, 0x1b, 0x45, + 0x17, 0xfe, 0x47, 0x23, 0xd9, 0xd2, 0x91, 0xe5, 0xc8, 0x9d, 0xc4, 0x99, 0xf8, 0xff, 0xcb, 0xbf, + 0x68, 0x52, 0x44, 0xa4, 0x2a, 0x26, 0x95, 0x50, 0xc5, 0x35, 0x55, 0x89, 0x2d, 0x27, 0x08, 0xb0, + 0x93, 0xb4, 0x9c, 0xec, 0xdb, 0xa3, 0xae, 0x78, 0xca, 0xa3, 0x19, 0x65, 0x2e, 0x8e, 0x1c, 0xaa, + 0xd8, 0x42, 0xc1, 0x8a, 0x62, 0xc3, 0x82, 0x05, 0xef, 0xc1, 0x0b, 0xb0, 0xe4, 0x11, 0xa8, 0xf0, + 0x14, 0xec, 0xa8, 0x3e, 0xdd, 0x3d, 0x17, 0x59, 0x8e, 0x4c, 0xc2, 0x6e, 0xce, 0xfd, 0x3b, 0x97, + 0x3e, 0xdd, 0x12, 0xb4, 0xc6, 0x91, 0x77, 0xc4, 0x13, 0xb1, 0x31, 0x8e, 0xc2, 0x24, 0x24, 0x75, + 0x2f, 0x48, 0x44, 0x14, 0x70, 0x7f, 0x6d, 0x69, 0x9c, 0xee, 0xfb, 0x9e, 0xab, 0xf8, 0xf4, 0x3e, + 0x34, 0xfa, 0xc1, 0x50, 0x4c, 0x76, 0x44, 0xc2, 0x09, 0x81, 0xea, 0x17, 0xe2, 0x38, 0x76, 0xec, + 0x8e, 0xd5, 0xad, 0x33, 0xfc, 0x26, 0xef, 0xc0, 0xf2, 0x5e, 0xc4, 0xdd, 0xc3, 0xed, 0x89, 0x17, + 0x27, 0x22, 0x70, 0x85, 0x53, 0x45, 0xe9, 0x14, 0x97, 0xfe, 0x62, 0xc3, 0xd2, 0x3d, 0x4f, 0xf8, + 0xc3, 0x07, 0xe3, 0xc4, 0x0b, 0x83, 0x58, 0x3a, 0xdb, 0x3b, 0x1e, 0x0b, 0xa7, 0xde, 0xb1, 0xba, + 0x0d, 0x86, 0xdf, 0xe4, 0x7f, 0xd0, 0xd8, 0xe2, 0xee, 0x81, 0x40, 0x81, 0x8d, 0x82, 0x9c, 0x91, + 0x49, 0x07, 0xde, 0x0b, 0x15, 0xa5, 0xc5, 0x72, 0x06, 0xe9, 0x40, 0x73, 0xcf, 0x1b, 0x89, 0x47, + 0x29, 0x0f, 0x92, 0x74, 0xe4, 0xd4, 0xd0, 0xba, 0xc8, 0x22, 0xab, 0xb0, 0xf0, 0xc0, 0x1f, 0xee, + 0x78, 0x81, 0xd3, 0xe8, 0x58, 0x5d, 0x9b, 0x69, 0xca, 0xf0, 0xf9, 0xc4, 0x81, 0x9c, 0xcf, 0x27, + 0x59, 0xba, 0xcd, 0x72, 0xba, 0xbb, 0xe1, 0x20, 0xe1, 0xc1, 0x90, 0x47, 0xc3, 0x27, 0x9e, 0x78, + 0xee, 0x2c, 0xa9, 0x74, 0xcb, 0x5c, 0x69, 0xbb, 0xc9, 0x63, 0xe1, 0xb4, 0xd0, 0x23, 0x7e, 0x93, + 0x35, 0xa8, 0x6f, 0x7a, 0x49, 0x4f, 0x8c, 0x93, 0x03, 0x67, 0xb9, 0x63, 0x75, 0xab, 0x2c, 0xa3, + 0xc9, 0x05, 0xa8, 0x0d, 0x5c, 0xee, 0x0b, 0xe7, 0x1c, 0x1a, 0x28, 0x82, 0x50, 0x58, 0xba, 0x17, + 0x46, 0xc2, 0x7b, 0x1a, 0x60, 0x13, 0x9c, 0x36, 0x26, 0x55, 0xe2, 0x91, 0xb7, 0xc1, 0x96, 0x29, + 0xad, 0x74, 0xac, 0x6e, 0xf3, 0xe6, 0xca, 0x86, 0xe9, 0xe3, 0x46, 0x4f, 0xb8, 0xde, 0x88, 0xfb, + 0x4c, 0x4a, 0x51, 0x89, 0x4f, 0x1c, 0x72, 0xba, 0x12, 0x9f, 0x50, 0x0a, 0xcb, 0xfd, 0xd1, 0x38, + 0x8c, 0x12, 0x26, 0xe2, 0x71, 0x18, 0xc4, 0x82, 0xb4, 0xc1, 0xde, 0x8e, 0x22, 0xc7, 0xc2, 0xb0, + 0xf2, 0x93, 0x7e, 0x0d, 0xed, 0x4d, 0x3f, 0x74, 0x0f, 0x7b, 0x3c, 0xe1, 0x4c, 0x3c, 0x4b, 0x45, + 0x9c, 0x48, 0xec, 0x0a, 0x9e, 0xd2, 0x53, 0x84, 0xe4, 0x62, 0xbf, 0x9d, 0x8a, 0xe2, 0x22, 0x21, + 0xeb, 0x82, 0x55, 0x53, 0xed, 0xc1, 0x6f, 0xcc, 0xfd, 0x80, 0x47, 0x43, 0xec, 0x69, 0x95, 0x29, + 0x42, 0x72, 0x31, 0x12, 0xce, 0x41, 0x95, 0x29, 0x82, 0xf6, 0x61, 0xa5, 0x10, 0x5f, 0xc3, 0x5c, + 0x85, 0x05, 0x16, 0x3e, 0xef, 0xf7, 0x62, 0xc7, 0xea, 0xd8, 0xdd, 0x2a, 0xd3, 0x14, 0x0e, 0x4c, + 0xe8, 0xa7, 0xa3, 0x40, 0x8a, 0x2a, 0x28, 0xca, 0x19, 0xf4, 0x32, 0xd4, 0x70, 0x7a, 0x64, 0x96, + 0xb9, 0xad, 0xfc, 0xa4, 0xdf, 0x58, 0xd0, 0xd8, 0xe1, 0x13, 0x04, 0x12, 0x93, 0xdb, 0x50, 0x37, + 0xbd, 0x45, 0xa5, 0xe6, 0xcd, 0xb7, 0xf2, 0x0a, 0x66, 0x6a, 0x1b, 0x46, 0x67, 0x3b, 0x48, 0xa2, + 0x63, 0x96, 0x99, 0xac, 0x7d, 0x02, 0xad, 0x92, 0x48, 0xc6, 0x3b, 0x14, 0xc7, 0xa6, 0xaa, 0x87, + 0xe2, 0x58, 0xe6, 0x7a, 0xc4, 0xfd, 0x54, 0x60, 0xad, 0xaa, 0x4c, 0x11, 0x1f, 0x57, 0x3e, 0xb4, + 0xe8, 0x13, 0x20, 0x5b, 0x91, 0xe0, 0x89, 0xc0, 0x20, 0x3b, 0x22, 0x8e, 0xf9, 0x53, 0x31, 0xaf, + 0xe2, 0x76, 0xb1, 0xe2, 0x59, 0x75, 0x2b, 0x85, 0xea, 0xd2, 0x6b, 0x40, 0x7a, 0xc2, 0x17, 0x89, + 0xd0, 0xa7, 0xfb, 0x15, 0x7e, 0xe9, 0x33, 0x83, 0x61, 0xbe, 0x2e, 0xb9, 0x0a, 0x55, 0xb9, 0x2a, + 0x30, 0x58, 0xf3, 0xe6, 0xf9, 0xbc, 0x4e, 0xd9, 0x16, 0x61, 0xa8, 0x80, 0xbd, 0x41, 0xa7, 0xc3, + 0xbb, 0x09, 0x02, 0xb6, 0x59, 0xce, 0xa0, 0xdf, 0x59, 0x26, 0x26, 0x26, 0x71, 0xc6, 0xbc, 0x4b, + 0x93, 0x76, 0x4d, 0x23, 0xb1, 0x11, 0xc9, 0x6a, 0x8e, 0xa4, 0xb8, 0x85, 0x66, 0x81, 0xa9, 0x4e, + 0x83, 0xb9, 0x63, 0x6a, 0xf5, 0xba, 0x58, 0xa8, 0x0b, 0xff, 0x55, 0x1e, 0xee, 0x1e, 0x71, 0xcf, + 0xe7, 0xfb, 0xfe, 0x3f, 0x6a, 0x67, 0x29, 0x2d, 0x07, 0x16, 0xd1, 0xb6, 0xdf, 0xd3, 0x07, 0xc3, + 0x90, 0xf4, 0x2b, 0xc8, 0xcf, 0xd8, 0x2e, 0x1f, 0x09, 0xed, 0x0d, 0xbf, 0xb3, 0x6a, 0x54, 0xce, + 0x50, 0x8d, 0x0b, 0x50, 0x93, 0xe7, 0x52, 0xee, 0x79, 0x5b, 0x06, 0x46, 0x62, 0x4e, 0x8d, 0x6e, + 0xc1, 0xc2, 0xc0, 0x3d, 0x10, 0x23, 0x4e, 0xde, 0x85, 0x45, 0xc4, 0x2f, 0x62, 0x7d, 0x58, 0xce, + 0x4d, 0x0d, 0x01, 0x33, 0x72, 0xfa, 0x83, 0xa5, 0x13, 0x9f, 0x09, 0xb9, 0x14, 0xb0, 0x32, 0x15, + 0x90, 0x5c, 0x87, 0x45, 0x8d, 0x1a, 0x77, 0xc9, 0x29, 0xb3, 0x66, 0x74, 0xc8, 0x55, 0x58, 0xc0, + 0x4c, 0x63, 0xa7, 0x3a, 0x0d, 0x0a, 0xf9, 0x4c, 0x8b, 0xe9, 0x36, 0xd8, 0x8f, 0x59, 0x5f, 0xae, + 0x14, 0xcc, 0xc7, 0x40, 0xd2, 0x94, 0x04, 0xfa, 0x59, 0x18, 0x27, 0xba, 0x27, 0xf8, 0x2d, 0x79, + 0x0f, 0xc3, 0x48, 0x4d, 0x71, 0x8b, 0xe1, 0x37, 0xfd, 0xd9, 0x82, 0xea, 0x6e, 0x38, 0x14, 0x64, + 0x19, 0x2a, 0xfd, 0x9e, 0x76, 0x52, 0xe9, 0xf7, 0xc8, 0xff, 0xd1, 0xbf, 0xee, 0x43, 0x2b, 0x47, + 0xf1, 0x98, 0xf5, 0x19, 0x46, 0xbe, 0x02, 0xad, 0x7e, 0xbc, 0x15, 0x86, 0xd1, 0xd0, 0x0b, 0x78, + 0x12, 0x46, 0xfa, 0xb6, 0x2d, 0x33, 0xf1, 0x54, 0x27, 0x3c, 0x51, 0xf7, 0x60, 0x83, 0x29, 0x82, + 0x5c, 0x85, 0xc5, 0xfb, 0xec, 0xe1, 0x96, 0x0c, 0x50, 0x9b, 0x15, 0xc0, 0x48, 0xe9, 0x1d, 0x68, + 0x4b, 0x74, 0x68, 0x65, 0xa6, 0x70, 0x15, 0x16, 0x24, 0x2f, 0x43, 0xab, 0xa9, 0x3c, 0x54, 0xa5, + 0x10, 0x8a, 0x7e, 0xa9, 0x3c, 0x6c, 0x1f, 0x89, 0x20, 0x29, 0xcc, 0x31, 0xd2, 0xe8, 0xa0, 0xc5, + 0x14, 0x41, 0xa8, 0xaa, 0x84, 0x4e, 0x79, 0x39, 0x47, 0x24, 0xb9, 0x0c, 0x65, 0xf4, 0x7b, 0x0b, + 0xc0, 0x00, 0x4a, 0xe3, 0xcc, 0xc4, 0x3a, 0xdd, 0x84, 0x74, 0xcd, 0xc4, 0xe9, 0x13, 0xde, 0xce, + 0xb5, 0x14, 0x9f, 0x99, 0x89, 0x7c, 0x2f, 0x9f, 0x48, 0xd5, 0xfc, 0x8b, 0x53, 0xa3, 0xa2, 0xa2, + 0xe6, 0x73, 0x19, 0x40, 0xb3, 0xc0, 0x9f, 0x39, 0x9c, 0xd7, 0xb3, 0x79, 0xaa, 0x4c, 0xbb, 0x44, + 0xbe, 0x76, 0xa9, 0x95, 0xe6, 0x6c, 0x3b, 0x0f, 0x9a, 0x05, 0xa3, 0x99, 0xf1, 0xba, 0x70, 0xae, + 0xbc, 0x3b, 0xcc, 0x85, 0x36, 0xcd, 0x9e, 0x13, 0xea, 0x47, 0x0b, 0x5a, 0x5b, 0x7e, 0x1a, 0x27, + 0x22, 0xd2, 0xd1, 0xa4, 0xbe, 0x62, 0x64, 0x9d, 0xcf, 0x19, 0xb3, 0x9b, 0x4f, 0xae, 0x40, 0x4d, + 0xf6, 0x40, 0x6d, 0x88, 0x93, 0x0d, 0x52, 0xc2, 0x42, 0x87, 0xaa, 0xaf, 0xee, 0x10, 0x7d, 0x02, + 0xf5, 0xcd, 0x41, 0xff, 0x7e, 0x14, 0xa6, 0xe3, 0x99, 0xd9, 0x9b, 0xb7, 0x62, 0xa5, 0xf0, 0x56, + 0x6c, 0xab, 0x77, 0x8f, 0xca, 0x10, 0x1f, 0x39, 0x6d, 0xf5, 0xc8, 0xa9, 0x6a, 0x0e, 0x9f, 0xd0, + 0x01, 0xac, 0xa8, 0xd4, 0xe5, 0x0a, 0x7b, 0x9d, 0x6d, 0x6b, 0x9e, 0x2b, 0x76, 0xfe, 0x5c, 0x91, + 0x4e, 0xd5, 0x32, 0xff, 0x37, 0x9d, 0xfe, 0x55, 0x81, 0x15, 0x26, 0x62, 0xef, 0x85, 0xe8, 0x07, + 0x71, 0x12, 0xa5, 0xae, 0x5c, 0x5b, 0xd2, 0xfe, 0xf3, 0x70, 0x5f, 0xf7, 0xc5, 0x66, 0x8a, 0x38, + 0xcb, 0x81, 0x22, 0x37, 0xa0, 0x39, 0xbd, 0x43, 0x4e, 0xaa, 0x16, 0x55, 0xc8, 0x0d, 0x58, 0x1c, + 0x84, 0x69, 0xe4, 0x66, 0xa7, 0xa4, 0x70, 0x49, 0x28, 0x64, 0x4a, 0xcc, 0x8c, 0x1a, 0x79, 0x04, + 0x64, 0x2f, 0xe2, 0x41, 0xec, 0x73, 0x09, 0xd6, 0x18, 0xd7, 0xa7, 0x5f, 0x48, 0x05, 0x9d, 0x92, + 0x9f, 0x19, 0xc6, 0xe4, 0xfd, 0xe2, 0x1a, 0x70, 0x16, 0x11, 0xf5, 0x85, 0x32, 0x6a, 0x7d, 0xb2, + 0x8a, 0xeb, 0xe2, 0xf6, 0xd4, 0x4c, 0x3b, 0x0b, 0x68, 0x78, 0x29, 0x37, 0x2c, 0x89, 0x59, 0x59, + 0x9b, 0x7e, 0x6b, 0xc1, 0x52, 0x11, 0xd9, 0x99, 0xd6, 0x4f, 0xd6, 0xf0, 0xca, 0xfc, 0x27, 0x98, + 0x69, 0x78, 0x75, 0xd6, 0xa3, 0xb7, 0x56, 0x7c, 0x96, 0xa5, 0x70, 0xe9, 0x94, 0x72, 0xbd, 0x01, + 0xa8, 0x0e, 0x34, 0x1f, 0xf2, 0x28, 0xf1, 0xa4, 0x4b, 0xfd, 0x6c, 0xa8, 0xb1, 0x22, 0x8b, 0x1e, + 0xc2, 0xe5, 0x13, 0xc3, 0xb7, 0x15, 0x8e, 0xc6, 0x72, 0xca, 0xdf, 0x60, 0x08, 0xe5, 0x7d, 0x10, + 0x45, 0x7a, 0xfc, 0x1a, 0x4c, 0x11, 0xf4, 0x23, 0xb8, 0x38, 0x10, 0x49, 0x61, 0xf4, 0xcc, 0x19, + 0xea, 0x80, 0xbd, 0x2b, 0x9e, 0x9f, 0x92, 0xa0, 0x14, 0xd1, 0x4f, 0xc1, 0x79, 0x3c, 0x1e, 0xf2, + 0x44, 0xbc, 0x96, 0xf5, 0x26, 0xd4, 0xf7, 0xc2, 0x71, 0xe8, 0x87, 0x4f, 0x8f, 0xe7, 0x6c, 0x3d, + 0x07, 0x16, 0xd5, 0xe5, 0xa7, 0xb6, 0x6c, 0x83, 0x19, 0x92, 0x9e, 0x97, 0xc7, 0xd4, 0xe5, 0xbe, + 0x9b, 0xfa, 0x12, 0x86, 0xfc, 0xfd, 0x10, 0x53, 0xa1, 0x0f, 0x02, 0xc7, 0xc2, 0x15, 0xee, 0xd3, + 0xbb, 0xc8, 0x30, 0xf7, 0xa9, 0xa2, 0xc8, 0x07, 0xd0, 0x2c, 0x68, 0xeb, 0x02, 0x5e, 0x9c, 0x3a, + 0x2f, 0x4a, 0xc8, 0x8a, 0x9a, 0xf4, 0x57, 0xab, 0x64, 0x79, 0xe2, 0x69, 0xa1, 0x03, 0x1e, 0xa9, + 0xa6, 0xd4, 0x99, 0xa6, 0x64, 0xae, 0xdb, 0x13, 0xd7, 0x4f, 0x63, 0x29, 0x52, 0xaf, 0x89, 0x9c, + 0x21, 0x73, 0x95, 0x3f, 0x92, 0xc3, 0xd4, 0xbc, 0xea, 0x0c, 0x29, 0x7f, 0xaf, 0xf6, 0x04, 0x1f, + 0xfa, 0x5e, 0x20, 0x70, 0x4a, 0x6d, 0x96, 0xd1, 0xe4, 0x86, 0xba, 0x17, 0xcc, 0x51, 0x5b, 0x9b, + 0x09, 0x1f, 0x35, 0xd4, 0x9d, 0x11, 0x53, 0x02, 0xed, 0x69, 0xd1, 0x66, 0xfb, 0xb7, 0x97, 0xeb, + 0xd6, 0xef, 0x2f, 0xd7, 0xad, 0x3f, 0x5e, 0xae, 0x5b, 0x3f, 0xfd, 0xb9, 0xfe, 0x9f, 0xfd, 0x05, + 0xfc, 0xdb, 0xe1, 0xd6, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x31, 0xb0, 0x31, 0x3c, 0x9f, 0x10, + 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -3836,6 +3845,18 @@ func (m *ClusterStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.Schema != nil { + { + size, err := m.Schema.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } if len(m.Nodes) > 0 { for iNdEx := len(m.Nodes) - 1; iNdEx >= 0; iNdEx-- { { @@ -5184,6 +5205,10 @@ func (m *ClusterStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if m.Schema != nil { + l = m.Schema.Size() + n += 1 + l + sovPrivate(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -9208,6 +9233,42 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Schema == nil { + m.Schema = &Schema{} + } + if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) diff --git a/internal/private.proto b/internal/private.proto index 18494a87f..d83a8d2c0 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -149,6 +149,7 @@ message ClusterStatus { string ClusterID = 1; string State = 2; repeated Node Nodes = 3; + Schema Schema = 4; } message BSIGroup { diff --git a/internal/public.pb.go b/internal/public.pb.go index 826c6fc62..9f71b0368 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1780,6 +1780,7 @@ type ImportColumnAttrsRequest struct { AttrKey string `protobuf:"bytes,3,opt,name=AttrKey,proto3" json:"AttrKey,omitempty"` AttrVals []string `protobuf:"bytes,4,rep,name=AttrVals,proto3" json:"AttrVals,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs,proto3" json:"ColumnIDs,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,6,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1853,6 +1854,13 @@ func (m *ImportColumnAttrsRequest) GetColumnIDs() []uint64 { return nil } +func (m *ImportColumnAttrsRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*SignedRow)(nil), "internal.SignedRow") @@ -1885,86 +1893,86 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1253 bytes of a gzipped FileDescriptorProto + // 1258 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x8e, 0x1b, 0x45, 0x10, 0x66, 0x3c, 0xe3, 0xbf, 0xb2, 0x77, 0x13, 0x3a, 0x4e, 0x18, 0xa1, 0xb0, 0xb1, 0x46, 0x01, - 0x19, 0x0e, 0x1b, 0x6d, 0x08, 0x51, 0x4e, 0x40, 0x36, 0xde, 0x80, 0x15, 0x65, 0x15, 0xca, 0x91, - 0xb9, 0x21, 0x4d, 0xec, 0x66, 0x33, 0x62, 0x3c, 0x63, 0xe6, 0x07, 0x67, 0x8f, 0x3c, 0x03, 0x17, - 0xc4, 0x13, 0x70, 0xe5, 0x15, 0x38, 0x71, 0xe4, 0x11, 0x50, 0xe0, 0xcc, 0x0b, 0x70, 0x41, 0x55, - 0x3d, 0xed, 0x6e, 0x3b, 0xde, 0x65, 0x15, 0x71, 0xeb, 0xaf, 0xaa, 0xa6, 0xba, 0xea, 0xeb, 0xea, - 0xaa, 0x1e, 0xe8, 0x2e, 0xca, 0x67, 0x71, 0x34, 0xdd, 0x5f, 0x64, 0x69, 0x91, 0x8a, 0x56, 0x94, - 0x14, 0x32, 0x4b, 0xc2, 0x38, 0xc8, 0xc1, 0xc5, 0x74, 0x29, 0x7c, 0x68, 0x3e, 0x48, 0xe3, 0x72, - 0x9e, 0xe4, 0xbe, 0xd3, 0x77, 0x07, 0x1e, 0x6a, 0x28, 0x04, 0x78, 0x8f, 0xe4, 0x69, 0xee, 0xbb, - 0x7d, 0x77, 0xd0, 0x46, 0x5e, 0x8b, 0x9b, 0x50, 0xbf, 0x5f, 0x14, 0x59, 0xee, 0xd7, 0xfa, 0xee, - 0xa0, 0x73, 0x7b, 0x77, 0x5f, 0xbb, 0xdb, 0x27, 0x31, 0x2a, 0x25, 0xf9, 0xc4, 0x34, 0xcc, 0xa2, - 0xe4, 0xc4, 0xf7, 0xfa, 0xce, 0xa0, 0x8b, 0x1a, 0x06, 0x8f, 0xa1, 0x3d, 0x8e, 0x4e, 0x12, 0x39, - 0xa3, 0xad, 0x6f, 0x80, 0xfb, 0x24, 0xa5, 0x6d, 0x9d, 0x41, 0xe7, 0xf6, 0x8e, 0x71, 0x85, 0xe9, - 0x12, 0x49, 0x43, 0x06, 0xc7, 0xf2, 0xc4, 0xaf, 0x6d, 0x35, 0x38, 0x96, 0x27, 0xc1, 0x3d, 0xd8, - 0xc5, 0x74, 0x39, 0x9a, 0xc9, 0xa4, 0x88, 0xbe, 0x8e, 0x64, 0xc6, 0x41, 0x63, 0xba, 0xd4, 0xb9, - 0xf0, 0x7a, 0x95, 0x48, 0xcd, 0x24, 0x12, 0x7c, 0x0c, 0xde, 0x93, 0x30, 0xca, 0xc4, 0x2e, 0xd4, - 0x46, 0x43, 0x0e, 0xc1, 0xc3, 0xda, 0x68, 0x28, 0x2e, 0x83, 0xfb, 0x48, 0x9e, 0xfa, 0x6e, 0xdf, - 0x19, 0xb4, 0x91, 0x96, 0xa2, 0x07, 0xf5, 0x07, 0x69, 0x99, 0x14, 0x1c, 0x86, 0x87, 0x0a, 0x04, - 0x47, 0xd0, 0xa6, 0xef, 0x1f, 0x46, 0x32, 0x9e, 0x89, 0x40, 0x39, 0xab, 0x32, 0xb1, 0x48, 0x21, - 0x29, 0xaa, 0x8d, 0x7a, 0x50, 0x67, 0x63, 0x76, 0xd3, 0x46, 0x05, 0x82, 0xcf, 0x01, 0x48, 0x9b, - 0x2b, 0x3f, 0x37, 0xa1, 0xce, 0x88, 0xa3, 0x7f, 0xd5, 0x91, 0x52, 0x9e, 0xe1, 0xe9, 0x1d, 0xa8, - 0x8f, 0x92, 0xe2, 0xee, 0x1d, 0x52, 0x4f, 0xc2, 0xb8, 0x94, 0x1c, 0x8d, 0x8b, 0x0a, 0x04, 0x25, - 0xb4, 0xd8, 0x8e, 0x78, 0x5f, 0x39, 0x70, 0x2c, 0x07, 0x24, 0x25, 0x2e, 0x87, 0x3a, 0x4f, 0x06, - 0xe2, 0x1a, 0x34, 0x30, 0x5d, 0x1a, 0x4a, 0x2a, 0x24, 0xde, 0xd5, 0xbb, 0x78, 0x9c, 0xf3, 0x25, - 0x13, 0x2a, 0x47, 0xa1, 0xb7, 0xfd, 0x0a, 0xe0, 0xb3, 0x2c, 0x2d, 0x17, 0x4c, 0x9a, 0x18, 0x40, - 0x9d, 0x51, 0x95, 0x9f, 0x30, 0x1f, 0xe9, 0xd8, 0x50, 0x19, 0x6c, 0x27, 0x9d, 0x0e, 0x67, 0x5c, - 0xce, 0x39, 0x12, 0x17, 0x69, 0x19, 0x7c, 0xef, 0x40, 0x6b, 0x12, 0xc6, 0x2b, 0xf5, 0x24, 0x8c, - 0xab, 0xbc, 0x69, 0xb9, 0xee, 0xc6, 0xd5, 0x6e, 0xde, 0x86, 0xd6, 0xc3, 0x38, 0x0d, 0x0b, 0x32, - 0x26, 0x5f, 0x0e, 0xae, 0xb0, 0x38, 0x00, 0x18, 0xca, 0x69, 0x34, 0x0f, 0x63, 0xd2, 0xaa, 0xe4, - 0xde, 0x34, 0x71, 0x56, 0x3a, 0xb4, 0x8c, 0x82, 0x8f, 0xa0, 0x59, 0xa1, 0xed, 0xdc, 0x93, 0x74, - 0x3c, 0x0d, 0x63, 0xa9, 0xa3, 0x60, 0x10, 0x7c, 0x09, 0x3b, 0xea, 0xa6, 0xd1, 0x9d, 0x19, 0xcb, - 0xe2, 0x02, 0xa5, 0x78, 0xa1, 0xdb, 0x17, 0xfc, 0xec, 0x80, 0x47, 0x2b, 0xed, 0xc0, 0x31, 0x0e, - 0x04, 0x78, 0x4f, 0x4f, 0x17, 0xb2, 0x62, 0x95, 0xd7, 0xa2, 0x0f, 0x9d, 0x71, 0x41, 0x97, 0x53, - 0x45, 0xae, 0xb6, 0xb3, 0x45, 0xc4, 0xd7, 0x28, 0x29, 0xcc, 0x71, 0xbb, 0xb8, 0xc2, 0xe2, 0x3a, - 0xb4, 0x0f, 0xd3, 0x34, 0x56, 0xca, 0x7a, 0xdf, 0x19, 0xb4, 0xd0, 0x08, 0xc4, 0x1e, 0x80, 0x66, - 0xb6, 0x94, 0x7e, 0x83, 0xb9, 0xb6, 0x24, 0xc1, 0x2d, 0x68, 0x52, 0xa4, 0x8f, 0xc3, 0x85, 0xc9, - 0xcd, 0x39, 0x2f, 0xb7, 0x7f, 0x1c, 0xe8, 0x7e, 0x51, 0xca, 0xec, 0x14, 0xe5, 0xb7, 0xa5, 0xcc, - 0x0b, 0xe2, 0x96, 0xb1, 0xae, 0x65, 0x06, 0x54, 0xb5, 0xe3, 0xe7, 0x61, 0x36, 0x53, 0x4c, 0x79, - 0x58, 0x21, 0xca, 0xd5, 0x70, 0x9e, 0x73, 0xae, 0x2d, 0xb4, 0x45, 0x5c, 0xef, 0x72, 0x9e, 0x16, - 0x3a, 0x99, 0x0a, 0x89, 0x01, 0x5c, 0x3a, 0x7a, 0x31, 0x8d, 0xcb, 0x99, 0xc4, 0x74, 0xa9, 0xbe, - 0x6e, 0xb0, 0xc1, 0xa6, 0x58, 0xbc, 0x07, 0xbb, 0x95, 0x48, 0xf7, 0xd5, 0x26, 0x1b, 0x6e, 0x48, - 0xc5, 0x01, 0x74, 0x8f, 0xe6, 0xcf, 0xe4, 0x6c, 0x26, 0x67, 0xc3, 0xb0, 0x08, 0xfd, 0x16, 0xe7, - 0xbd, 0xd1, 0xe5, 0xd6, 0x4c, 0x82, 0x1f, 0x1c, 0xd8, 0xa9, 0xb2, 0xcf, 0x17, 0x69, 0x92, 0x4b, - 0x3a, 0xe2, 0xa3, 0x2c, 0xd3, 0x47, 0x7c, 0x94, 0x65, 0xe2, 0x16, 0x34, 0x51, 0xe6, 0x65, 0x5c, - 0xe8, 0x2a, 0xb9, 0x6a, 0x3c, 0xea, 0x6f, 0xcb, 0xb8, 0x40, 0x6d, 0x25, 0x3e, 0x81, 0xdd, 0xb5, - 0x3a, 0x54, 0x0d, 0xbf, 0x73, 0xfb, 0x2d, 0xf3, 0xdd, 0x9a, 0x1e, 0x37, 0xcc, 0x83, 0xbf, 0x5d, - 0xe8, 0x58, 0x9e, 0x57, 0x45, 0x46, 0xfc, 0xec, 0x54, 0x45, 0x76, 0x83, 0x87, 0xcd, 0x19, 0xad, - 0x9e, 0x7a, 0x52, 0x17, 0x9c, 0xe3, 0xaa, 0x2c, 0x9d, 0x63, 0xd3, 0x08, 0xdd, 0xf3, 0x1a, 0x21, - 0x8d, 0xae, 0xe7, 0x61, 0x72, 0x22, 0x67, 0x5c, 0x96, 0x2d, 0xd4, 0x50, 0xec, 0x9b, 0xae, 0xc0, - 0xe7, 0xb8, 0xd6, 0x6b, 0xb4, 0x06, 0x4d, 0xe7, 0x50, 0x5d, 0x6e, 0x34, 0xa4, 0xb3, 0xe2, 0x7a, - 0x51, 0x48, 0xdc, 0x85, 0x8e, 0x69, 0x5f, 0x79, 0x75, 0x44, 0x3d, 0xe3, 0xca, 0x28, 0xd1, 0x36, - 0x14, 0x9f, 0x6e, 0xce, 0x25, 0xbf, 0xcd, 0x51, 0xf8, 0x6b, 0x99, 0x5b, 0x7a, 0xdc, 0x9c, 0x63, - 0x07, 0xd6, 0xa0, 0xf4, 0x81, 0x3f, 0xbe, 0x62, 0x3e, 0x5e, 0xa9, 0xd0, 0x1a, 0xa7, 0x77, 0xec, - 0x59, 0xe2, 0x77, 0xf8, 0x9b, 0xde, 0x3a, 0x73, 0x4a, 0x87, 0xf6, 0xcc, 0x39, 0xb0, 0x06, 0x99, - 0xdf, 0xdd, 0xdc, 0x68, 0xa5, 0x42, 0x63, 0x15, 0xfc, 0x52, 0x83, 0x9d, 0xd1, 0x7c, 0x91, 0x66, - 0x85, 0x75, 0x0b, 0x47, 0xc9, 0x4c, 0xbe, 0xd0, 0xb7, 0x90, 0xc1, 0xf6, 0x41, 0xc5, 0xdd, 0x90, - 0x6e, 0x23, 0xdf, 0x3e, 0x0f, 0x15, 0xb0, 0x4e, 0xc0, 0x5b, 0x3b, 0x81, 0xeb, 0xd0, 0x56, 0xe5, - 0x46, 0xaa, 0x3a, 0xab, 0x8c, 0x40, 0x3d, 0x34, 0x96, 0x3c, 0xdc, 0x9b, 0x3c, 0xdc, 0x35, 0xa4, - 0xce, 0xa3, 0xcc, 0x58, 0xd9, 0x62, 0xa5, 0x25, 0x21, 0xfd, 0xd3, 0x68, 0x2e, 0xf3, 0x22, 0x9c, - 0x2f, 0xe8, 0x2a, 0xbb, 0x03, 0x17, 0x2d, 0x09, 0xdd, 0x62, 0x4e, 0xe2, 0x41, 0x26, 0xc3, 0x42, - 0xce, 0xee, 0x17, 0x7c, 0x82, 0x2e, 0x6e, 0x48, 0xc9, 0x8e, 0xd3, 0x32, 0x76, 0xa0, 0xec, 0xd6, - 0xa5, 0xc1, 0xaf, 0x35, 0x10, 0x8a, 0x33, 0xee, 0x7c, 0xff, 0x1f, 0x71, 0xe7, 0x13, 0xb4, 0x4e, - 0x43, 0xf3, 0x15, 0x1a, 0xae, 0x41, 0x83, 0xe3, 0xd1, 0x14, 0x54, 0x88, 0x1a, 0xa5, 0x69, 0xd3, - 0x8a, 0x3f, 0x07, 0x6d, 0x91, 0x08, 0xa0, 0x6b, 0xcd, 0x08, 0x2a, 0x70, 0xf2, 0xbd, 0x26, 0xdb, - 0x42, 0x22, 0x5c, 0x90, 0xc4, 0xce, 0x56, 0x12, 0x27, 0xd0, 0x7b, 0x9a, 0x85, 0x49, 0x1e, 0x87, - 0x85, 0xa4, 0xf0, 0x5f, 0x87, 0xc5, 0x2d, 0xaf, 0xda, 0xe0, 0x7d, 0xb8, 0xba, 0xe1, 0xd7, 0xb4, - 0x57, 0xa2, 0xd5, 0x65, 0x5a, 0x69, 0x19, 0x8c, 0xe1, 0xca, 0xca, 0x74, 0x34, 0x7c, 0xad, 0x08, - 0x5e, 0x75, 0xfa, 0x81, 0x95, 0x17, 0x3b, 0xad, 0xb6, 0xdf, 0x16, 0xeb, 0x21, 0xf8, 0xd5, 0xdd, - 0x53, 0x4f, 0xea, 0x2a, 0x82, 0x49, 0x24, 0x97, 0x64, 0x7f, 0x1c, 0xce, 0x65, 0x15, 0x04, 0xaf, - 0x49, 0xc6, 0xe3, 0xa5, 0xc6, 0x0f, 0x71, 0x5e, 0x07, 0x7f, 0x39, 0xd0, 0xdb, 0xe6, 0x84, 0xdf, - 0x4b, 0xb1, 0x0c, 0xd5, 0x40, 0x69, 0xa1, 0x02, 0xe2, 0x1e, 0xd4, 0xbf, 0x8b, 0xe4, 0x52, 0x0f, - 0x94, 0xc0, 0x7a, 0xeb, 0x9d, 0x11, 0x09, 0xaa, 0x0f, 0xa8, 0xbc, 0xee, 0x4f, 0x8b, 0x28, 0x4d, - 0xf4, 0xeb, 0x51, 0x21, 0xda, 0xe7, 0x30, 0x4e, 0xa7, 0xdf, 0x70, 0xdf, 0xf6, 0x50, 0x81, 0x2d, - 0xe5, 0x52, 0xbf, 0x60, 0xb9, 0x34, 0xb6, 0x96, 0xcb, 0x4f, 0x8e, 0xe6, 0xca, 0x9a, 0xf0, 0xff, - 0x79, 0x62, 0xea, 0x8e, 0xe9, 0xa7, 0x1a, 0xdf, 0x31, 0x5f, 0x3d, 0x53, 0xcc, 0x6b, 0x4c, 0x43, - 0x7a, 0x1a, 0xd1, 0x72, 0x12, 0xc6, 0xaa, 0x71, 0xb5, 0x71, 0x85, 0xcf, 0xbf, 0x99, 0x87, 0x97, - 0x7f, 0x7b, 0xb9, 0xe7, 0xfc, 0xfe, 0x72, 0xcf, 0xf9, 0xe3, 0xe5, 0x9e, 0xf3, 0xe3, 0x9f, 0x7b, - 0x6f, 0x3c, 0x6b, 0xf0, 0x1f, 0xda, 0x87, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x91, 0x85, - 0xb9, 0xb1, 0x0d, 0x00, 0x00, + 0x19, 0x0e, 0x1b, 0x6d, 0x08, 0x51, 0x4e, 0x40, 0x36, 0xde, 0x80, 0x15, 0x65, 0x15, 0xca, 0x2b, + 0x73, 0x43, 0x9a, 0xb5, 0x9b, 0xcd, 0x88, 0xf1, 0x8c, 0x99, 0x1f, 0x9c, 0x3d, 0xf2, 0x0c, 0x5c, + 0x78, 0x04, 0xae, 0xbc, 0x02, 0x27, 0x8e, 0x3c, 0x02, 0x5a, 0x38, 0xf3, 0x02, 0x5c, 0x50, 0x55, + 0x4f, 0xbb, 0xc7, 0xde, 0xd9, 0xcd, 0x2a, 0xe2, 0xd6, 0x5f, 0x55, 0x4d, 0x75, 0xd5, 0xd7, 0xd5, + 0x55, 0x3d, 0xd0, 0x5d, 0xe4, 0xc7, 0x61, 0x30, 0xdd, 0x5d, 0x24, 0x71, 0x16, 0x8b, 0x56, 0x10, + 0x65, 0x32, 0x89, 0xfc, 0xd0, 0x4b, 0xc1, 0xc6, 0x78, 0x29, 0x5c, 0x68, 0x3e, 0x89, 0xc3, 0x7c, + 0x1e, 0xa5, 0xae, 0xd5, 0xb7, 0x07, 0x0e, 0x6a, 0x28, 0x04, 0x38, 0xcf, 0xe4, 0x69, 0xea, 0xda, + 0x7d, 0x7b, 0xd0, 0x46, 0x5e, 0x8b, 0xbb, 0x50, 0x7f, 0x9c, 0x65, 0x49, 0xea, 0xd6, 0xfa, 0xf6, + 0xa0, 0x73, 0x7f, 0x7b, 0x57, 0xbb, 0xdb, 0x25, 0x31, 0x2a, 0x25, 0xf9, 0xc4, 0xd8, 0x4f, 0x82, + 0xe8, 0xc4, 0x75, 0xfa, 0xd6, 0xa0, 0x8b, 0x1a, 0x7a, 0xcf, 0xa1, 0x3d, 0x0e, 0x4e, 0x22, 0x39, + 0xa3, 0xad, 0xef, 0x80, 0xfd, 0x22, 0xa6, 0x6d, 0xad, 0x41, 0xe7, 0xfe, 0x96, 0x71, 0x85, 0xf1, + 0x12, 0x49, 0x43, 0x06, 0x87, 0xf2, 0xc4, 0xad, 0x55, 0x1a, 0x1c, 0xca, 0x13, 0xef, 0x11, 0x6c, + 0x63, 0xbc, 0x1c, 0xcd, 0x64, 0x94, 0x05, 0xdf, 0x06, 0x32, 0xe1, 0xa0, 0x31, 0x5e, 0xea, 0x5c, + 0x78, 0xbd, 0x4a, 0xa4, 0x66, 0x12, 0xf1, 0x3e, 0x05, 0xe7, 0x85, 0x1f, 0x24, 0x62, 0x1b, 0x6a, + 0xa3, 0x21, 0x87, 0xe0, 0x60, 0x6d, 0x34, 0x14, 0xd7, 0xc1, 0x7e, 0x26, 0x4f, 0x5d, 0xbb, 0x6f, + 0x0d, 0xda, 0x48, 0x4b, 0xd1, 0x83, 0xfa, 0x93, 0x38, 0x8f, 0x32, 0x0e, 0xc3, 0x41, 0x05, 0xbc, + 0x03, 0x68, 0xd3, 0xf7, 0x4f, 0x03, 0x19, 0xce, 0x84, 0xa7, 0x9c, 0x15, 0x99, 0x94, 0x48, 0x21, + 0x29, 0xaa, 0x8d, 0x7a, 0x50, 0x67, 0x63, 0x76, 0xd3, 0x46, 0x05, 0xbc, 0x2f, 0x01, 0x48, 0x9b, + 0x2a, 0x3f, 0x77, 0xa1, 0xce, 0x88, 0xa3, 0x3f, 0xef, 0x48, 0x29, 0x2f, 0xf0, 0xf4, 0x1e, 0xd4, + 0x47, 0x51, 0xf6, 0xf0, 0x01, 0xa9, 0x27, 0x7e, 0x98, 0x4b, 0x8e, 0xc6, 0x46, 0x05, 0xbc, 0x1c, + 0x5a, 0x6c, 0x47, 0xbc, 0xaf, 0x1c, 0x58, 0x25, 0x07, 0x24, 0x25, 0x2e, 0x87, 0x3a, 0x4f, 0x06, + 0xe2, 0x16, 0x34, 0x30, 0x5e, 0x1a, 0x4a, 0x0a, 0x24, 0xde, 0xd7, 0xbb, 0x38, 0x9c, 0xf3, 0x35, + 0x13, 0x2a, 0x47, 0xa1, 0xb7, 0xfd, 0x06, 0xe0, 0x8b, 0x24, 0xce, 0x17, 0x4c, 0x9a, 0x18, 0x40, + 0x9d, 0x51, 0x91, 0x9f, 0x30, 0x1f, 0xe9, 0xd8, 0x50, 0x19, 0x54, 0x93, 0x4e, 0x87, 0x33, 0xce, + 0xe7, 0x1c, 0x89, 0x8d, 0xb4, 0xf4, 0x7e, 0xb4, 0xa0, 0x35, 0xf1, 0xc3, 0x95, 0x7a, 0xe2, 0x87, + 0x45, 0xde, 0xb4, 0x5c, 0x77, 0x63, 0x6b, 0x37, 0xef, 0x42, 0xeb, 0x69, 0x18, 0xfb, 0x19, 0x19, + 0x93, 0x2f, 0x0b, 0x57, 0x58, 0xec, 0x01, 0x0c, 0xe5, 0x34, 0x98, 0xfb, 0x21, 0x69, 0x55, 0x72, + 0x6f, 0x9b, 0x38, 0x0b, 0x1d, 0x96, 0x8c, 0xbc, 0x4f, 0xa0, 0x59, 0xa0, 0x6a, 0xee, 0x49, 0x3a, + 0x9e, 0xfa, 0xa1, 0xd4, 0x51, 0x30, 0xf0, 0xbe, 0x86, 0x2d, 0x75, 0xd3, 0xe8, 0xce, 0x8c, 0x65, + 0x76, 0x85, 0x52, 0xbc, 0xd2, 0xed, 0xf3, 0x7e, 0xb1, 0xc0, 0xa1, 0x95, 0x76, 0x60, 0x19, 0x07, + 0x02, 0x9c, 0xa3, 0xd3, 0x85, 0x2c, 0x58, 0xe5, 0xb5, 0xe8, 0x43, 0x67, 0x9c, 0xd1, 0xe5, 0x54, + 0x91, 0xab, 0xed, 0xca, 0x22, 0xe2, 0x6b, 0x14, 0x65, 0xe6, 0xb8, 0x6d, 0x5c, 0x61, 0x71, 0x1b, + 0xda, 0xfb, 0x71, 0x1c, 0x2a, 0x65, 0xbd, 0x6f, 0x0d, 0x5a, 0x68, 0x04, 0x62, 0x07, 0x40, 0x33, + 0x9b, 0x4b, 0xb7, 0xc1, 0x5c, 0x97, 0x24, 0xde, 0x3d, 0x68, 0x52, 0xa4, 0xcf, 0xfd, 0x85, 0xc9, + 0xcd, 0xba, 0x2c, 0xb7, 0x7f, 0x2d, 0xe8, 0x7e, 0x95, 0xcb, 0xe4, 0x14, 0xe5, 0xf7, 0xb9, 0x4c, + 0x33, 0xe2, 0x96, 0xb1, 0xae, 0x65, 0x06, 0x54, 0xb5, 0xe3, 0x97, 0x7e, 0x32, 0x53, 0x4c, 0x39, + 0x58, 0x20, 0xca, 0xd5, 0x70, 0x9e, 0x72, 0xae, 0x2d, 0x2c, 0x8b, 0xb8, 0xde, 0xe5, 0x3c, 0xce, + 0x74, 0x32, 0x05, 0x12, 0x03, 0xb8, 0x76, 0xf0, 0x6a, 0x1a, 0xe6, 0x33, 0x89, 0xf1, 0x52, 0x7d, + 0xdd, 0x60, 0x83, 0x4d, 0xb1, 0xf8, 0x00, 0xb6, 0x0b, 0x91, 0xee, 0xab, 0x4d, 0x36, 0xdc, 0x90, + 0x8a, 0x3d, 0xe8, 0x1e, 0xcc, 0x8f, 0xe5, 0x6c, 0x26, 0x67, 0x43, 0x3f, 0xf3, 0xdd, 0x16, 0xe7, + 0xbd, 0xd1, 0xe5, 0xd6, 0x4c, 0xbc, 0x9f, 0x2c, 0xd8, 0x2a, 0xb2, 0x4f, 0x17, 0x71, 0x94, 0x4a, + 0x3a, 0xe2, 0x83, 0x24, 0xd1, 0x47, 0x7c, 0x90, 0x24, 0xe2, 0x1e, 0x34, 0x51, 0xa6, 0x79, 0x98, + 0xe9, 0x2a, 0xb9, 0x69, 0x3c, 0xea, 0x6f, 0xf3, 0x30, 0x43, 0x6d, 0x25, 0x3e, 0x83, 0xed, 0xb5, + 0x3a, 0x54, 0x0d, 0xbf, 0x73, 0xff, 0x1d, 0xf3, 0xdd, 0x9a, 0x1e, 0x37, 0xcc, 0xbd, 0x7f, 0x6c, + 0xe8, 0x94, 0x3c, 0xaf, 0x8a, 0x8c, 0xf8, 0xd9, 0x2a, 0x8a, 0xec, 0x0e, 0x0f, 0x9b, 0x0b, 0x5a, + 0x3d, 0xf5, 0xa4, 0x2e, 0x58, 0x87, 0x45, 0x59, 0x5a, 0x87, 0xa6, 0x11, 0xda, 0x97, 0x35, 0x42, + 0x1a, 0x5d, 0x2f, 0xfd, 0xe8, 0x44, 0xce, 0xb8, 0x2c, 0x5b, 0xa8, 0xa1, 0xd8, 0x35, 0x5d, 0x81, + 0xcf, 0x71, 0xad, 0xd7, 0x68, 0x0d, 0x9a, 0xce, 0xa1, 0xba, 0xdc, 0x68, 0x48, 0x67, 0xc5, 0xf5, + 0xa2, 0x90, 0x78, 0x08, 0x1d, 0xd3, 0xbe, 0xd2, 0xe2, 0x88, 0x7a, 0xc6, 0x95, 0x51, 0x62, 0xd9, + 0x50, 0x7c, 0xbe, 0x39, 0x97, 0xdc, 0x36, 0x47, 0xe1, 0xae, 0x65, 0x5e, 0xd2, 0xe3, 0xe6, 0x1c, + 0xdb, 0x2b, 0x0d, 0x4a, 0x17, 0xf8, 0xe3, 0x1b, 0xe6, 0xe3, 0x95, 0x0a, 0x4b, 0xe3, 0xf4, 0x41, + 0x79, 0x96, 0xb8, 0x1d, 0xfe, 0xa6, 0xb7, 0xce, 0x9c, 0xd2, 0x61, 0x79, 0xe6, 0xec, 0x95, 0x06, + 0x99, 0xdb, 0xdd, 0xdc, 0x68, 0xa5, 0x42, 0x63, 0xe5, 0xfd, 0x5a, 0x83, 0xad, 0xd1, 0x7c, 0x11, + 0x27, 0x59, 0xe9, 0x16, 0x8e, 0xa2, 0x99, 0x7c, 0xa5, 0x6f, 0x21, 0x83, 0xea, 0x41, 0xc5, 0xdd, + 0x90, 0x6e, 0x23, 0xdf, 0x3e, 0x07, 0x15, 0x28, 0x9d, 0x80, 0xb3, 0x76, 0x02, 0xb7, 0xa1, 0xad, + 0xca, 0x8d, 0x54, 0x75, 0x56, 0x19, 0x81, 0x7a, 0x68, 0x2c, 0x79, 0xb8, 0x37, 0x79, 0xb8, 0x6b, + 0x48, 0x9d, 0x47, 0x99, 0xb1, 0xb2, 0xc5, 0xca, 0x92, 0x84, 0xf4, 0x47, 0xc1, 0x5c, 0xa6, 0x99, + 0x3f, 0x5f, 0xd0, 0x55, 0xb6, 0x07, 0x36, 0x96, 0x24, 0x74, 0x8b, 0x39, 0x89, 0x27, 0x89, 0xf4, + 0x33, 0x39, 0x7b, 0x9c, 0xf1, 0x09, 0xda, 0xb8, 0x21, 0x25, 0x3b, 0x4e, 0xcb, 0xd8, 0x81, 0xb2, + 0x5b, 0x97, 0x7a, 0xbf, 0xd5, 0x40, 0x28, 0xce, 0xb8, 0xf3, 0xfd, 0x7f, 0xc4, 0x5d, 0x4e, 0xd0, + 0x3a, 0x0d, 0xcd, 0x73, 0x34, 0xdc, 0x82, 0x06, 0xc7, 0xa3, 0x29, 0x28, 0x10, 0x35, 0x4a, 0xd3, + 0xa6, 0x15, 0x7f, 0x16, 0x96, 0x45, 0xc2, 0x83, 0x6e, 0x69, 0x46, 0x50, 0x81, 0x93, 0xef, 0x35, + 0x59, 0x05, 0x89, 0x70, 0x45, 0x12, 0x3b, 0x95, 0x24, 0x4e, 0xa0, 0x77, 0x94, 0xf8, 0x51, 0x1a, + 0xfa, 0x99, 0xa4, 0xf0, 0xdf, 0x84, 0xc5, 0x8a, 0x57, 0xad, 0xf7, 0x21, 0xdc, 0xdc, 0xf0, 0x6b, + 0xda, 0x2b, 0xd1, 0x6a, 0x33, 0xad, 0xb4, 0xf4, 0xc6, 0x70, 0x63, 0x65, 0x3a, 0x1a, 0xbe, 0x51, + 0x04, 0xe7, 0x9d, 0x7e, 0x54, 0xca, 0x8b, 0x9d, 0x16, 0xdb, 0x57, 0xc5, 0xba, 0x0f, 0x6e, 0x71, + 0xf7, 0xd4, 0x93, 0xba, 0x88, 0x60, 0x12, 0xc8, 0x25, 0xd9, 0x1f, 0xfa, 0x73, 0x59, 0x04, 0xc1, + 0x6b, 0x92, 0xf1, 0x78, 0xa9, 0xf1, 0x43, 0x9c, 0xd7, 0xde, 0xdf, 0x16, 0xf4, 0xaa, 0x9c, 0xf0, + 0x7b, 0x29, 0x94, 0xbe, 0x1a, 0x28, 0x2d, 0x54, 0x40, 0x3c, 0x82, 0xfa, 0x0f, 0x81, 0x5c, 0xea, + 0x81, 0xe2, 0x95, 0xde, 0x7a, 0x17, 0x44, 0x82, 0xea, 0x03, 0x2a, 0xaf, 0xc7, 0xd3, 0x2c, 0x88, + 0x23, 0xfd, 0x7a, 0x54, 0x88, 0xf6, 0xd9, 0x0f, 0xe3, 0xe9, 0x77, 0xdc, 0xb7, 0x1d, 0x54, 0xa0, + 0xa2, 0x5c, 0xea, 0x57, 0x2c, 0x97, 0x46, 0xf5, 0x9d, 0xb3, 0x34, 0x57, 0xa5, 0x09, 0xff, 0xda, + 0x13, 0x53, 0x77, 0x4c, 0x3f, 0xd5, 0xf8, 0x8e, 0xb9, 0xea, 0x99, 0x62, 0x5e, 0x63, 0x1a, 0xd2, + 0xd3, 0x88, 0x96, 0x13, 0x3f, 0x54, 0x8d, 0xab, 0x8d, 0x2b, 0xfc, 0x9a, 0x9b, 0x79, 0x3e, 0xd9, + 0x46, 0x55, 0xb2, 0xfb, 0xd7, 0x7f, 0x3f, 0xdb, 0xb1, 0xfe, 0x38, 0xdb, 0xb1, 0xfe, 0x3c, 0xdb, + 0xb1, 0x7e, 0xfe, 0x6b, 0xe7, 0xad, 0xe3, 0x06, 0xff, 0xc9, 0x7d, 0xfc, 0x5f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xb8, 0x93, 0x5b, 0x24, 0xd9, 0x0d, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -3590,6 +3598,11 @@ func (m *ImportColumnAttrsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x30 + } if len(m.ColumnIDs) > 0 { dAtA36 := make([]byte, len(m.ColumnIDs)*10) var j35 int @@ -4405,6 +4418,9 @@ func (m *ImportColumnAttrsRequest) Size() (n int) { } n += 1 + sovPublic(uint64(l)) + l } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -9229,6 +9245,25 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/internal/public.proto b/internal/public.proto index f1a1fa9fa..698581215 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -182,4 +182,5 @@ message ImportColumnAttrsRequest { string AttrKey = 3; repeated string AttrVals = 4; repeated uint64 ColumnIDs = 5; + int64 IndexCreatedAt = 6; } From d8a417f657a5eb0b6538b91d3d9959693156c8f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Fri, 29 May 2020 16:08:20 +0200 Subject: [PATCH 17/45] Move applyCreatedAt from mergeClusterStatus directly to ClusterStatus message, to avoid deadlocks --- api.go | 17 ++++++++--------- cluster.go | 3 --- pilosa.go | 2 +- server.go | 6 ++++++ 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/api.go b/api.go index 54bb8f3be..08df4889c 100644 --- a/api.go +++ b/api.go @@ -181,15 +181,17 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index if err != nil { return nil, errors.Wrap(err, "creating index") } + + createdAt := timestamp() index.mu.Lock() - index.createdAt = timestamp() + index.createdAt = createdAt index.mu.Unlock() // Send the create index message to all nodes. err = api.server.SendSync( &CreateIndexMessage{ Index: indexName, - CreatedAt: index.CreatedAt(), + CreatedAt: createdAt, Meta: &options, }) if err != nil { @@ -274,15 +276,16 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str if err != nil { return nil, errors.Wrap(err, "creating field") } + createdAt := timestamp() field.mu.Lock() - field.createdAt = timestamp() + field.createdAt = createdAt field.mu.Unlock() // Send the create field message to all nodes. err = api.server.SendSync(&CreateFieldMessage{ Index: indexName, Field: fieldName, - CreatedAt: field.CreatedAt(), + CreatedAt: createdAt, Meta: &fo, }) if err != nil { @@ -836,11 +839,7 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { } } - if err := api.holder.applySchema(s); err != nil { - return errors.Wrap(err, "applying schema") - } - - return nil + return errors.Wrap(api.holder.applySchema(s), "applying schema") } // Views returns the views in the given field. diff --git a/cluster.go b/cluster.go index ced55e67a..2c50bceb9 100644 --- a/cluster.go +++ b/cluster.go @@ -2214,9 +2214,6 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { } } - if cs.Schema != nil { - c.holder.applyCreatedAt(cs.Schema.Indexes) - } c.unprotectedSetState(cs.State) c.markAsJoined() diff --git a/pilosa.go b/pilosa.go index 060141467..53733d9ce 100644 --- a/pilosa.go +++ b/pilosa.go @@ -205,7 +205,7 @@ func stringSlicesAreEqual(a, b []string) bool { } func timestamp() int64 { - return time.Now().UTC().UnixNano() + return time.Now().UnixNano() } // AddressWithDefaults converts addr into a valid address, diff --git a/server.go b/server.go index 8684a6061..9fbb30811 100644 --- a/server.go +++ b/server.go @@ -770,6 +770,12 @@ func (s *Server) receiveMessage(m Message) error { if err != nil { return err } + if !s.isCoordinator { + if obj.Schema != nil { + s.holder.applyCreatedAt(obj.Schema.Indexes) + } + } + case *ResizeInstruction: err := s.cluster.followResizeInstruction(obj) if err != nil { From 6a892102e42be5fbca7e3eb87ed58a0ebe88501d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Wed, 3 Jun 2020 13:25:37 +0200 Subject: [PATCH 18/45] Update api-reference.md --- docs/api-reference.md | 176 ++++++++++++++++++++++++++---------------- 1 file changed, 109 insertions(+), 67 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 68dae46b5..1c365bf83 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -24,21 +24,25 @@ curl -XGET localhost:10101/index/user ``` ``` response { - "fields": [ - { - "name": "event", - "options": { - "keys": false, - "timeQuantum": "YMD", - "type": "time" - } - } - ], - "name": "user", - "options": { - "keys": false, - "trackExistence": true + "name": "user", + "createdAt": 1591178953061239000, + "options": { + "keys": false, + "trackExistence": true + }, + "fields": [ + { + "name": "event", + "createdAt": 1591178962332452000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } } + ], + "shardWidth": 1048576 } ``` @@ -57,7 +61,7 @@ The request payload is in JSON, and may contain the `options` field. The `option curl -XPOST localhost:10101/index/user -d '{"options":{"keys":true}}' ``` ``` response -{"success":true} +{"success":true,"name":"user","createdAt":1591179042178854000} ``` ### Remove index @@ -151,20 +155,34 @@ represents a particular bit to be set. Timestamps are optional, but if they exist must also contain the same number of items as rows and columns. The column IDs must all be in the shard specified in the request. +Some endpoints and data structures include a `CreatedAt` fields. +This is typically stored as a timestamp, but it's purpose is not to inform of the creation date of a particular index or field, +but to serve as a unique identifier for use in cache invalidation. + +The problem is that users of Pilosa (such as ingesters e.g. the IDK), +can usually assume that translation keys for records and field values never change - they are only appended to, and can therefore be trivially cached. +This is true except in cases where an index or field gets deleted and then recreated, +or if Pilosa is restored from a backup. +So the ingesters must send their current `CreatedAt` value which will have changed if either of those two conditions has occured (or if Pilosa was just restarted), +and the ingester will know that it needs to drop its cache. + ``` message ImportRequest { - string Index = 1; - string Field = 2; - uint64 Shard = 3; - repeated uint64 RowIDs = 4; - repeated uint64 ColumnIDs = 5; - repeated string RowKeys = 7; - repeated string ColumnKeys = 8; - repeated int64 Timestamps = 6; + string Index = 1; + string Field = 2; + uint64 Shard = 3; + repeated uint64 RowIDs = 4; + repeated uint64 ColumnIDs = 5; + repeated int64 Timestamps = 6; + repeated string RowKeys = 7; + repeated string ColumnKeys = 8; + int64 IndexCreatedAt = 9; + int64 FieldCreatedAt = 10; } ``` + ### Create field `POST /index//field/` @@ -200,7 +218,7 @@ curl localhost:10101/index/user/field/quantity \ -d '{"options": {"type": "int", "min": -1000, "max":2000}}' ``` ``` response -{"success":true} +{"success":true,"name":"quantity","createdAt":1591180110914425000} ``` Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`. @@ -209,16 +227,16 @@ Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, curl localhost:10101/index/user/field/language -X POST ``` ``` response -{"success":true} +{"success":true,"name":"language","createdAt":1591180128294321000} ``` ``` request curl localhost:10101/index/repository/field/stats \ -X POST \ - -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' + -d '{"options":{"type": "int", "min": 0, "max": 1000000}}' ``` ``` response -{"success":true} +{"success":true,"name":"stats","createdAt":1591180737881627000} ``` ### Remove field @@ -245,34 +263,52 @@ curl -XGET localhost:10101/schema ``` ``` response { - "indexes": [ + "indexes": [ + { + "name": "user", + "createdAt": 1591178953061239000, + "options": { + "keys": false, + "trackExistence": true + }, + "fields": [ { - "fields": [ - { - "name": "event", - "options": { - "keys": false, - "timeQuantum": "YMD", - "type": "time" - } - }, - { - "name": "language", - "options": { - "cacheSize": 50000, - "cacheType": "ranked", - "keys": false, - "type": "set" - } - } - ], - "name": "user", - "options": { - "keys": false, - "trackExistence": true - } + "name": "event", + "createdAt": 1591178962332452000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }, + { + "name": "language", + "createdAt": 1591180128294321000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }, + { + "name": "quantity", + "createdAt": 1591180110914425000, + "options": { + "type": "int", + "base": 0, + "bitDepth": 0, + "min": -1000, + "max": 2000, + "keys": false, + "foreignIndex": "" + } } - ] + ], + "shardWidth": 1048576 + } + ] } ``` @@ -304,7 +340,7 @@ Returns the version of the Pilosa server. curl -XGET localhost:10101/version ``` ``` response -{"version":"v0.6.0"} +{"version":"2.0.0-alpha.20-6-gb9d8d6b4"} ``` ### Get status @@ -318,19 +354,25 @@ curl -XGET localhost:10101/status ``` ```response { - "localID": "d3369125-29d8-4305-a351-b4474d14a542", - "nodes": [ - { - "id": "d3369125-29d8-4305-a351-b4474d14a542", - "isCoordinator": true, - "uri": { - "host": "localhost", - "port": 10101, - "scheme": "http" - } - } - ], - "state": "NORMAL" + "state": "NORMAL", + "nodes": [ + { + "id": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9", + "uri": { + "scheme": "http", + "host": "localhost", + "port": 10101 + }, + "grpc-uri": { + "scheme": "http", + "host": "localhost", + "port": 20101 + }, + "isCoordinator": true, + "state": "READY" + } + ], + "localID": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9" } ``` From bb048da2414629f8e7b846e0cd96e642c85fc4b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Wed, 3 Jun 2020 14:47:51 +0200 Subject: [PATCH 19/45] Update docs/api-reference.md Co-authored-by: Matthew Jaffee --- docs/api-reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 1c365bf83..e8a2ac1a3 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -159,7 +159,7 @@ Some endpoints and data structures include a `CreatedAt` fields. This is typically stored as a timestamp, but it's purpose is not to inform of the creation date of a particular index or field, but to serve as a unique identifier for use in cache invalidation. -The problem is that users of Pilosa (such as ingesters e.g. the IDK), +The problem is that users of Pilosa (such as ingesters e.g. the [IDK](https://github.com/molecula/idk)) can usually assume that translation keys for records and field values never change - they are only appended to, and can therefore be trivially cached. This is true except in cases where an index or field gets deleted and then recreated, or if Pilosa is restored from a backup. From 0320228b99fed8cfa5d8f97d3349955b14e65df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Wed, 3 Jun 2020 14:49:03 +0200 Subject: [PATCH 20/45] Change order of cluster/index locks --- cluster.go | 5 +++++ holder.go | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cluster.go b/cluster.go index 2c50bceb9..b3fc86e88 100644 --- a/cluster.go +++ b/cluster.go @@ -1044,6 +1044,11 @@ func (c *cluster) partitionNodes(partitionID int) []*Node { func (c *cluster) ownsPartition(nodeID string, partition int) bool { c.mu.RLock() defer c.mu.RUnlock() + return c.unprotectedOwnsPartition(nodeID, partition) +} + +// unprotectedOwnsPartition returns true if a host owns a partition. +func (c *cluster) unprotectedOwnsPartition(nodeID string, partition int) bool { return Nodes(c.partitionNodes(partition)).ContainsID(nodeID) } diff --git a/holder.go b/holder.go index 70501a4b1..de3cd0701 100644 --- a/holder.go +++ b/holder.go @@ -1120,7 +1120,8 @@ func (s *holderSyncer) stopTranslationSync() error { // writing new translation keys. Index stores are writable if the node owns the // partition. Field stores are writable if the node is the coordinator. func (s *holderSyncer) setTranslateReadOnlyFlags() { - isCoordinator := s.Cluster.isCoordinator() + s.Cluster.mu.RLock() + isCoordinator := s.Cluster.unprotectedIsCoordinator() for _, index := range s.Holder.Indexes() { // There is a race condition here: @@ -1140,7 +1141,7 @@ func (s *holderSyncer) setTranslateReadOnlyFlags() { // done using it. index.mu.RLock() for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ { - ownsPartition := s.Cluster.ownsPartition(s.Node.ID, partitionID) + ownsPartition := s.Cluster.unprotectedOwnsPartition(s.Node.ID, partitionID) if ts := index.TranslateStore(partitionID); ts != nil { ts.SetReadOnly(!ownsPartition) } @@ -1151,6 +1152,7 @@ func (s *holderSyncer) setTranslateReadOnlyFlags() { field.TranslateStore().SetReadOnly(!isCoordinator) } } + s.Cluster.mu.RUnlock() } // initializeIndexTranslateReplication connects to each node that is the From 6abe7dc12fb049efc3be489c96b6a2c880edf70d Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 16 Apr 2020 17:05:55 -0500 Subject: [PATCH 21/45] Don't automatically freeze the results of RowSegment ops I think when this code was written, I thought "freeze" would be really cheap. It's not actually that cheap. As a result, freezing things preemptively when it may be that nothing ever tries to write to them anyway is possibly disadvantageous, to the tune of being roughly 20% of a sample profile we were shown. Instead, we don't mark the components "writable", so if anything wants to write to them, it'll end up freezing itself new copies of their bitmaps later. But in practice that probably doesn't happen. --- row.go | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/row.go b/row.go index f1c67123c..8d2c121d2 100644 --- a/row.go +++ b/row.go @@ -574,13 +574,11 @@ func (s *rowSegment) IntersectionCount(other *rowSegment) uint64 { // Intersect returns the itersection of s and other. func (s *rowSegment) Intersect(other *rowSegment) *rowSegment { data := s.data.Intersect(other.data) - data = data.Freeze() return &rowSegment{ - data: data, - shard: s.shard, - n: data.Count(), - writable: true, + data: data, + shard: s.shard, + n: data.Count(), } } @@ -591,13 +589,11 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment { datas[i] = other.data } data := s.data.Union(datas...) - data.Freeze() return &rowSegment{ - data: data, - shard: s.shard, - n: data.Count(), - writable: true, + data: data, + shard: s.shard, + n: data.Count(), } } @@ -635,26 +631,22 @@ func (s *rowSegment) Difference(others ...*rowSegment) *rowSegment { datas[i] = other.data } data := s.data.Difference(datas...) - data.Freeze() return &rowSegment{ - data: data, - shard: s.shard, - n: data.Count(), - writable: true, + data: data, + shard: s.shard, + n: data.Count(), } } // Xor returns the xor of s and other. func (s *rowSegment) Xor(other *rowSegment) *rowSegment { data := s.data.Xor(other.data) - data = data.Freeze() return &rowSegment{ - data: data, - shard: s.shard, - n: data.Count(), - writable: true, + data: data, + shard: s.shard, + n: data.Count(), } } @@ -666,13 +658,11 @@ func (s *rowSegment) Shift() (*rowSegment, error) { if err != nil { return nil, errors.Wrap(err, "shifting roaring data") } - data = data.Freeze() return &rowSegment{ - data: data, - shard: s.shard, - n: data.Count(), - writable: true, + data: data, + shard: s.shard, + n: data.Count(), }, nil } From 439c710ca99a5ae48ec774e23ea75575fbdcef08 Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 28 May 2020 14:22:33 -0500 Subject: [PATCH 22/45] thread contexts better through executor When a mapper hits an error, we want it to immediately tell the other things in that same mapper that they can stop now. But we don't want to propagate that all the way back up; if a specific node has a failure executing a query, we will in some cases want to send a new query to other backup nodes, so the overall context isn't cancelled yet. In general, mapFn and reduceFn have been closures that inherit a context from the function defining them -- but we don't want that! We want them to be stopped if their specific mapper gets cancelled, too, because otherwise they can consume a lot of resources long after the mapper has stopped being interested in them. So now those are parameters passed into them, and mapperLocal puts *those* contexts in the jobs shoved into the job queue, and the workers pass the context in to the mapFn/reduceFn. We also check responses from reduceFn now; both mapReduce and mapperLocal check for a possible error, and return that, and reduce functions doing anything nontrivial check their context. We also add a few more explicit checks for context cancellation in various places, especially in the GroupByIterator which is what bit us that one time. The explicit check against ctx.Err is officially safe as of Go 1.9 or so. (It was previously unspecified, but on further study, the Go team concluded that no actual implementation did anything else, and existing code was already depending on that.) This also affects the rows function, because that could potentially take quite a while to run for a large fragment. --- executor.go | 168 +++++++++++++++++++++++++------------- fragment.go | 16 ++-- fragment_internal_test.go | 17 ++-- 3 files changed, 131 insertions(+), 70 deletions(-) diff --git a/executor.go b/executor.go index 1ed645475..5a486d81a 100644 --- a/executor.go +++ b/executor.go @@ -358,7 +358,9 @@ func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call row = r.Pos default: return fmt.Errorf("precomputed call %s returned unexpected non-Row data: %T", c.Name, v) - + } + if err := ctx.Err(); err != nil { + return err } c.Children = []*pql.Call{} c.Name = "Precomputed" @@ -377,6 +379,9 @@ func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call // handlePreCallChildren handles any pre-calls in the children of a given call. func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error { for i := range c.Children { + if err := ctx.Err(); err != nil { + return err + } if err := e.handlePreCalls(ctx, index, c.Children[i], shards, opt); err != nil { return err } @@ -384,6 +389,9 @@ func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *p for _, val := range c.Args { // Handle Call() operations which exist inside named arguments, too. if call, ok := val.(*pql.Call); ok { + if err := ctx.Err(); err != nil { + return err + } if err := e.handlePreCalls(ctx, index, call, shards, opt); err != nil { return err } @@ -650,12 +658,12 @@ func (e *executor) executeIncludesColumnCall(ctx context.Context, index string, } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeIncludesColumnCallShard(ctx, index, c, shard, col) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(bool) return other || v.(bool) } @@ -705,12 +713,12 @@ func (e *executor) executeFieldValueCall(ctx context.Context, index string, c *p shard := colID / ShardWidth // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeFieldValueCallShard(ctx, field, colID, shard) } // Select single returned result at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(ValCount) if other.Count == 1 { return other @@ -834,12 +842,15 @@ func (e *executor) executeAllCall(ctx context.Context, index string, c *pql.Call // using the executor.mapReduce() method. func (e *executor) executeAllCallMapReduce(ctx context.Context, index string, c *pql.Call, shard uint64, opt *execOptions) (*Row, error) { // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeAllCallShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { + if err := ctx.Err(); err != nil { + return err + } other, _ := prev.(*Row) if other == nil { other = NewRow() @@ -889,12 +900,12 @@ func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, sh } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeSumCountShard(ctx, index, c, nil, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(ValCount) return other.add(v.(ValCount)) } @@ -941,13 +952,16 @@ func (e *executor) executeGenericField(ctx context.Context, index string, c *pql } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeGenericFieldShard(ctx, index, c, op, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(SignedRow) + if err := ctx.Err(); err != nil { + return err + } return other.union(v.(SignedRow)) } @@ -974,12 +988,12 @@ func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, sh } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeMinShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(ValCount) return other.smaller(v.(ValCount)) } @@ -1010,12 +1024,12 @@ func (e *executor) executeMax(ctx context.Context, index string, c *pql.Call, sh } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeMaxShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(ValCount) return other.larger(v.(ValCount)) } @@ -1042,12 +1056,12 @@ func (e *executor) executeMinRow(ctx context.Context, index string, c *pql.Call, } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeMinRowShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { // if minRowID exists, and if it is smaller than the other one return it. // otherwise return the minRowID of the one which exists. if prev == nil { @@ -1081,12 +1095,12 @@ func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call, } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeMaxRowShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { // if minRowID exists, and if it is smaller than the other one return it. // otherwise return the minRowID of the one which exists. if prev == nil { @@ -1138,16 +1152,19 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeBitmapCallShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(*Row) if other == nil { other = NewRow() } + if err := ctx.Err(); err != nil { + return err + } other.Merge(v.(*Row)) return other } @@ -1505,12 +1522,12 @@ func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.C defer span.Finish() // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeTopNShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(*PairsField) vpf, _ := v.(*PairsField) if other == nil { @@ -1518,6 +1535,9 @@ func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.C } else if vpf == nil { return other } + if err := ctx.Err(); err != nil { + return err + } other.Pairs = Pairs(other.Pairs).Add(vpf.Pairs) return other } @@ -1780,12 +1800,15 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeGroupByShard(ctx, index, c, filter, shard, childRows) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.([]GroupCount) + if err := ctx.Err(); err != nil { + return err + } return mergeGroupCounts(other, v.([]GroupCount), limit) } // Get full result set. @@ -2199,7 +2222,7 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeRowsShard(ctx, index, fieldName, c, shard) } @@ -2212,8 +2235,11 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(RowIDs) + if err := ctx.Err(); err != nil { + return err + } return other.merge(v.(RowIDs), limit) } // Get full result set. @@ -2225,7 +2251,7 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s return results, nil } -func (e *executor) executeRowsShard(_ context.Context, index string, fieldName string, c *pql.Call, shard uint64) (RowIDs, error) { +func (e *executor) executeRowsShard(ctx context.Context, index string, fieldName string, c *pql.Call, shard uint64) (RowIDs, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { @@ -2335,12 +2361,15 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s } for _, view := range views { + if err := ctx.Err(); err != nil { + return nil, err + } frag := e.Holder.fragment(index, fieldName, view, shard) if frag == nil { continue } - viewRows := frag.rows(start, filters...) + viewRows := frag.rows(ctx, start, filters...) rowIDs = rowIDs.merge(viewRows, limit) } @@ -2814,7 +2843,7 @@ func (e *executor) executeGenericCount(ctx context.Context, index string, c *pql } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return 0, err @@ -2823,7 +2852,7 @@ func (e *executor) executeGenericCount(ctx context.Context, index string, c *pql } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(uint64) return other + v.(uint64) } @@ -2849,7 +2878,7 @@ func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return 0, err @@ -2858,7 +2887,7 @@ func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(uint64) return other + v.(uint64) } @@ -2971,12 +3000,12 @@ func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Cal } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeClearRowShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { val := v.(bool) if prev == nil { return val @@ -3059,12 +3088,12 @@ func (e *executor) executeSetRow(ctx context.Context, indexName string, c *pql.C } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeSetRowShard(ctx, indexName, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { val := v.(bool) if prev == nil { return val @@ -3593,7 +3622,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, } // Start mapping across all primary owners. - if err := e.mapper(ctx, ch, nodes, index, shards, c, opt, mapFn, reduceFn); err != nil { + if err := e.mapper(ctx, cancel, ch, nodes, index, shards, c, opt, mapFn, reduceFn); err != nil { return nil, errors.Wrap(err, "starting mapper") } @@ -3613,7 +3642,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, nodes = Nodes(nodes).Filter(resp.node) // Begin mapper against secondary nodes. - if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable { + if err := e.mapper(ctx, cancel, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable { return nil, resp.err } else if err != nil { return nil, errors.Wrap(err, "calling mapper") @@ -3622,7 +3651,11 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, } // Reduce value. - result = reduceFn(result, resp.result) + result = reduceFn(ctx, result, resp.result) + if err, ok := result.(error); ok { + cancel() + return nil, err + } // If all shards have been processed then return. shardN += len(resp.shards) @@ -3670,9 +3703,10 @@ func makeEmbeddedDataForShards(allRows []*Row, shards []uint64) []*Row { return newRows } -func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error { +func (e *executor) mapper(ctx context.Context, cancel context.CancelFunc, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapper") defer span.Finish() + done := ctx.Done() // Group shards together by nodes. m, err := e.shardsByNode(nodes, index, shards) @@ -3699,11 +3733,16 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod } resp.err = err } - // Return response to the channel. select { - case <-ctx.Done(): + case <-done: case ch <- resp: + // The cancel coming after the above send is intentional. + // We want to report the actual error that happened + // before we cause anything to return "context canceled". + if resp.err != nil { + cancel() + } } }(n, nodeShards) } @@ -3720,7 +3759,7 @@ type job struct { func worker(work chan job) { for j := range work { - result, err := j.mapFn(j.shard) + result, err := j.mapFn(j.ctx, j.shard) select { case <-j.ctx.Done(): @@ -3733,6 +3772,9 @@ func worker(work chan job) { func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapperLocal") defer span.Finish() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + done := ctx.Done() ch := make(chan mapResponse, len(shards)) @@ -3750,13 +3792,17 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu var result interface{} for { select { - case <-ctx.Done(): + case <-done: return nil, ctx.Err() case resp := <-ch: if resp.err != nil { return nil, resp.err } - result = reduceFn(result, resp.result) + result = reduceFn(ctx, result, resp.result) + if err, ok := result.(error); ok { + cancel() + return nil, err + } maxShard++ } @@ -4268,9 +4314,9 @@ func validateQueryContext(ctx context.Context) error { // errShardUnavailable is a marker error if no nodes are available. var errShardUnavailable = errors.New("shard unavailable") -type mapFunc func(shard uint64) (interface{}, error) +type mapFunc func(ctx context.Context, shard uint64) (interface{}, error) -type reduceFunc func(prev, v interface{}) interface{} +type reduceFunc func(ctx context.Context, prev, v interface{}) interface{} type mapResponse struct { node *Node @@ -4770,18 +4816,21 @@ func newGroupByIterator(executor *executor, rowIDs []RowIDs, children []*pql.Cal // nextAtIdx is a recursive helper method for getting the next row for the field // at index i, and then updating the rows in the "higher" fields if it wraps. -func (gbi *groupByIterator) nextAtIdx(i int) { +func (gbi *groupByIterator) nextAtIdx(ctx context.Context, i int) (err error) { // loop until we find a non-empty row. This is an optimization - the loop and if/break can be removed. for { + if err = ctx.Err(); err != nil { + return err + } nr, rowID, value, wrapped := gbi.rowIters[i].Next() if nr == nil { gbi.done = true - return + return nil } if wrapped && i != 0 { - gbi.nextAtIdx(i - 1) - if gbi.done { - return + err = gbi.nextAtIdx(ctx, i-1) + if gbi.done || err != nil { + return err } } if i == 0 && gbi.filter != nil { @@ -4798,6 +4847,7 @@ func (gbi *groupByIterator) nextAtIdx(i int) { break } } + return nil } // Next returns a GroupCount representing the next group by record. When there @@ -4805,6 +4855,9 @@ func (gbi *groupByIterator) nextAtIdx(i int) { func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool, err error) { // loop until we find a result with count > 0 for { + if err := ctx.Err(); err != nil { + return ret, false, err + } if gbi.done { return ret, true, nil } @@ -4832,7 +4885,10 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool } } if ret.Count == 0 { - gbi.nextAtIdx(len(gbi.rows) - 1) + err := gbi.nextAtIdx(ctx, len(gbi.rows)-1) + if err != nil { + return ret, false, err + } continue } break @@ -4847,9 +4903,9 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool } // set up for next call - gbi.nextAtIdx(len(gbi.rows) - 1) + err = gbi.nextAtIdx(ctx, len(gbi.rows)-1) - return ret, false, nil + return ret, false, err } // getCondIntSlice looks at the field, the cond op type (which is diff --git a/fragment.go b/fragment.go index 81f644e3c..7e1bd8c54 100644 --- a/fragment.go +++ b/fragment.go @@ -2607,14 +2607,14 @@ func filterWithRows(rows []uint64) rowFilter { // returning done == true will cause processing to stop after all filters for // this container have been processed. The rows accumulated up to this point // (including this row if all filters passed) will be returned. -func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 { +func (f *fragment) rows(ctx context.Context, start uint64, filters ...rowFilter) []uint64 { f.mu.RLock() defer f.mu.RUnlock() - return f.unprotectedRows(start, filters...) + return f.unprotectedRows(ctx, start, filters...) } // unprotectedRows calls rows without grabbing the mutex. -func (f *fragment) unprotectedRows(start uint64, filters ...rowFilter) []uint64 { +func (f *fragment) unprotectedRows(ctx context.Context, start uint64, filters ...rowFilter) []uint64 { startKey := rowToKey(start) i, _ := f.storage.Containers.Iterator(startKey) rows := make([]uint64, 0) @@ -2622,6 +2622,10 @@ func (f *fragment) unprotectedRows(start uint64, filters ...rowFilter) []uint64 // Loop over the existing containers. for i.Next() { + // caller doesn't need a result anymore. + if ctx.Err() != nil { + return nil + } key, c := i.Value() // virtual row for the current container @@ -2861,7 +2865,7 @@ type setRowIterator struct { func (f *fragment) setRowIterator(wrap bool, filters ...rowFilter) rowIterator { return &setRowIterator{ f: f, - rowIDs: f.rows(0, filters...), // TODO: this may be memory intensive in high cardinality cases + rowIDs: f.rows(context.Background(), 0, filters...), // TODO: this may be memory intensive in high cardinality cases wrap: wrap, } } @@ -3282,7 +3286,7 @@ func newRowsVector(f *fragment) *rowsVector { // otherwise it returns false. Ensure that you already // have the mutex before calling this. func (v *rowsVector) Get(colID uint64) (uint64, bool, error) { - rows := v.f.unprotectedRows(0, filterColumn(colID)) + rows := v.f.unprotectedRows(context.Background(), 0, filterColumn(colID)) if len(rows) > 1 { return 0, false, errors.New("found multiple row values for column") } else if len(rows) == 1 { @@ -3316,7 +3320,7 @@ func newBoolVector(f *fragment) *boolVector { // otherwise it returns false. Ensure that you already // have the fragment mutex before calling this. func (v *boolVector) Get(colID uint64) (uint64, bool, error) { - rows := v.f.unprotectedRows(0, filterColumn(colID)) + rows := v.f.unprotectedRows(context.Background(), 0, filterColumn(colID)) if len(rows) > 1 { return 0, false, errors.New("found multiple row values for column") } else if len(rows) == 1 { diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 3394825f9..b8a60d86a 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -2412,8 +2412,9 @@ func TestGetZipfRowsSliceRoaring(t *testing.T) { if err != nil { t.Fatalf("importing roaring: %v", err) } - if !reflect.DeepEqual(f.rows(0), []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) { - t.Fatalf("unexpected rows: %v", f.rows(0)) + rows := f.rows(context.Background(), 0) + if !reflect.DeepEqual(rows, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) { + t.Fatalf("unexpected rows: %v", rows) } for i := uint64(1); i < 10; i++ { if f.row(i).Count() >= f.row(i-1).Count() { @@ -2714,12 +2715,12 @@ func TestFragment_RowsIteration(t *testing.T) { } } - ids := f.rows(0) + ids := f.rows(context.Background(), 0) if !reflect.DeepEqual(expectedAll, ids) { t.Fatalf("Do not match %v %v", expectedAll, ids) } - ids = f.rows(0, filterColumn(1)) + ids = f.rows(context.Background(), 0, filterColumn(1)) if !reflect.DeepEqual(expectedOdd, ids) { t.Fatalf("Do not match %v %v", expectedOdd, ids) } @@ -2738,12 +2739,12 @@ func TestFragment_RowsIteration(t *testing.T) { t.Fatal(err) } - ids := f.rows(0) + ids := f.rows(context.Background(), 0) if !reflect.DeepEqual(expected, ids) { t.Fatalf("Do not match %v %v", expected, ids) } - ids = f.rows(0, filterColumn(66000)) + ids = f.rows(context.Background(), 0, filterColumn(66000)) if !reflect.DeepEqual(expected, ids) { t.Fatalf("Do not match %v %v", expected, ids) } @@ -2761,11 +2762,11 @@ func TestFragment_RowsIteration(t *testing.T) { t.Fatal(err) } - ids := f.rows(0) + ids := f.rows(context.Background(), 0) if !reflect.DeepEqual(expectedRows, ids) { t.Fatalf("Do not match %v %v", expectedRows, ids) } - ids = f.rows(0, filterColumn(c)) + ids = f.rows(context.Background(), 0, filterColumn(c)) if !reflect.DeepEqual(expectedRows, ids) { t.Fatalf("Do not match %v %v", expectedRows, ids) } From 023efcaba6b616dda2f02f4deae24b42b8033cf7 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Wed, 3 Jun 2020 19:48:22 -0400 Subject: [PATCH 23/45] track active queries --- api.go | 14 ++++++ http/handler.go | 74 +++++++++++++++++++++++++++- tracker.go | 125 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 tracker.go diff --git a/api.go b/api.go index 08df4889c..5f5b3807f 100644 --- a/api.go +++ b/api.go @@ -45,6 +45,7 @@ type API struct { holder *Holder cluster *cluster server *Server + tracker *queryTracker importWorkersWG sync.WaitGroup importWorkerPoolSize int @@ -95,6 +96,8 @@ func NewAPI(opts ...apiOption) (*API, error) { }() } + api.tracker = newQueryTracker() + return api, nil } @@ -130,6 +133,7 @@ func (api *API) validate(f apiMethod) error { func (api *API) Close() error { close(api.importWork) api.importWorkersWG.Wait() + api.tracker.Stop() return nil } @@ -146,6 +150,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er if err != nil { return QueryResponse{}, errors.Wrap(err, "parsing") } + defer api.tracker.Finish(api.tracker.Start(req.Query)) execOpts := &execOptions{ Remote: req.Remote, Profile: req.Profile, @@ -1701,6 +1706,13 @@ func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Tr return t, err } +func (api *API) ActiveQueries(ctx context.Context) ([]ActiveQueryStatus, error) { + if err := api.validate(apiActiveQueries); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + return api.tracker.ActiveQueries(), nil +} + type serverInfo struct { ShardWidth uint64 `json:"shardWidth"` Memory uint64 `json:"memory"` @@ -1752,6 +1764,7 @@ const ( apiFinishTransaction apiTransactions apiGetTransaction + apiActiveQueries ) var methodsCommon = map[apiMethod]struct{}{ @@ -1791,4 +1804,5 @@ var methodsNormal = map[apiMethod]struct{}{ apiFinishTransaction: {}, apiTransactions: {}, apiGetTransaction: {}, + apiActiveQueries: {}, } diff --git a/http/handler.go b/http/handler.go index 12773f3ee..654eea644 100644 --- a/http/handler.go +++ b/http/handler.go @@ -23,6 +23,7 @@ import ( "fmt" "io" "math" + "mime" "net" "net/http" _ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server. @@ -358,6 +359,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/transaction/{id}", handler.handlePostTransaction).Methods("POST").Name("PostTransaction") router.HandleFunc("/transaction/{id}/finish", handler.handlePostFinishTransaction).Methods("POST").Name("PostFinishTransaction") router.HandleFunc("/transactions", handler.handleGetTransactions).Methods("GET").Name("GetTransactions") + router.HandleFunc("/queries", handler.handleGetActiveQueries).Methods("GET").Name("GetActiveQueries") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion") // /internal endpoints are for internal use only; they may change at any time. @@ -475,9 +477,33 @@ func (h *Handler) handleHome(w http.ResponseWriter, _ *http.Request) { // headers are present, but none of them are "application/json" // (or any matching wildcard). Otherwise returns true. func validHeaderAcceptJSON(header http.Header) bool { + return validHeaderAcceptType(header, "application", "json") +} + +func validHeaderAcceptType(header http.Header, typ, subtyp string) bool { if v, found := header["Accept"]; found { for _, v := range v { - if v == "application/json" || v == "*/*" || v == "*/json" || v == "application/*" { + t, _, err := mime.ParseMediaType(v) + if err != nil { + switch err { + case mime.ErrInvalidMediaParameter: + // This is an optional feature, so we can keep going anyway. + default: + continue + } + } + spl := strings.SplitN(t, "/", 2) + if len(spl) < 2 { + continue + } + switch { + case spl[0] == typ && spl[1] == subtyp: + return true + case spl[0] == "*" && spl[1] == subtyp: + return true + case spl[0] == typ && spl[1] == "*": + return true + case spl[0] == "*" && spl[1] == "*": return true } } @@ -835,6 +861,52 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request } } +func (h *Handler) handleGetActiveQueries(w http.ResponseWriter, r *http.Request) { + var rtype string + switch { + case validHeaderAcceptType(r.Header, "text", "plain"): + rtype = "text/plain" + case validHeaderAcceptJSON(r.Header): + rtype = "application/json" + default: + http.Error(w, "no acceptable response type selected", http.StatusNotAcceptable) + return + } + queries, err := h.api.ActiveQueries(r.Context()) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", rtype) + switch rtype { + case "text/plain": + lines := make([]string, len(queries)) + for i, q := range queries { + lines[i] = q.Age.String() + } + var maxlen int + for _, l := range lines { + if len(l) > maxlen { + maxlen = len(l) + } + } + spaces := strings.Repeat(" ", maxlen+2) + for i, l := range lines { + lines[i] += spaces[len(l):] + } + for i, q := range queries { + lines[i] += q.Query + } + if _, err := w.Write([]byte(strings.Join(lines, "\n") + "\n")); err != nil { + h.logger.Printf("sending GetActiveQueries response: %s", err) + } + case "application/json": + if err := json.NewEncoder(w).Encode(queries); err != nil { + h.logger.Printf("encoding GetActiveQueries response: %s", err) + } + } +} + type postIndexAttrDiffRequest struct { Blocks []pilosa.AttrBlock `json:"blocks"` } diff --git a/tracker.go b/tracker.go new file mode 100644 index 000000000..af5788137 --- /dev/null +++ b/tracker.go @@ -0,0 +1,125 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "sort" + "sync" + "time" +) + +type ActiveQueryStatus struct { + Query string `json:"query"` + Age time.Duration `json:"age"` +} + +type activeQuery struct { + query string + started time.Time +} + +type queryStatusUpdate struct { + q *activeQuery + end bool +} + +type queryTracker struct { + updates chan<- queryStatusUpdate + checks chan<- chan<- []*activeQuery + wg sync.WaitGroup + stop chan struct{} +} + +func newQueryTracker() *queryTracker { + done := make(chan struct{}) + updates := make(chan queryStatusUpdate, 128) + checks := make(chan chan<- []*activeQuery) + tracker := &queryTracker{ + updates: updates, + checks: checks, + stop: done, + } + tracker.wg.Add(1) + go func() { + defer tracker.wg.Done() + + activeQueries := make(map[*activeQuery]struct{}) + + for { + select { + case update := <-updates: + if update.end { + delete(activeQueries, update.q) + } else { + activeQueries[update.q] = struct{}{} + } + case check := <-checks: + out := make([]*activeQuery, len(activeQueries)) + i := 0 + for q := range activeQueries { + out[i] = q + i++ + } + check <- out + close(check) + case <-done: + return + } + } + }() + return tracker +} + +func (t *queryTracker) Start(query string) *activeQuery { + now := time.Now() + q := &activeQuery{query, now} + t.updates <- queryStatusUpdate{q, false} + return q +} + +func (t *queryTracker) Finish(q *activeQuery) { + t.updates <- queryStatusUpdate{q, true} +} + +func (t *queryTracker) ActiveQueries() []ActiveQueryStatus { + ch := make(chan []*activeQuery, 1) + t.checks <- ch + queries := <-ch + sort.Slice(queries, func(i, j int) bool { + switch { + case queries[i].started.Before(queries[j].started): + return true + case queries[i].started.After(queries[j].started): + return false + case queries[i].query < queries[j].query: + return true + case queries[i].query > queries[j].query: + return false + default: + return false + } + }) + now := time.Now() + out := make([]ActiveQueryStatus, len(queries)) + for i, v := range queries { + out[i] = ActiveQueryStatus{v.query, now.Sub(v.started)} + } + return out +} + +func (t *queryTracker) Stop() { + close(t.stop) + t.wg.Wait() +} From 1099a57945481439b91f9498512521458bcc84af Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Thu, 4 Jun 2020 13:40:34 -0400 Subject: [PATCH 24/45] fix pretty printing of active queries list to handle special characters and multiline queries --- http/handler.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/http/handler.go b/http/handler.go index 654eea644..f6e0ac001 100644 --- a/http/handler.go +++ b/http/handler.go @@ -880,24 +880,24 @@ func (h *Handler) handleGetActiveQueries(w http.ResponseWriter, r *http.Request) w.Header().Set("Content-Type", rtype) switch rtype { case "text/plain": - lines := make([]string, len(queries)) + durations := make([]string, len(queries)) for i, q := range queries { - lines[i] = q.Age.String() + durations[i] = q.Age.String() } var maxlen int - for _, l := range lines { + for _, l := range durations { if len(l) > maxlen { maxlen = len(l) } } - spaces := strings.Repeat(" ", maxlen+2) - for i, l := range lines { - lines[i] += spaces[len(l):] - } for i, q := range queries { - lines[i] += q.Query + _, err := fmt.Fprintf(w, "%*s%q\n", -(maxlen + 2), durations[i], q.Query) + if err != nil { + h.logger.Printf("sending GetActiveQueries response: %s", err) + return + } } - if _, err := w.Write([]byte(strings.Join(lines, "\n") + "\n")); err != nil { + if _, err := w.Write([]byte{'\n'}); err != nil { h.logger.Printf("sending GetActiveQueries response: %s", err) } case "application/json": From 06517075bf982a145a49370a6b4086f48476e084 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Thu, 4 Jun 2020 14:05:06 -0400 Subject: [PATCH 25/45] add unit test to active query tracker --- tracker_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tracker_test.go diff --git a/tracker_test.go b/tracker_test.go new file mode 100644 index 000000000..80f6f84f2 --- /dev/null +++ b/tracker_test.go @@ -0,0 +1,42 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import "testing" + +func TestQueryTracker(t *testing.T) { + tracker := newQueryTracker() + defer tracker.Stop() + + if queries := tracker.ActiveQueries(); len(queries) > 0 { + t.Fatalf("expected no active queries; found %v", queries) + } + + qs := tracker.Start("test query") + + var queries []ActiveQueryStatus + for len(queries) < 1 { + queries = tracker.ActiveQueries() + } + if len(queries) > 1 || queries[0].Query != "test query" { + t.Fatalf("unexpected queries: %v", queries) + } + + tracker.Finish(qs) + + for len(queries) > 0 { + queries = tracker.ActiveQueries() + } +} From 0ce5d9240713bc800d749ed62f8d2051ecf86623 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Fri, 5 Jun 2020 13:38:13 -0400 Subject: [PATCH 26/45] simplify BSI comparisons --- fragment.go | 303 +++++++++++++++++++++++++++------------------------- 1 file changed, 155 insertions(+), 148 deletions(-) diff --git a/fragment.go b/fragment.go index 7e1bd8c54..3cc4ca9ba 100644 --- a/fragment.go +++ b/fragment.go @@ -1159,14 +1159,24 @@ func (f *fragment) rangeOp(op pql.Token, bitDepth uint, predicate int64) (*Row, } } +func absInt64(v int64) uint64 { + switch { + case v > 0: + return uint64(v) + case v == -9223372036854775808: + return 9223372036854775808 + default: + return uint64(-v) + } +} + func (f *fragment) rangeEQ(bitDepth uint, predicate int64) (*Row, error) { // Start with set of columns with values set. b := f.row(bsiExistsBit) // Filter to only positive/negative numbers. - upredicate := uint64(predicate) + upredicate := absInt64(predicate) if predicate < 0 { - upredicate = uint64(-predicate) b = b.Intersect(f.row(bsiSignBit)) // only negatives } else { b = b.Difference(f.row(bsiSignBit)) // only positives @@ -1204,27 +1214,42 @@ func (f *fragment) rangeNEQ(bitDepth uint, predicate int64) (*Row, error) { } func (f *fragment) rangeLT(bitDepth uint, predicate int64, allowEquality bool) (*Row, error) { + if predicate == 1 && !allowEquality { + predicate, allowEquality = 0, true + } + // Start with set of columns with values set. b := f.row(bsiExistsBit) - // Create predicate without sign bit. - upredicate := uint64(predicate) - if predicate < 0 { - upredicate = uint64(-predicate) - } + // Get the sign bit row. + sign := f.row(bsiSignBit) - // If predicate is positive, return all positives less than predicate and all negatives. - if (predicate >= 0 && allowEquality) || (predicate >= -1 && !allowEquality) { - pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) + // Create predicate without sign bit. + upredicate := absInt64(predicate) + + switch { + case predicate == 0 && !allowEquality: + // Match all negative integers. + return b.Intersect(sign), nil + case predicate == 0 && allowEquality: + // Match all integers that are either negative or 0. + zeroes, err := f.rangeEQ(bitDepth, 0) if err != nil { return nil, err } - neg := f.row(bsiSignBit) - return neg.Union(pos), nil + return b.Intersect(sign).Union(zeroes), nil + case predicate < 0: + // Match all every negative number beyond the predicate. + return f.rangeGTUnsigned(b.Intersect(sign), bitDepth, upredicate, allowEquality) + default: + // Match positive numbers less than the predicate, and all negatives. + pos, err := f.rangeLTUnsigned(b.Difference(sign), bitDepth, upredicate, allowEquality) + if err != nil { + return nil, err + } + neg := b.Intersect(sign) + return pos.Union(neg), nil } - - // Otherwise if predicate is negative, return all negatives greater than upredicate. - return f.rangeGTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) } // msb gives the 1-indexed position (counting from lsb) of the most @@ -1237,110 +1262,99 @@ func msb(x uint64) uint { // rangeLTUnsigned returns all bits LT/LTE the predicate without considering the sign bit. func (f *fragment) rangeLTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - keep := NewRow() - - // if the predicate is larger than all representable numbers given - // our bitDepth... then just return everything. - if msb(predicate) > bitDepth { + if msb(predicate) > bitDepth || (allowEquality && predicate == (1<= 0; i-- { row := f.row(uint64(bsiOffsetBit + i)) - bit := (predicate >> uint(i)) & 1 - - // Remove any columns with higher bits set. - if leadingZeros { - if bit == 0 { - filter = filter.Difference(row) - continue - } else { - leadingZeros = false - } - } - - // Handle last bit differently. - // If bit is zero then return only already kept columns. - // If bit is one then remove any one columns. - if i == 0 && !allowEquality { - if bit == 0 { - return keep, nil - } - return filter.Difference(row.Difference(keep)), nil - } - - // If bit is zero then remove all set columns not in excluded bitmap. - if bit == 0 { - filter = filter.Difference(row.Difference(keep)) - continue - } - - // If bit is set then add columns for set bits to exclude. - // Don't bother to compute this on the final iteration. - if i > 0 { - keep = keep.Union(filter.Difference(row)) + zeroes := remaining.Difference(row) + switch (predicate >> uint(i)) & 1 { + case 1: + // Match everything with a zero bit here. + matched = matched.Union(zeroes) + case 0: + // Discard everything with a one bit here. + remaining = zeroes } } - return filter, nil + if allowEquality { + matched = matched.Union(remaining) + } + + return matched, nil } func (f *fragment) rangeGT(bitDepth uint, predicate int64, allowEquality bool) (*Row, error) { + if predicate == -1 && !allowEquality { + predicate, allowEquality = 0, true + } + b := f.row(bsiExistsBit) // Create predicate without sign bit. - upredicate := uint64(predicate) - if predicate < 0 { - upredicate = uint64(-predicate) - } + upredicate := absInt64(predicate) - // If predicate is positive, return all positives greater than predicate. - if (predicate >= 0 && allowEquality) || (predicate >= -1 && !allowEquality) { - return f.rangeGTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) - } + sign := f.row(bsiSignBit) - // If predicate is negative, return all negatives less than than upredicate and all positives. - neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) - if err != nil { - return nil, err + switch { + case predicate == 0 && !allowEquality: + // Match all positive numbers except zero. + nonzero, err := f.rangeNEQ(bitDepth, 0) + if err != nil { + return nil, err + } + b = nonzero + fallthrough + case predicate == 0 && allowEquality: + // Match all positive numbers. + return b.Difference(sign), nil + case predicate >= 0: + // Match all positive numbers greater than the predicate. + return f.rangeGTUnsigned(b.Difference(sign), bitDepth, upredicate, allowEquality) + default: + // Match all positives and greater negatives. + neg, err := f.rangeLTUnsigned(b.Intersect(sign), bitDepth, upredicate, allowEquality) + if err != nil { + return nil, err + } + pos := b.Difference(sign) + return pos.Union(neg), nil } - pos := b.Difference(f.row(bsiSignBit)) - return pos.Union(neg), nil } func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - keep := NewRow() - // Filter any bits that don't match the current bit value. + if allowEquality && predicate == 0 { + // This query matches all possible values. + return filter, nil + } + + // Compare intermediate bits. + matched := NewRow() + remaining := filter for i := int(bitDepth - 1); i >= 0; i-- { row := f.row(uint64(bsiOffsetBit + i)) - bit := (predicate >> uint(i)) & 1 - - // Handle last bit differently. - // If bit is one then return only already kept columns. - // If bit is zero then remove any unset columns. - if i == 0 && !allowEquality { - if bit == 1 { - return keep, nil - } - return filter.Difference(filter.Difference(row, keep)), nil - } - - // If bit is set then remove all unset columns not already kept. - if bit == 1 { - filter = filter.Difference(filter.Difference(row, keep)) - continue - } - - // If bit is unset then add columns with set bit to keep. - // Don't bother to compute this on the final iteration. - if i > 0 { - keep = keep.Union(filter.Intersect(row)) + ones := remaining.Intersect(row) + switch (predicate >> uint(i)) & 1 { + case 1: + // Discard everything with a zero bit here. + remaining = ones + case 0: + // Match everything with a one bit here. + matched = matched.Union(ones) } } - return filter, nil + if allowEquality { + matched = matched.Union(remaining) + } + + return matched, nil } // notNull returns the exists row. @@ -1353,73 +1367,66 @@ func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax int64) b := f.row(bsiExistsBit) // Convert predicates to unsigned values. - upredicateMin, upredicateMax := uint64(predicateMin), uint64(predicateMax) - if predicateMin < 0 { - upredicateMin = uint64(-predicateMin) - } - if predicateMax < 0 { - upredicateMax = uint64(-predicateMax) - } + upredicateMin, upredicateMax := absInt64(predicateMin), absInt64(predicateMax) - // Handle positive-only values. - if predicateMin >= 0 { + switch { + case predicateMin >= 0: + // Handle positive-only values. return f.rangeBetweenUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMin, upredicateMax) - } - - // Handle negative-only values. Swap unsigned min/max predicates. - if predicateMax < 0 { + case predicateMax < 0: + // Handle negative-only values. Swap unsigned min/max predicates. return f.rangeBetweenUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMax, upredicateMin) + default: + // If predicate crosses positive/negative boundary then handle separately and union. + pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMax, true) + if err != nil { + return nil, err + } + neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMin, true) + if err != nil { + return nil, err + } + return pos.Union(neg), nil } - - // If predicate crosses positive/negative boundary then handle separately and union. - pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMax, true) - if err != nil { - return nil, err - } - neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMin, true) - if err != nil { - return nil, err - } - return pos.Union(neg), nil } // rangeBetweenUnsigned returns BSI columns for a range of values. Disregards the sign bit. func (f *fragment) rangeBetweenUnsigned(filter *Row, bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { - keep1 := NewRow() // GTE - keep2 := NewRow() // LTE + switch { + case predicateMax-predicateMin < 2: + // This query matches all possible values. + return filter, nil + case predicateMax > (1<= 0; i-- { + // Compare any upper bits which are equal. + firstDiff := int(msb(predicateMax^predicateMin)) - 1 + remaining := filter + for i := int(bitDepth - 1); i > firstDiff; i-- { row := f.row(uint64(bsiOffsetBit + i)) - bit1 := (predicateMin >> uint(i)) & 1 - bit2 := (predicateMax >> uint(i)) & 1 - - // GTE predicateMin - // If bit is set then remove all unset columns not already kept. - if bit1 == 1 { - filter = filter.Difference(filter.Difference(row, keep1)) - } else { - // If bit is unset then add columns with set bit to keep. - // Don't bother to compute this on the final iteration. - if i > 0 { - keep1 = keep1.Union(filter.Intersect(row)) - } - } - - // LTE predicateMax - // If bit is zero then remove all set bits not in excluded bitmap. - if bit2 == 0 { - filter = filter.Difference(row.Difference(keep2)) - } else { - // If bit is set then add columns for set bits to exclude. - // Don't bother to compute this on the final iteration. - if i > 0 { - keep2 = keep2.Union(filter.Difference(row)) - } + switch (predicateMin >> uint(i)) & 1 { + case 1: + remaining = remaining.Intersect(row) + case 0: + remaining = remaining.Difference(row) } } - return filter, nil + var err error + remaining, err = f.rangeGTUnsigned(remaining, uint(firstDiff+1), predicateMin, true) + if err != nil { + return nil, err + } + remaining, err = f.rangeLTUnsigned(remaining, uint(firstDiff+1), predicateMax, true) + if err != nil { + return nil, err + } + return remaining, nil } // pos translates the row ID and column ID into a position in the storage bitmap. From b0a0524ffe49feaac2e34d4560006e4e646041f3 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Fri, 5 Jun 2020 15:25:35 -0400 Subject: [PATCH 27/45] cleanly shut down the executor --- executor.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/executor.go b/executor.go index 5a486d81a..390f85654 100644 --- a/executor.go +++ b/executor.go @@ -59,6 +59,8 @@ type executor struct { // Maximum number of Set() or Clear() commands per request. MaxWritesPerRequest int + shutdown bool + workMu sync.RWMutex workersWG sync.WaitGroup workerPoolSize int work chan job @@ -115,6 +117,9 @@ func newExecutor(opts ...executorOption) *executor { } func (e *executor) Close() error { + e.workMu.Lock() + defer e.workMu.Unlock() + e.shutdown = true close(e.work) e.workersWG.Wait() return nil @@ -3768,6 +3773,8 @@ func worker(work chan job) { } } +var errShutdown = errors.New("executor has shut down") + // mapperLocal performs map & reduce entirely on the local node. func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapperLocal") @@ -3775,6 +3782,12 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu ctx, cancel := context.WithCancel(ctx) defer cancel() done := ctx.Done() + e.workMu.RLock() + defer e.workMu.RUnlock() + + if e.shutdown { + return nil, errShutdown + } ch := make(chan mapResponse, len(shards)) From 7695e63bc5a3c187c2aa967815eb82075598be84 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Mon, 8 Jun 2020 11:40:53 -0400 Subject: [PATCH 28/45] fix CPU speed on non-Intel platforms --- gopsutil/systeminfo.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go index cb526ca4d..b251c39a4 100644 --- a/gopsutil/systeminfo.go +++ b/gopsutil/systeminfo.go @@ -15,6 +15,7 @@ package gopsutil import ( + "math" "runtime" "strings" @@ -114,6 +115,13 @@ func (s *systemInfo) collectPlatformInfo() error { } s.cpuModel = infos[0].ModelName s.cpuMHz = computeMHz(s.cpuModel) + if s.cpuMHz < 0 { + s.cpuMHz = int(math.Round(infos[0].Mhz)) + } + if s.cpuMHz < 0 { + // This is supposed to be unsigned. + s.cpuMHz = 0 + } // gopsutil reports core and clock speed info inconsistently // by OS From 55ff03a2d678c834c229c2c2bdbc48d7c9927935 Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 5 Jun 2020 12:09:40 -0500 Subject: [PATCH 29/45] Lower scale of some random-value tests The random-value tests can be pathological, and in particular, the test of arbitrarily-spaced values is in effect O(N^2), and with race testing on, that test *alone* can take ten minutes to run, but it's not really all that exciting. We just reduce a bunch of values and/or test fewer things for these, which doesn't significantly alter coverage, but reduces test runtime on my laptop with `-race` from 21 minutes to a bit under 5. --- roaring/naive_test.go | 5 ++++- roaring/roaring_test.go | 18 +++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/roaring/naive_test.go b/roaring/naive_test.go index 154d0f8f5..c1accfb5e 100644 --- a/roaring/naive_test.go +++ b/roaring/naive_test.go @@ -15,6 +15,7 @@ package roaring import ( + "math/rand" "reflect" "testing" ) @@ -110,13 +111,15 @@ func TestUnionSlice(t *testing.T) { } func TestMaxInSlice(t *testing.T) { + // arbitrary, we just want to get the same values every time + r := rand.New(rand.NewSource(23)) a := []uint64{1, 4, 9, 5, 24, 13} v := maxInSlice(a) if uint64(24) != v { t.Fatalf("expected %v, but got %v", uint64(24), v) } - for i := uint64(1000); i <= uint64(100000); i++ { + for i := uint64(1000); i <= uint64(100000); i += uint64(r.Intn(35)) + 1 { a = append(a, i) if v = maxInSlice(a); v != i { t.Fatalf("expected %v, but got %v", i, v) diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 4f079102a..e31cbf2ee 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -866,7 +866,7 @@ func TestBitmap_UnionInPlaceProp(t *testing.T) { seed = time.Now().UnixNano() source = rand.NewSource(seed) rng = rand.New(source) - numTests = 100 + numTests = 20 maxNumIntsPerBatch = 100 maxNumBatches = 100 maxRangePercent = 2 @@ -1425,10 +1425,10 @@ func TestBitmap_Shift(t *testing.T) { } func TestBitmap_Quick_Array1(t *testing.T) { testBitmapQuick(t, 1000, 1000, 2000) } -func TestBitmap_Quick_Array2(t *testing.T) { testBitmapQuick(t, 10000, 0, 1000) } -func TestBitmap_Quick_Bitmap1(t *testing.T) { testBitmapQuick(t, 10000, 0, 10000) } -func TestBitmap_Quick_Bitmap2(t *testing.T) { testBitmapQuick(t, 10000, 10000, 20000) } -func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 10000, 0, math.MaxInt64) } +func TestBitmap_Quick_Array2(t *testing.T) { testBitmapQuick(t, 1000, 0, 1000) } +func TestBitmap_Quick_Bitmap1(t *testing.T) { testBitmapQuick(t, 1000, 0, 10000) } +func TestBitmap_Quick_Bitmap2(t *testing.T) { testBitmapQuick(t, 1000, 10000, 20000) } +func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 1000, 0, math.MaxInt64) } // Ensure a bitmap can perform basic operations on randomly generated values. func testBitmapQuick(t *testing.T, n int, min, max uint64) { @@ -1504,20 +1504,20 @@ func TestBitmap_Marshal_Quick_Array1(t *testing.T) { testBitmapMarshalQuick(t, 1000, 1000, 2000, false) } func TestBitmap_Marshal_Quick_Array2(t *testing.T) { - testBitmapMarshalQuick(t, 10000, 0, 1000, false) + testBitmapMarshalQuick(t, 1000, 0, 1000, false) } func TestBitmap_Marshal_Quick_Bitmap1(t *testing.T) { - testBitmapMarshalQuick(t, 10000, 0, 10000, false) + testBitmapMarshalQuick(t, 1000, 0, 10000, false) } func TestBitmap_Marshal_Quick_Bitmap2(t *testing.T) { - testBitmapMarshalQuick(t, 10000, 10000, 20000, false) + testBitmapMarshalQuick(t, 1000, 10000, 20000, false) } func TestBitmap_Marshal_Quick_LargeValue(t *testing.T) { testBitmapMarshalQuick(t, 100, 0, math.MaxInt64, false) } func TestBitmap_Marshal_Quick_Bitmap_Sorted(t *testing.T) { - testBitmapMarshalQuick(t, 10000, 0, 10000, true) + testBitmapMarshalQuick(t, 1000, 0, 10000, true) } // TODO update for RLE From 1460756b3f047cb82b0c19283bf20e5bbb6b9306 Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 5 Jun 2020 12:24:07 -0500 Subject: [PATCH 30/45] Provide option for adjusting node timeouts, set it for tests. There's no reason to have 10-20 seconds of delays for testing this, because in testing, we're running things on the local machine and don't need to worry about significant network lag. Make retry count and delay settable options, and set them lower. Moves the Replica2 test in server/server_test.go from ~21s to ~2s. --- cluster.go | 25 +++++++++++++++---------- cluster_internal_test.go | 21 ++++++++++++++++----- server.go | 18 ++++++++++++++++++ test/pilosa.go | 14 +++++++------- 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/cluster.go b/cluster.go index b3fc86e88..eacad25f7 100644 --- a/cluster.go +++ b/cluster.go @@ -62,9 +62,8 @@ const ( resizeJobActionAdd = "ADD" resizeJobActionRemove = "REMOVE" - confirmDownRetries = 10 - confirmDownSleep = 1 - confirmDownTimeout = 2 + defaultConfirmDownRetries = 10 + defaultConfirmDownSleep = 1 * time.Second ) // Node represents a node in the cluster. @@ -239,6 +238,9 @@ type cluster struct { // nolint: maligned logger logger.Logger InternalClient InternalClient + + confirmDownRetries int + confirmDownSleep time.Duration } // newCluster returns a new instance of Cluster with defaults. @@ -258,6 +260,9 @@ func newCluster() *cluster { InternalClient: newNopInternalClient(), logger: logger.NopLogger, + + confirmDownRetries: defaultConfirmDownRetries, + confirmDownSleep: defaultConfirmDownSleep, } } @@ -1923,7 +1928,7 @@ func (c *cluster) considerTopology() error { // band aid to protect against false nodeLeave events from memberlist // the test is the lightest weight endpoint of the node in question /version // TODO provide more robust solution to false nodeLeave events -func confirmNodeDown(uri URI, log logger.Logger) bool { +func (c *cluster) confirmNodeDown(uri URI) bool { u := url.URL{ Scheme: uri.Scheme, Host: uri.HostPort(), @@ -1931,11 +1936,11 @@ func confirmNodeDown(uri URI, log logger.Logger) bool { } req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - log.Printf("bad request:%s %s", u.String(), err) + c.logger.Printf("bad request:%s %s", u.String(), err) return false } - for i := 0; i < confirmDownRetries; i++ { - ctx, cancel := context.WithTimeout(context.Background(), confirmDownTimeout*time.Second) + for i := 0; i < c.confirmDownRetries; i++ { + ctx, cancel := context.WithTimeout(context.Background(), c.confirmDownSleep*2) defer cancel() resp, err := http.DefaultClient.Do(req.WithContext(ctx)) var bod []byte @@ -1946,8 +1951,8 @@ func confirmNodeDown(uri URI, log logger.Logger) bool { } } - log.Printf("NodeLeave confirm with %s %d. err: '%v' bod: '%s'", uri.HostPort(), i, err, bod) - time.Sleep(confirmDownSleep * time.Second) + c.logger.Printf("NodeLeave confirm with %s %d. err: '%v' bod: '%s'", uri.HostPort(), i, err, bod) + time.Sleep(c.confirmDownSleep) } return true } @@ -1975,7 +1980,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) { // not already removed by a removeNode request. We treat this as the // host being temporarily unavailable, and expect it to come back // up. - if confirmNodeDown(e.Node.URI, c.logger) { + if c.confirmNodeDown(e.Node.URI) { if c.removeNodeBasicSorted(e.Node.ID) { c.Topology.nodeStates[e.Node.ID] = nodeStateDown // put the cluster into STARTING if we've lost a number of nodes diff --git a/cluster_internal_test.go b/cluster_internal_test.go index de87ad5ae..b9e714326 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -943,18 +943,22 @@ func TestCluster_confirmNodeDownUp(t *testing.T) { t.Error(err) } uri.Port = uint16(iport) - if confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) { + c := newCluster() + c.logger = logger.NewVerboseLogger(os.Stdout) + if c.confirmNodeDown(uri) { t.Errorf("expected node to be up") } } func TestCluster_confirmNodeDownTimeout(t *testing.T) { + sleep := 50 * time.Millisecond + retries := 5 if testing.Short() { t.Skip() } r := mux.NewRouter() r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(confirmDownSleep * time.Second * confirmDownRetries) + time.Sleep(sleep * time.Duration(retries)) fmt.Fprintln(w, "ignored") })) server := httptest.NewServer(r) @@ -973,8 +977,11 @@ func TestCluster_confirmNodeDownTimeout(t *testing.T) { t.Error(err) } uri.Port = uint16(iport) - - if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) { + c := newCluster() + c.confirmDownSleep = sleep + c.confirmDownRetries = retries + c.logger = logger.NewVerboseLogger(os.Stdout) + if !c.confirmNodeDown(uri) { t.Errorf("expected node to be down") } } @@ -987,8 +994,12 @@ func TestCluster_confirmNodeDownDown(t *testing.T) { uri.Scheme = "http" uri.Host = "DoesntMatter" uri.Port = 6666 + c := newCluster() + c.confirmDownSleep = 50 * time.Millisecond + c.confirmDownRetries = 5 + c.logger = logger.NewVerboseLogger(os.Stdout) - if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) { + if !c.confirmNodeDown(uri) { t.Errorf("expected node to be down") } } diff --git a/server.go b/server.go index 9fbb30811..131a6971c 100644 --- a/server.go +++ b/server.go @@ -77,6 +77,8 @@ type Server struct { // nolint: maligned metricInterval time.Duration diagnosticInterval time.Duration maxWritesPerRequest int + confirmDownSleep time.Duration + confirmDownRetries int isCoordinator bool syncer holderSyncer @@ -229,6 +231,17 @@ func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { } } +// OptServerNodeDownRetries is a functional option on Server +// used to specify the retries and sleep duration for node down +// checks. +func OptServerNodeDownRetries(retries int, sleep time.Duration) ServerOption { + return func(s *Server) error { + s.confirmDownRetries = retries + s.confirmDownSleep = sleep + return nil + } +} + // OptServerURI is a functional option on Server // used to set the server URI. func OptServerURI(uri *URI) ServerOption { @@ -330,6 +343,9 @@ func NewServer(opts ...ServerOption) (*Server, error) { metricInterval: 0, diagnosticInterval: 0, + confirmDownRetries: defaultConfirmDownRetries, + confirmDownSleep: defaultConfirmDownSleep, + resetTranslationSyncCh: make(chan struct{}), logger: logger.NopLogger, @@ -399,6 +415,8 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.executor.MaxWritesPerRequest = s.maxWritesPerRequest s.cluster.broadcaster = s s.cluster.maxWritesPerRequest = s.maxWritesPerRequest + s.cluster.confirmDownRetries = s.confirmDownRetries + s.cluster.confirmDownSleep = s.confirmDownSleep s.holder.broadcaster = s err = s.loadAllExtensions() if err != nil { diff --git a/test/pilosa.go b/test/pilosa.go index 635e0f3d9..9b5f49a48 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -92,7 +92,7 @@ func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command { // We want tests to default to using the in-memory translate store, so we // prepend opts with that functional option. If a different translate store // has been specified, it will override this one. - opts = prependWithMemStore(opts) + opts = prependTestServerOpts(opts) m := newCommand(opts...) m.Config.Cluster.Disabled = false m.Config.Cluster.Coordinator = isCoordinator @@ -434,25 +434,25 @@ func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Clu return c } -// prependOpts applies prependWithMemStore to each of the ops (one per +// prependOpts applies prependTestServerOpts to each of the ops (one per // node, or one for the entire cluser). func prependOpts(opts [][]server.CommandOption) [][]server.CommandOption { if len(opts) == 0 { opts = [][]server.CommandOption{ - prependWithMemStore([]server.CommandOption{}), + prependTestServerOpts([]server.CommandOption{}), } } else { for i := range opts { - opts[i] = prependWithMemStore(opts[i]) + opts[i] = prependTestServerOpts(opts[i]) } } return opts } -// prependWithMemStore prepends opts with the OpenInMemTranslateStore. -func prependWithMemStore(opts []server.CommandOption) []server.CommandOption { +// prependTestServerOpts prepends opts with the OpenInMemTranslateStore. +func prependTestServerOpts(opts []server.CommandOption) []server.CommandOption { defaultOpts := []server.CommandOption{ - server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)), + server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore), pilosa.OptServerNodeDownRetries(5, 100*time.Millisecond)), } return append(defaultOpts, opts...) } From 1952a43ed46b76cbcf00dd4ca5270a332b2be28a Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 5 Jun 2020 12:29:32 -0500 Subject: [PATCH 31/45] Write fewer bits to test the rowcache behavior The failure mode in question was pretty predictable and tied to number of snapshots, not to number of bits written, so we can probably use a lot fewer bits and still get good results, but this is really slow under -race testing. --- fragment_internal_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index b8a60d86a..9230519e7 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -111,6 +111,11 @@ func TestFragment_ClearBit(t *testing.T) { func TestFragment_RowcacheMap(t *testing.T) { var done int64 f := mustOpenFragment("i", "f", viewStandard, 0, "") + // Under -race, this test turns out to take a fairly long time + // to run with larger OpN, because we write 50,000 bits to + // the bitmap, and everything is being race-detected, and we don't + // actually need that many to get the result we care about. + f.MaxOpN = 2000 defer f.Clean(t) ch := make(chan struct{}) From 1484674a1c668d33ab9a3bfb1cefc172bf3925e0 Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 5 Jun 2020 12:54:00 -0500 Subject: [PATCH 32/45] Add and use bitmap-to-slice-or-set comparison functions The generation of slices from things, and use of reflect.DeepEqual to compare the slices, is a lot more expensive than it needs to be. Omitting it removes most of the runtime of the marshal tests. --- roaring/roaring.go | 32 +++++++++++++++++++++++++++++++- roaring/roaring_test.go | 10 ++++------ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 22d2821db..9a636a519 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -5756,7 +5756,37 @@ func xorBitmapRun(a, b *Container) *Container { return output } -// CompareEquality is used mostly in test cases to confirm that two bitmaps came +// CompareBitmapSlice checks whether a bitmap has the same values in it +// that a provided slice does. +func CompareBitmapSlice(b *Bitmap, vals []uint64) (bool, error) { + count := b.Count() + if count != uint64(len(vals)) { + return false, fmt.Errorf("length mismatch: bitmap has %d bits, slice has %d", count, len(vals)) + } + for _, v := range vals { + if !b.Contains(v) { + return false, fmt.Errorf("bitmap lacks expected value %d", v) + } + } + return true, nil +} + +// CompareBitmapMap checks whether a bitmap has the same values in it +// that a provided map[uint64]struct{} has as keys. +func CompareBitmapMap(b *Bitmap, vals map[uint64]struct{}) (bool, error) { + count := b.Count() + if count != uint64(len(vals)) { + return false, fmt.Errorf("length mismatch: bitmap has %d bits, map has %d", count, len(vals)) + } + for v := range vals { + if !b.Contains(v) { + return false, fmt.Errorf("bitmap lacks expected value %d", v) + } + } + return true, nil +} + +// BitwiseEqual is used mostly in test cases to confirm that two bitmaps came // out the same. It does not expect corresponding opN, or OpWriter, but expects // identical bit contents. It does not expect identical representations; a bitmap // container can be identical to an array container. It returns a boolean value, diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index e31cbf2ee..c8eb2617a 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1570,14 +1570,12 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { t.Fatal(err) } - // Verify the original bitmap has the correct set of values. - if exp, got := generator.Uint64SetSlice(set), bm.Slice(); !reflect.DeepEqual(exp, got) { - t.Fatalf("mismatch: %s\n\nexp=%+v\n\ngot=%+v\n\n", diff(exp, got), exp, got) + if _, err := roaring.CompareBitmapMap(bm, set); err != nil { + t.Fatalf("source mismatch: %v", err) } - // Verify the bitmap loaded with the ops log has the correct set of values. - if exp, got := generator.Uint64SetSlice(set), bm2.Slice(); !reflect.DeepEqual(exp, got) { - t.Fatalf("mismatch: %s\n\nexp=%+v\n\ngot=%+v\n\n", diff(exp, got), exp, got) + if _, err := roaring.CompareBitmapMap(bm2, set); err != nil { + t.Fatalf("unmarshalled mismatch: %v", err) } } From e223c79acef6e891d771ad0c8bf82811ca708c3c Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 5 Jun 2020 13:07:16 -0500 Subject: [PATCH 33/45] Don't use a whole shard of values for Execute_All test. This is pretty expensive even for default shard width, and very expensive for ShardWidth = 1<<22, and we don't really get much extra benefit from having a million values instead of a hundred or so. --- executor_test.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/executor_test.go b/executor_test.go index dfb5d3af3..45389d2a0 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3476,15 +3476,15 @@ func TestExecutor_Execute_All(t *testing.T) { t.Fatal(err) } - // Create an import request that sets a full shard, - // plus a couple bits set on either side of it, and - // a final bit set in a fourth shard. + // Create an import request that sets things on either end + // of a shard, plus a couple bits set on either side of it, + // and a final bit set in a fourth shard. // // shard0 shard1 shard2 shard3 // |----------|----------|----------|----------| - // | **|**********|** | * + // | **|** **|** | * // - bitCount := ShardWidth + 5 + bitCount := 100 + 5 req := &pilosa.ImportRequest{ Index: index.Name(), Field: fld.Name(), @@ -3492,10 +3492,14 @@ func TestExecutor_Execute_All(t *testing.T) { RowIDs: make([]uint64, bitCount), ColumnIDs: make([]uint64, bitCount), } - for i := 0; i < bitCount-1; i++ { + for i := 0; i < bitCount/2; i++ { req.RowIDs[i] = 10 req.ColumnIDs[i] = uint64(i + ShardWidth - 2) } + for i := bitCount / 2; i < bitCount-1; i++ { + req.RowIDs[i] = 10 + req.ColumnIDs[i] = uint64(i + (ShardWidth * 2) - bitCount + 5) + } req.RowIDs[bitCount-1] = 10 req.ColumnIDs[bitCount-1] = uint64((3 * ShardWidth) + 2) @@ -3521,7 +3525,7 @@ func TestExecutor_Execute_All(t *testing.T) { {qry: fmt.Sprintf("All(limit=2, offset=%d)", bitCount-5), expCols: req.ColumnIDs[bitCount-5 : bitCount-3], expCnt: 2}, {qry: "All(limit=2, offset=2)", expCols: req.ColumnIDs[2:4], expCnt: 2}, {qry: "All(limit=1, offset=1)", expCols: req.ColumnIDs[1:2], expCnt: 1}, - {qry: fmt.Sprintf("All(limit=%d, offset=2)", ShardWidth), expCols: req.ColumnIDs[2 : bitCount-3], expCnt: ShardWidth}, + {qry: fmt.Sprintf("All(limit=%d, offset=2)", bitCount-3), expCols: req.ColumnIDs[2 : bitCount-1], expCnt: uint64(bitCount - 3)}, } for i, test := range tests { if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil { From 3f0c9925f4aa88a2b9e893c9027594f14267008e Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 5 Jun 2020 13:09:33 -0500 Subject: [PATCH 34/45] Don't test quite so many values for BtreeSeek and BtreeDelete BtreeSeek is O(N^2) on its N, and there's not a ton of extra utility to testing a larger range of values, so we reduce N by a bit, cutting runtime from ~10s to <1s on my laptop. Also reduce the scale of the BtreeDelete1/BtreeDelete2 tests a bit because, again, lots of runtime for little marginal information. --- roaring/btree_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roaring/btree_test.go b/roaring/btree_test.go index c3c02af3d..f0f528a7b 100644 --- a/roaring/btree_test.go +++ b/roaring/btree_test.go @@ -689,7 +689,7 @@ func TestBtreeDelete0(t *testing.T) { } func TestBtreeDelete1(t *testing.T) { - const N = 130000 + const N = 13000 for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x333333, 0xcccccc, 0x314159} { r := treeNew() set := r.Set @@ -788,7 +788,7 @@ func benchmarkDelRnd(b *testing.B, n int) { } func TestBtreeDelete2(t *testing.T) { - const N = 100000 + const N = 10000 for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x333333, 0xcccccc, 0x314159} { r := treeNew() set := r.Set @@ -1468,7 +1468,7 @@ func TestBtreePut(t *testing.T) { } func TestBtreeSeek(t *testing.T) { - const N = 1 << 13 + const N = 1 << 11 tr := treeNew() for i := 0; i < N; i++ { k := 2*i + 1 From 52aa3e2e23338ca184970190c0341805f9acebe8 Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 5 Jun 2020 13:38:33 -0500 Subject: [PATCH 35/45] Improve container/bitmap comparison logic for testing We have a "deadcode" bitmapsEqual which is actually used in testing but probably shouldn't be, and we don't have a good container equality test. Problem is, equality tests are sort of slow in the things-are-equal case, which is the most common case, so we've got some moderately-specialized code here; specifically, special comparison code that takes advantage of knowing that if two containers have the same number of bits, you only have to check whether all the bits from one are present in the other, because that can't be true for differing containers with the same number of bits. This reduces the runtime for the ContainerCombinations case from about 24 seconds to a bit under 2 on my laptop, or from around 10 minutes to about 37 seconds with the race detector on. Also simplify the InPlaceWrapper functions not to invoke bitmaps, because it's not really necessary. --- roaring/roaring.go | 154 +++++++++++++++++-------------- roaring/roaring_helpers_test.go | 89 ++++++++++-------- roaring/roaring_internal_test.go | 103 ++++++++------------- roaring/roaring_test.go | 12 --- 4 files changed, 169 insertions(+), 189 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 9a636a519..db63b0edd 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -4552,50 +4552,95 @@ func (c *Container) bitmapZeroRange(i, j uint64) { c.setN(n) } -// equals reports whether two containers are equal. -func (c *Container) equals(c2 *Container) bool { - if c == nil || c2 == nil { - if c != c2 { - return false +func typePair(ct1, ct2 byte) int { + return int((ct1 << 4) | ct2) +} + +// compareArrayBitmap actually only verifies that everything in the array +// is in the bitmap. It's used only after comparing the N for the containers, +// so if there's anything in the bitmap that's not in the array, either there's +// something in the array that's not in the bitmap, or we didn't get here. +func compareArrayBitmap(a []uint16, b []uint64) error { + for _, v := range a { + w, bit := b[v>>6], v&63 + if w>>bit&1 == 0 { + return fmt.Errorf("value %d missing", v) } } - if c.Mapped() != c2.Mapped() || c.typ() != c2.typ() || c.N() != c2.N() { - return false + return nil +} + +// compareArrayRuns determines whether an array matches a provided +// set of runs. As with compareArrayBitmap, it only verifies presence +// of the array's values in the run collection. the run collection +// can't be empty; if it were, N would have been 0, and we wouldn't +// have gotten here. +func compareArrayRuns(a []uint16, r []interval16) error { + ri := 0 + ru := r[ri] + ri++ + for _, v := range a { + if v < ru.start { + return fmt.Errorf("value %d missing", v) + } + if v > ru.last { + if ri >= len(r) { + return fmt.Errorf("value %d missing", v) + } + ru = r[ri] + ri++ + // if they're identical, the array value must be + // the start of the next run. + if v != ru.start { + return fmt.Errorf("value %d missing", v) + } + } } - if c.typ() == containerArray { - ca, c2a := c.array(), c2.array() - if len(ca) != len(c2a) { - return false - } - for i := 0; i < len(ca); i++ { - if ca[i] != c2a[i] { - return false - } - } - } else if c.typ() == containerBitmap { - cb, c2b := c.bitmap(), c2.bitmap() - if len(cb) != len(c2b) { - return false - } - for i := 0; i < len(cb); i++ { - if cb[i] != c2b[i] { - return false - } - } - } else if c.typ() == containerRun { - cr, c2r := c.runs(), c2.runs() - if len(cr) != len(c2r) { - return false - } - for i := 0; i < len(cr); i++ { - if cr[i] != c2r[i] { - return false - } - } - } else { - panic(fmt.Sprintf("unknown container type: %v", c.typ())) + return nil +} + +// compareArrayArray reports whether everything in a1 is equal to everything +// in a2. +func compareArrayArray(a1, a2 []uint16) error { + if len(a1) != len(a2) { + return fmt.Errorf("unexpected length mismatch, %d vs %d", len(a1), len(a2)) } - return true + for i := range a1 { + if a1[i] != a2[i] { + return fmt.Errorf("item %d: %d vs %d", i, a1[i], a2[i]) + } + } + return nil +} + +// BitwiseCompare reports whether two containers are equal. It returns +// an error describing any difference it finds. This is mostly intended +// for use in tests that expect equality. +func (c *Container) BitwiseCompare(c2 *Container) error { + if c.N() != c2.N() { + return errors.New("containers are different lengths") + } + if c.N() == 0 { + return nil + } + switch typePair(c.typ(), c2.typ()) { + case typePair(containerArray, containerArray): + return compareArrayArray(c.array(), c2.array()) + case typePair(containerArray, containerBitmap): + return compareArrayBitmap(c.array(), c2.bitmap()) + case typePair(containerBitmap, containerArray): + return compareArrayBitmap(c2.array(), c.bitmap()) + case typePair(containerArray, containerRun): + return compareArrayRuns(c.array(), c2.runs()) + case typePair(containerRun, containerArray): + return compareArrayRuns(c2.array(), c.runs()) + default: + c3 := xor(c, c2) + if c3.N() != 0 { + return fmt.Errorf("%d bits differenct between containers", c3.N()) + } + } + return nil } func unionArrayBitmap(a, b *Container) *Container { @@ -5855,35 +5900,6 @@ func (b *Bitmap) BitwiseEqual(c *Bitmap) (bool, error) { return true, nil } -func bitmapsEqual(b, c *Bitmap) error { // nolint: deadcode - statsHit("bitmapsEqual") - if b.OpWriter != c.OpWriter { - return errors.New("opWriters not equal") - } - if b.opN != c.opN { - return errors.New("opNs not equal") - } - - biter, _ := b.Containers.Iterator(0) - citer, _ := c.Containers.Iterator(0) - bn, cn := biter.Next(), citer.Next() - for ; bn && cn; bn, cn = biter.Next(), citer.Next() { - bk, bc := biter.Value() - ck, cc := citer.Value() - if bk != ck { - return errors.New("keys not equal") - } - if !bc.equals(cc) { - return errors.New("containers not equal") - } - } - if bn && !cn || cn && !bn { - return errors.New("different numbers of containers") - } - - return nil -} - func popcount(x uint64) uint64 { return uint64(bits.OnesCount64(x)) } diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index 41d4059fb..f8de764fb 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -14,6 +14,8 @@ package roaring +import "sync" + /////////////////////////////////////////////////////////////////////////// var containerWidth uint64 = 65536 @@ -261,51 +263,56 @@ func doContainer(typ byte, data interface{}) *Container { return nil } +var makeCts sync.Once +var sampleTestContainers map[byte]map[string]*Container + func setupContainerTests() map[byte]map[string]*Container { - cts := make(map[byte]map[string]*Container) + makeCts.Do(func() { + sampleTestContainers = make(map[byte]map[string]*Container) - // array containers - cts[containerArray] = map[string]*Container{ - "empty": doContainer(containerArray, arrayEmpty()), - "full": doContainer(containerArray, arrayFull()), - "firstBitSet": doContainer(containerArray, arrayFirstBitSet()), - "lastBitSet": doContainer(containerArray, arrayLastBitSet()), - "firstBitUnset": doContainer(containerArray, arrayFirstBitUnset()), - "lastBitUnset": doContainer(containerArray, arrayLastBitUnset()), - "innerBitsSet": doContainer(containerArray, arrayInnerBitsSet()), - "outerBitsSet": doContainer(containerArray, arrayOuterBitsSet()), - "oddBitsSet": doContainer(containerArray, arrayOddBitsSet()), - "evenBitsSet": doContainer(containerArray, arrayEvenBitsSet()), - } + // array containers + sampleTestContainers[containerArray] = map[string]*Container{ + "empty": doContainer(containerArray, arrayEmpty()), + "full": doContainer(containerArray, arrayFull()), + "firstBitSet": doContainer(containerArray, arrayFirstBitSet()), + "lastBitSet": doContainer(containerArray, arrayLastBitSet()), + "firstBitUnset": doContainer(containerArray, arrayFirstBitUnset()), + "lastBitUnset": doContainer(containerArray, arrayLastBitUnset()), + "innerBitsSet": doContainer(containerArray, arrayInnerBitsSet()), + "outerBitsSet": doContainer(containerArray, arrayOuterBitsSet()), + "oddBitsSet": doContainer(containerArray, arrayOddBitsSet()), + "evenBitsSet": doContainer(containerArray, arrayEvenBitsSet()), + } - // bitmap containers - cts[containerBitmap] = map[string]*Container{ - "empty": doContainer(containerBitmap, bitmapEmpty()), - "full": doContainer(containerBitmap, bitmapFull()), - "firstBitSet": doContainer(containerBitmap, bitmapFirstBitSet()), - "lastBitSet": doContainer(containerBitmap, bitmapLastBitSet()), - "firstBitUnset": doContainer(containerBitmap, bitmapFirstBitUnset()), - "lastBitUnset": doContainer(containerBitmap, bitmapLastBitUnset()), - "innerBitsSet": doContainer(containerBitmap, bitmapInnerBitsSet()), - "outerBitsSet": doContainer(containerBitmap, bitmapOuterBitsSet()), - "oddBitsSet": doContainer(containerBitmap, bitmapOddBitsSet()), - "evenBitsSet": doContainer(containerBitmap, bitmapEvenBitsSet()), - } + // bitmap containers + sampleTestContainers[containerBitmap] = map[string]*Container{ + "empty": doContainer(containerBitmap, bitmapEmpty()), + "full": doContainer(containerBitmap, bitmapFull()), + "firstBitSet": doContainer(containerBitmap, bitmapFirstBitSet()), + "lastBitSet": doContainer(containerBitmap, bitmapLastBitSet()), + "firstBitUnset": doContainer(containerBitmap, bitmapFirstBitUnset()), + "lastBitUnset": doContainer(containerBitmap, bitmapLastBitUnset()), + "innerBitsSet": doContainer(containerBitmap, bitmapInnerBitsSet()), + "outerBitsSet": doContainer(containerBitmap, bitmapOuterBitsSet()), + "oddBitsSet": doContainer(containerBitmap, bitmapOddBitsSet()), + "evenBitsSet": doContainer(containerBitmap, bitmapEvenBitsSet()), + } - // run containers - cts[containerRun] = map[string]*Container{ - "empty": doContainer(containerRun, runEmpty()), - "full": doContainer(containerRun, runFull()), - "firstBitSet": doContainer(containerRun, runFirstBitSet()), - "lastBitSet": doContainer(containerRun, runLastBitSet()), - "firstBitUnset": doContainer(containerRun, runFirstBitUnset()), - "lastBitUnset": doContainer(containerRun, runLastBitUnset()), - "innerBitsSet": doContainer(containerRun, runInnerBitsSet()), - "outerBitsSet": doContainer(containerRun, runOuterBitsSet()), - "oddBitsSet": doContainer(containerRun, runOddBitsSet()), - "evenBitsSet": doContainer(containerRun, runEvenBitsSet()), - } + // run containers + sampleTestContainers[containerRun] = map[string]*Container{ + "empty": doContainer(containerRun, runEmpty()), + "full": doContainer(containerRun, runFull()), + "firstBitSet": doContainer(containerRun, runFirstBitSet()), + "lastBitSet": doContainer(containerRun, runLastBitSet()), + "firstBitUnset": doContainer(containerRun, runFirstBitUnset()), + "lastBitUnset": doContainer(containerRun, runLastBitUnset()), + "innerBitsSet": doContainer(containerRun, runInnerBitsSet()), + "outerBitsSet": doContainer(containerRun, runOuterBitsSet()), + "oddBitsSet": doContainer(containerRun, runOddBitsSet()), + "evenBitsSet": doContainer(containerRun, runEvenBitsSet()), + } + }) - return cts + return sampleTestContainers } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 7784064ab..3a0749cc8 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -2632,7 +2632,7 @@ func TestBitmapClone(t *testing.T) { } } c := b.Clone() - if err := bitmapsEqual(b, c); err != nil { + if _, err := b.BitwiseEqual(c); err != nil { t.Fatalf("Clone Objects not equal: %v\n", err) } d := func() *Bitmap { //anybody know how to declare a nil value? @@ -2735,34 +2735,46 @@ func getFunctionName(i interface{}) string { return y[0] } -// UnionInPlace is defined at the Bitmap level, but this wrapper lets us insert -// it into our ContainerCombinations tests so that it gets exercised on a wide -// variety of container data. func unionInPlaceWrapper(a, b *Container) *Container { - out := NewBitmap() - out.Containers.Put(0, a.Clone()) - B := NewBitmap() - B.Containers.Put(0, b) - out.UnionInPlace(B) - return out.Containers.Get(0) + ret := a.Clone().unionInPlace(b) + ret.Repair() + return ret } func differenceInPlaceWrapper(a, b *Container) *Container { - out := NewBitmap() - out.Containers.Put(0, a.Clone()) - B := NewBitmap() - B.Containers.Put(0, b) - out.DifferenceInPlace(B) - return out.Containers.Get(0) + a = a.Clone() + // this should probably return its new value, but currently does not + a.differenceInPlace(b) + return a } func intersectInPlaceWrapper(a, b *Container) *Container { - out := NewBitmap() - out.Containers.Put(0, a.Clone()) - B := NewBitmap() - B.Containers.Put(0, b) - out.IntersectInPlace(B) - return out.Containers.Get(0) + return a.Clone().intersectInPlace(b) +} + +func TestContainerBitwiseCompare(t *testing.T) { + cts := setupContainerTests() + + for t1, containers := range cts { + for name, c := range containers { + for t2, other := range cts { + for otherName, otherC := range other { + err := c.BitwiseCompare(otherC) + if err != nil { + if otherName == name { + t.Fatalf("container types %d/%d, contents %s: unexpected error %v", + t1, t2, name, err) + } + } else { + if name != otherName { + t.Fatalf("container types %d/%d, unexpected %s == %s", + t1, t2, name, otherName) + } + } + } + } + } + } } func TestContainerCombinations(t *testing.T) { @@ -3562,51 +3574,8 @@ func TestContainerCombinations(t *testing.T) { // Convert to all container types and check result. for _, ct := range containerTypes { - clone := ret.Clone() - if ct == containerArray { - if clone == nil { - clone = NewContainerArray(nil) - } else if clone.isBitmap() { - clone = clone.bitmapToArray() - } else if clone.isRun() { - clone = clone.runToArray() - } - if clone.N() != cts[ct][exp].N() { - t.Errorf("test %s expected array n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N()) - } - // Because xorRunRun resulting in an empty container returns an array container with a - // nil slice array, then we need to check len() on array first (look for 0). - if !(len(clone.array()) == 0 && len(cts[ct][exp].array()) == 0) && !reflect.DeepEqual(clone.array(), cts[ct][exp].array()) { - t.Errorf("test %s expected array %X, but got %X", desc, cts[ct][exp].array(), clone.array()) - } - } else if ct == containerBitmap { - if clone == nil { - clone = NewContainerBitmap(0, nil) - } else if clone.isArray() { - clone = clone.arrayToBitmap() - } else if clone.isRun() { - clone = clone.runToBitmap() - } - if clone.N() != cts[ct][exp].N() { - t.Errorf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N()) - } - if !reflect.DeepEqual(clone.bitmap(), cts[ct][exp].bitmap()) { - t.Errorf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap(), clone.bitmap()) - } - } else if ct == containerRun { - if clone == nil { - clone = NewContainerRun(nil) - } else if clone.isArray() { - clone = clone.arrayToRun(0) - } else if clone.isBitmap() { - clone = clone.bitmapToRun(0) - } - if clone.N() != cts[ct][exp].N() { - t.Errorf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N()) - } - if !reflect.DeepEqual(clone.runs(), cts[ct][exp].runs()) { - t.Errorf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs(), clone.runs()) - } + if err := ret.BitwiseCompare(cts[ct][exp]); err != nil { + t.Errorf("test %s: %v", desc, err) } } } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index c8eb2617a..55395ea73 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1837,18 +1837,6 @@ func getBenchData(tb testing.TB) *benchmarkSampleData { return data } -func diff(a, b []uint64) string { - if len(a) != len(b) { - return fmt.Sprintf("len: %d != %d", len(a), len(b)) - } - for i := range a { - if a[i] != b[i] { - return fmt.Sprintf("index %d: %d != %d", i, a[i], b[i]) - } - } - return "" -} - func TestBitmap_Intersect(t *testing.T) { bm0 := testBM() result := bm0.Intersect(bm0) From 44569fa21090a08b3ca692bdc8c48d71d45e19c2 Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 5 Jun 2020 13:54:17 -0500 Subject: [PATCH 36/45] Reduce iterations in TestFragment_RowsIteration We don't really learn more from trying every multiple of 10,000 than we do from trying maybe 32 values, and it's worse at larger shard widths. --- fragment_internal_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 9230519e7..2a554bfc1 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -2760,9 +2760,9 @@ func TestFragment_RowsIteration(t *testing.T) { defer f.Clean(t) expectedRows := make([]uint64, 0) - for r := uint64(1); r < uint64(10000); r += 100 { + for r := uint64(1); r < uint64(10000); r += 250 { expectedRows = append(expectedRows, r) - for c := uint64(1); c < uint64(ShardWidth-1); c += 10000 { + for c := uint64(1); c < uint64(ShardWidth-1); c += (ShardWidth >> 5) { if _, err := f.setBit(r, c); err != nil { t.Fatal(err) } From 120cc02536659b87ccfec7957a69cde043b2e1ff Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Mon, 8 Jun 2020 14:45:56 -0400 Subject: [PATCH 37/45] process BSI ops more efficiently --- fragment.go | 48 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/fragment.go b/fragment.go index 3cc4ca9ba..cc2dfcc20 100644 --- a/fragment.go +++ b/fragment.go @@ -1262,31 +1262,41 @@ func msb(x uint64) uint { // rangeLTUnsigned returns all bits LT/LTE the predicate without considering the sign bit. func (f *fragment) rangeLTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - if msb(predicate) > bitDepth || (allowEquality && predicate == (1< bitDepth: + fallthrough + case predicate == (1<= 0; i-- { + for i := int(bitDepth - 1); i >= 0 && predicate > 0 && remaining.Any(); i-- { row := f.row(uint64(bsiOffsetBit + i)) zeroes := remaining.Difference(row) switch (predicate >> uint(i)) & 1 { case 1: // Match everything with a zero bit here. matched = matched.Union(zeroes) + predicate &^= 1 << uint(i) case 0: // Discard everything with a one bit here. remaining = zeroes } } - if allowEquality { - matched = matched.Union(remaining) - } - return matched, nil } @@ -1329,15 +1339,27 @@ func (f *fragment) rangeGT(bitDepth uint, predicate int64, allowEquality bool) ( } func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - if allowEquality && predicate == 0 { + switch { + case predicate == 0 && allowEquality: // This query matches all possible values. return filter, nil + case predicate == 0 && !allowEquality: + // This query matches everything that is not 0. + remaining := filter + for i := uint(0); i < bitDepth && remaining.Any(); i++ { + row := f.row(uint64(bsiOffsetBit + i)) + remaining = remaining.Difference(row) + } + return remaining, nil + case allowEquality: + predicate-- } // Compare intermediate bits. matched := NewRow() remaining := filter - for i := int(bitDepth - 1); i >= 0; i-- { + predicate |= (^uint64(0)) << bitDepth + for i := int(bitDepth - 1); i >= 0 && predicate < ^uint64(0) && remaining.Any(); i-- { row := f.row(uint64(bsiOffsetBit + i)) ones := remaining.Intersect(row) switch (predicate >> uint(i)) & 1 { @@ -1347,13 +1369,10 @@ func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, case 0: // Match everything with a one bit here. matched = matched.Union(ones) + predicate |= 1 << uint(i) } } - if allowEquality { - matched = matched.Union(remaining) - } - return matched, nil } @@ -1370,6 +1389,8 @@ func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax int64) upredicateMin, upredicateMax := absInt64(predicateMin), absInt64(predicateMax) switch { + case predicateMin == predicateMax: + return f.rangeEQ(bitDepth, predicateMin) case predicateMin >= 0: // Handle positive-only values. return f.rangeBetweenUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMin, upredicateMax) @@ -1393,9 +1414,6 @@ func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax int64) // rangeBetweenUnsigned returns BSI columns for a range of values. Disregards the sign bit. func (f *fragment) rangeBetweenUnsigned(filter *Row, bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { switch { - case predicateMax-predicateMin < 2: - // This query matches all possible values. - return filter, nil case predicateMax > (1< Date: Mon, 8 Jun 2020 14:32:23 -0500 Subject: [PATCH 38/45] Use xlarge executor in CircleCI --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9904cff3e..4503b8801 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,6 +8,7 @@ executors: default: "1.14" docker: - image: circleci/golang:<< parameters.version >> + resource_class: xlarge working_directory: /go/src/github.com/pilosa/pilosa environment: GO111MODULE: "on" # TODO: Only needed for Go <1.13, remove when dropping support for 1.11/1.12. From 8352f5d273e037c83e6f18d7e706b4ed4e568e96 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 8 Jun 2020 14:59:59 -0500 Subject: [PATCH 39/45] Add configurable resource class, enable only for test-race. --- .circleci/config.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4503b8801..f2e70731e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,9 +6,12 @@ executors: version: type: string default: "1.14" + resource_class: + type: string + default: medium docker: - image: circleci/golang:<< parameters.version >> - resource_class: xlarge + resource_class: << parameters.resource_class >> working_directory: /go/src/github.com/pilosa/pilosa environment: GO111MODULE: "on" # TODO: Only needed for Go <1.13, remove when dropping support for 1.11/1.12. @@ -65,6 +68,9 @@ jobs: - run: make build GOOS=linux GOARCH=arm64 test: parameters: + resource_class: + type: string + default: medium golang_version: type: string default: "1.14" @@ -83,6 +89,7 @@ jobs: executor: name: golang version: << parameters.golang_version >> + resource_class: << parameters.resource_class >> steps: - attach_workspace: at: . @@ -192,6 +199,7 @@ workflows: - test: name: test-race test_make_target: test-race + resource_class: xlarge requires: - setup - test: From 9c3b080bf0919d9834b653cba3f346ae87207558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Mon, 8 Jun 2020 13:53:05 +0200 Subject: [PATCH 40/45] Check result before return --- executor.go | 10 +++++++++- executor_test.go | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/executor.go b/executor.go index 390f85654..79a876023 100644 --- a/executor.go +++ b/executor.go @@ -3107,7 +3107,15 @@ func (e *executor) executeSetRow(ctx context.Context, indexName string, c *pql.C } result, err := e.mapReduce(ctx, indexName, shards, c, opt, mapFn, reduceFn) - return result.(bool), err + if err != nil { + return false, err + } + + b, ok := result.(bool) + if !ok { + return false, errors.New("unsupported result type") + } + return b, nil } // executeSetRowShard executes a SetRow() call for a single shard. diff --git a/executor_test.go b/executor_test.go index 45389d2a0..5241d3f7e 100644 --- a/executor_test.go +++ b/executor_test.go @@ -4018,6 +4018,25 @@ func TestExecutor_Execute_SetRow(t *testing.T) { t.Fatalf("unexpected columns: %+v", bits) } }) + t.Run("Err_Store(Distinct)", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + f1, err := index.CreateField("f1", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + f2, err := index.CreateField("f2", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + q := fmt.Sprintf(`Store(Distinct(field=%s), %s=2)`, f1.Name(), f2.Name()) + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: index.Name(), Query: q}); err == nil { + t.Fatalf("expected 'unsupported result type' error, got: %+v", res) + } + }) } func benchmarkExistence(nn bool, b *testing.B) { From d478dd9d94e6a89b5c4bc29bd48eb8c168eedb74 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Wed, 10 Jun 2020 09:23:39 -0400 Subject: [PATCH 41/45] fix BSI comparison match-all-but-one operation --- fragment.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fragment.go b/fragment.go index cc2dfcc20..5f26828ec 100644 --- a/fragment.go +++ b/fragment.go @@ -1270,12 +1270,12 @@ func (f *fragment) rangeLTUnsigned(filter *Row, bitDepth uint, predicate uint64, return filter, nil case predicate == (1< Date: Wed, 10 Jun 2020 10:09:33 -0400 Subject: [PATCH 42/45] add regression test for BSI match-all-but-one operations --- fragment_internal_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 2a554bfc1..5912eb132 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -624,6 +624,23 @@ func TestFragment_Range(t *testing.T) { } }) + t.Run("LTMaxRegression", func(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Clean(t) + + if _, err := f.setValue(1, 2, 3); err != nil { + t.Fatal(err) + } else if _, err := f.setValue(2, 2, 0); err != nil { + t.Fatal(err) + } + + if b, err := f.rangeLTUnsigned(NewRow(1, 2), 2, 3, false); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Columns(), []uint64{2}) { + t.Fatalf("unepxected coulmns: %+v", b.Columns()) + } + }) + t.Run("GT", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) @@ -672,6 +689,23 @@ func TestFragment_Range(t *testing.T) { } }) + t.Run("GTMinRegression", func(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Clean(t) + + if _, err := f.setValue(1, 2, 0); err != nil { + t.Fatal(err) + } else if _, err := f.setValue(2, 2, 1); err != nil { + t.Fatal(err) + } + + if b, err := f.rangeGTUnsigned(NewRow(1, 2), 2, 0, false); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Columns(), []uint64{2}) { + t.Fatalf("unepxected coulmns: %+v", b.Columns()) + } + }) + t.Run("BETWEEN", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) From 416f332070b29dd3cc48fd73ab8e92e35e8a22c1 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 10 Jun 2020 13:12:38 -0500 Subject: [PATCH 43/45] Optimize row.Includes --- row.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/row.go b/row.go index 8d2c121d2..eea6871f0 100644 --- a/row.go +++ b/row.go @@ -510,14 +510,10 @@ func (r *Row) Columns() []uint64 { // Includes returns true if the row contains the given column. func (r *Row) Includes(col uint64) bool { - // TODO: improve the efficiency of this method by - // performing the column filter at the bitmap level - // rather than iterating through the results here. + shard := col / ShardWidth for i := range r.segments { - for _, c := range r.segments[i].Columns() { - if c == col { - return true - } + if r.segments[i].shard == shard { + return r.segments[i].data.Contains(col) } } return false From 535257af7531e411c8351a38dc5ad69705a8fce7 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Wed, 10 Jun 2020 16:16:53 -0400 Subject: [PATCH 44/45] apply base in GroupBy --- executor.go | 29 +++++++++++++++++++++++++++-- executor_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 79a876023..02991553b 100644 --- a/executor.go +++ b/executor.go @@ -1770,10 +1770,16 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call return nil, err } + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + // perform necessary Rows queries (any that have limit or columns args) - // TODO, call async? would only help if multiple Rows queries had a column // or limit arg. // TODO support TopN in here would be really cool - and pretty easy I think. + bases := make(map[int]int64) childRows := make([]RowIDs, len(c.Children)) for i, child := range c.Children { // Check "field" first for backwards compatibility, then set _field. @@ -1793,6 +1799,18 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call if err != nil { return nil, errors.Wrap(err, "getting column") } + if _, ok := child.Args["_field"].(string); !ok { + return nil, errors.Errorf("%s call must have field with valid (string) field name. Got %v of type %[2]T", child.Name, child.Args["_field"]) + } + f := idx.Field(child.Args["_field"].(string)) + if f == nil { + return nil, ErrFieldNotFound + } + switch f.Type() { + case FieldTypeInt: + bases[i] = f.bsiGroup(f.name).Base + } + if hasLimit || hasCol { // we need to perform this query cluster-wide ahead of executeGroupByShard childRows[i], err = e.executeRows(ctx, index, child, shards, opt) if err != nil { @@ -1806,7 +1824,7 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeGroupByShard(ctx, index, c, filter, shard, childRows) + return e.executeGroupByShard(ctx, index, c, filter, shard, childRows, bases) } // Merge returned results at coordinating node. reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { @@ -2157,7 +2175,7 @@ func applyConditionToGroupCounts(gcs []GroupCount, subj string, cond *pql.Condit return gcs[:i] } -func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs) (_ []GroupCount, err error) { +func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs, bases map[int]int64) (_ []GroupCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGroupByShard") defer span.Finish() @@ -2205,6 +2223,13 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql } } + // Apply bases. + for i, base := range bases { + for _, r := range results { + *r.Group[i].Value += base + } + } + return results, nil } diff --git a/executor_test.go b/executor_test.go index 5241d3f7e..bf146ab50 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2996,6 +2996,37 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { test.CheckGroupBy(t, expected, results) } }) + + t.Run("groupbBy on ints with offset regression", func(t *testing.T) { + _, err = c[0].API.CreateField(context.Background(), "i", "hint", pilosa.OptFieldTypeInt(1, 1000)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, hint=1) + Set(1, hint=2) + Set(2, hint=3) + `}); err != nil { + t.Fatalf("querying remote: %v", err) + } + + if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `GroupBy(Rows(hint))`, + }); err != nil { + t.Fatalf("GroupBy querying: %v", err) + } else { + var a, b, c int64 = 1, 2, 3 + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "hint", Value: &a}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "hint", Value: &b}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "hint", Value: &c}}, Count: 1}, + } + + results := res.Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, expected, results) + } + }) } // Ensure executor returns an error if too many writes are in a single request. From 32e47642aeebb13804529d77e0320bc507d68aa9 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Wed, 10 Jun 2020 17:49:00 -0400 Subject: [PATCH 45/45] address review of "Apply base in GroupBy on BSI" --- executor.go | 5 +++-- executor_test.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/executor.go b/executor.go index 02991553b..f2dd49fcb 100644 --- a/executor.go +++ b/executor.go @@ -1799,10 +1799,11 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call if err != nil { return nil, errors.Wrap(err, "getting column") } - if _, ok := child.Args["_field"].(string); !ok { + fieldName, ok := child.Args["_field"].(string) + if !ok { return nil, errors.Errorf("%s call must have field with valid (string) field name. Got %v of type %[2]T", child.Name, child.Args["_field"]) } - f := idx.Field(child.Args["_field"].(string)) + f := idx.Field(fieldName) if f == nil { return nil, ErrFieldNotFound } diff --git a/executor_test.go b/executor_test.go index bf146ab50..6bf3f2a14 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2997,7 +2997,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { } }) - t.Run("groupbBy on ints with offset regression", func(t *testing.T) { + t.Run("groupBy on ints with offset regression", func(t *testing.T) { _, err = c[0].API.CreateField(context.Background(), "i", "hint", pilosa.OptFieldTypeInt(1, 1000)) if err != nil { t.Fatalf("creating field: %v", err)