From b8432b6400357f46610a2ee03d28aa66fe06649c Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 1 Sep 2021 14:27:01 -0500 Subject: [PATCH 1/6] backport fix broken intersectionCallback functions --- roaring/roaring.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 5504b9bf1..32d573911 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -4448,10 +4448,13 @@ func intersectionCallbackArrayArray(a, b *Container, fn func(uint16)) { if (na << 2) < nb { for _, va := range ca { for cb[0] < va { - if len(cb) > 8 && cb[0] < va { + // try to skip ahead a bit faster + for len(cb) > 7 && cb[7] < va { cb = cb[8:] } - cb = cb[1:] + for len(cb) > 0 && cb[0] < va { + cb = cb[1:] + } if len(cb) == 0 { return } @@ -4540,7 +4543,7 @@ func intersectionCallbackBitmapRun(a, b *Container, fn func(uint16)) { } } -func intersectionCallbackArrayBitmap(a, b *Container, fn func(uint16)) (n int32) { +func intersectionCallbackArrayBitmap(a, b *Container, fn func(uint16)) { statsHit("intersectionCount/ArrayBitmap") bitmap := b.bitmap() ln := len(bitmap) @@ -4550,9 +4553,10 @@ func intersectionCallbackArrayBitmap(a, b *Container, fn func(uint16)) (n int32) break } off := val % 64 - n += int32(bitmap[i]>>off) & 1 + if (bitmap[i]>>off)&1 != 0 { + fn(val) + } } - return n } func intersectionCallbackBitmapBitmap(a, b *Container, fn func(uint16)) { From dfea30e18a19300835699fd76e68efcd6f139c10 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 1 Sep 2021 15:51:01 -0500 Subject: [PATCH 2/6] backup mutex sanity check (1681) --- api.go | 134 ++++++++++++++++++++++++++++++++ client.go | 6 ++ field.go | 17 ++++ field_internal_test.go | 25 ++++++ fragment.go | 11 +++ http/client.go | 28 +++++++ http/handler.go | 54 +++++++++++++ roaring/filter.go | 69 ++++++++++++++++ roaring/filter_internal_test.go | 47 +++++++++++ view.go | 44 +++++++++++ 10 files changed, 435 insertions(+) diff --git a/api.go b/api.go index d2c09fe85..4fca1a776 100644 --- a/api.go +++ b/api.go @@ -2188,6 +2188,138 @@ func (api *API) TranslateFieldDB(ctx context.Context, indexName, fieldName strin _, err := store.ReadFrom(rd) return err } +func (api *API) mutexCheckThisNode(ctx context.Context, qcx *Qcx, indexName string, fieldName string) (map[uint64]map[uint64][]uint64, error) { + index := api.holder.Index(indexName) + if index == nil { + return nil, newNotFoundError(ErrIndexNotFound, indexName) + } + field := index.Field(fieldName) + if field == nil { + return nil, newNotFoundError(ErrFieldNotFound, fieldName) + } + return field.MutexCheck(ctx, qcx) +} + +// mergeMutexCollisions adds collisions to an existing map if they aren't already +// present. It modifies dst. +func mergeMutexCollisions(dst, src map[uint64][]uint64) { + for k, v := range src { + if len(v) == 0 { + continue + } + existing := dst[k] + if len(existing) == 0 { + dst[k] = v + continue + } + // existing and v are both non-empty lists of collisions for this key. + // but if replication is working, both nodes should have the SAME list + // of collisions, so it's probably worth special-casing that check: + if len(existing) == len(v) { + different := false + for i := range existing { + if v[i] != existing[i] { + different = true + break + } + } + // yay, we can just ignore this + if !different { + continue + } + } + // combine... + existing = append(existing, v...) + // sort... + sort.Slice(existing, func(i, j int) bool { + return existing[i] < existing[j] + }) + // dedup. + n := 0 + prev := existing[0] + for i := 0; i < len(existing); i++ { + if existing[i] != prev { + existing[n] = existing[i] + n++ + } + prev = existing[i] + } + dst[k] = existing[:n] + } +} + +// MutexCheck checks for collisions in a given mutex field. The response is +// a map[shard]map[column]values. +func (api *API) MutexCheck(ctx context.Context, qcx *Qcx, indexName string, fieldName string, remote bool) (map[uint64]map[uint64][]uint64, error) { + if err := api.validate(apiMutexCheck); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + // short path: if this is an internal remote request, only try to solve the + // question for these shards. + if remote { + out, err := api.mutexCheckThisNode(ctx, qcx, indexName, fieldName) + return out, err + } + var nodes []*Node + if !remote { + nodes = Nodes(api.cluster.nodes).Clone() + } else { + nodes = []*Node{api.cluster.nodeByID(api.server.nodeID)} + } + + /* + // request data from other nodes as well + snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN) + */ + eg, _ := errgroup.WithContext(ctx) + myID := api.server.nodeID + results := make([]map[uint64]map[uint64][]uint64, len(nodes)) + for i, node := range nodes { + i := i // loop variable shadowing is a war crime + if node.ID != myID { + node := node // loop variable shadowing again + eg.Go(func() (err error) { + results[i], err = api.server.defaultClient.MutexCheck(ctx, &node.URI, indexName, fieldName) + return err + }) + } else { + eg.Go(func() (err error) { + results[i], err = api.mutexCheckThisNode(ctx, qcx, indexName, fieldName) + return err + }) + } + } + err := eg.Wait() + if err != nil { + return nil, err + } + var out map[uint64]map[uint64][]uint64 + for _, nodeResults := range results { + if len(nodeResults) == 0 { + continue + } + if out == nil { + out = nodeResults + continue + } + for k, v := range nodeResults { + if len(v) == 0 { + continue + } + existing := out[k] + // results from two nodes. might be normal with replication. + if len(existing) == 0 { + out[k] = v + continue + } + // we have two maps for this shard. whee. + mergeMutexCollisions(existing, v) + // we don't store it back into out[k] because it was already + // a non-empty map, so stores to it update it. yay? + } + } + return out, nil +} type serverInfo struct { ShardWidth uint64 `json:"shardWidth"` @@ -2249,11 +2381,13 @@ const ( apiIDReserve apiIDCommit apiIDReset + apiMutexCheck //Backported from 1681 Caution ) var methodsCommon = map[apiMethod]struct{}{ apiClusterMessage: {}, apiSetCoordinator: {}, + apiMutexCheck: {}, } var methodsResizing = map[apiMethod]struct{}{ diff --git a/client.go b/client.go index 4cd410345..d04791e60 100644 --- a/client.go +++ b/client.go @@ -84,6 +84,8 @@ type InternalClient interface { GetNodeUsage(ctx context.Context, uri *URI) (map[string]NodeUsage, error) GetPastQueries(ctx context.Context, uri *URI) ([]PastQueryStatus, error) + + MutexCheck(ctx context.Context, uri *URI, index string, field string) (map[uint64]map[uint64][]uint64, error) } //=============== @@ -251,3 +253,7 @@ func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *URI) (map[stri func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *URI) ([]PastQueryStatus, error) { return nil, nil } + +func (n nopInternalClient) MutexCheck(ctx context.Context, uri *URI, index, field string) (map[uint64]map[uint64][]uint64, error) { + return nil, nil +} diff --git a/field.go b/field.go index baec3a931..827e4484b 100644 --- a/field.go +++ b/field.go @@ -1229,6 +1229,23 @@ func (f *Field) Row(tx Tx, rowID uint64) (*Row, error) { } } +// mutexCheck performs a sanity-check on the available fragments for a +// field. The return is map[column]map[shard][]values for collisions only. +func (f *Field) MutexCheck(ctx context.Context, qcx *Qcx) (map[uint64]map[uint64][]uint64, error) { + if f.Type() != FieldTypeMutex { + return nil, errors.New("mutex check only valid for mutex fields") + } + f.mu.RLock() + defer f.mu.RUnlock() + standard := f.viewMap[viewStandard] + if standard == nil { + // no standard view present means we've never needed to create it, + // so it has no bits set, so it has no extra bits set. + return nil, nil + } + return standard.mutexCheck(ctx, qcx) +} + // SetBit sets a bit on a view within the field. func (f *Field) SetBit(tx Tx, rowID, colID uint64, t *time.Time) (changed bool, err error) { viewName := viewStandard diff --git a/field_internal_test.go b/field_internal_test.go index d187d5ab9..1a9dad798 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -27,6 +27,7 @@ import ( "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/shardwidth" "github.com/pilosa/pilosa/v2/testhook" ) @@ -922,3 +923,27 @@ func TestBSIGroup_TxReopenDB(t *testing.T) { // the test: can we re-open a BSI fragment under Tx store _ = f.Reopen() } + +func CorruptAMutex(tb testing.TB, field *Field, qcx *Qcx) { + v := field.view(viewStandard) + if v == nil { + tb.Fatalf("creating view failed") + } + frags := v.allFragments() + for _, frag := range frags { + func() { + tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: field.idx, Shard: frag.shard}) + defer finisher(&err) + if err != nil { + tb.Fatalf("getting tx: %v", err) + } + // set a bonus bit, bypassing the mutex handling + frag.mu.Lock() + _, err = frag.unprotectedSetBit(tx, 3, frag.shard< 0 { + return key.NeedData() + } + return key.RejectOne() +} + +func (b *BitmapMutexDupFilter) ConsiderData(key FilterKey, data *Container) FilterResult { + value, basePos := uint64(key)>>rowExponent, uint64(key&keyMask)<<16 + containerCallback(data, func(u uint16) { + pos := basePos + uint64(u) + if b.first[pos] != ^uint64(0) { + b.extra[pos+b.base] = append(b.extra[pos+b.base], value) + } else { + b.first[pos] = value + } + }) + return key.MatchOne() +} + +// Report returns the set of duplicate values identified. +func (b *BitmapMutexDupFilter) Report() map[uint64][]uint64 { + // copy values into extra, and remove them from first, so calling + // Report() again won't cause double-appends. + for k, v := range b.extra { + kpos := k % (1 << shardwidth.Exponent) + if b.first[kpos] != ^uint64(0) { + v = append(v, 0) + // prepend so the lowest value goes at the beginning + copy(v[1:], v[:]) + v[0] = b.first[kpos] + b.first[kpos] = ^uint64(0) + b.extra[k] = v + } + } + return b.extra +} + // ApplyFilterToIterator is a simplistic implementation that applies a bitmap // filter to a ContainerIterator, returning an error if it encounters an error. // diff --git a/roaring/filter_internal_test.go b/roaring/filter_internal_test.go index 0fc9c0fe0..1811acff4 100644 --- a/roaring/filter_internal_test.go +++ b/roaring/filter_internal_test.go @@ -349,3 +349,50 @@ func TestFilterWithRows(t *testing.T) { } } + +func TestMutexDupFilter(t *testing.T) { + tests := []struct{ + pairs [][2]uint64 + expect map[uint64][]uint64 + }{ + { + pairs: [][2]uint64{{0, 0}, {1, 0}, {0, 1}}, + expect: map[uint64][]uint64{0: {0, 1}}, + }, + { + pairs: [][2]uint64{{0, 0}, {1, 0}, {0, 1}, {0, 2}}, + expect: map[uint64][]uint64{0: {0, 1, 2}}, + }, + } + for num, test := range tests { + t.Run(fmt.Sprintf("case%d", num), func(t *testing.T) { + b := NewSliceBitmap() + for _, p := range test.pairs { + v := (p[1] << shardwidth.Exponent) | p[0] + b.DirectAdd(v) + } + dup := NewBitmapMutexDupFilter(0) + iter, _ := b.Containers.Iterator(0) + err := ApplyFilterToIterator(dup, iter) + if err != nil { + t.Fatalf("applying filter: %v", err) + } + expected := test.expect + got := dup.Report() + if len(expected) != len(got) { + t.Fatalf("expected %d entries in duplicate map, got %d", len(expected), len(got)) + } + for k, v := range expected { + gv := got[k] + if len(v) != len(gv) { + t.Fatalf("for id %d, expected %d (len %d), got %d (len %d)", k, v, len(v), gv, len(gv)) + } + for j := range v { + if gv[j] != v[j] { + t.Fatalf("for id %d, expected %d, got %d", k, v[j], gv[j]) + } + } + } + }) + } +} diff --git a/view.go b/view.go index 15173f5e4..0af36b71a 100644 --- a/view.go +++ b/view.go @@ -447,6 +447,50 @@ func (v *view) row(txOrig Tx, rowID uint64) (*Row, error) { } +// mutexCheck checks all available fragments for duplicate values. The return +// is map[column]map[shard][]values for collisions only. +func (v *view) mutexCheck(ctx context.Context, qcx *Qcx) (map[uint64]map[uint64][]uint64, error) { + // We don't need the context, we just want the context-awareness on the error groups. + // It would be nice if the inner functions could use this too... + eg, _ := errgroup.WithContext(ctx) + throttle := make(chan struct{}, runtime.NumCPU()) + frags := v.allFragments() + results := make([]map[uint64][]uint64, len(frags)) + for i, frag := range frags { + // local copies for the goroutine to use + i, frag := i, frag + eg.Go(func() error { + // limit simultaneous parallel goroutines associated with this + throttle <- struct{}{} + defer func() { + <-throttle + }() + tx, finisher, err := qcx.GetTx(Txo{Index: v.idx, Shard: frag.shard}) + if err != nil { + return err + } + defer finisher(&err) + results[i], err = frag.mutexCheck(tx) + if err != nil { + return err + } + return nil + }) + } + err := eg.Wait() + if err != nil { + return nil, err + } + out := map[uint64]map[uint64][]uint64{} + for i, result := range results { + if len(result) == 0 { + continue + } + out[frags[i].shard] = result + } + return out, nil +} + // setBit sets a bit within the view. func (v *view) setBit(txOrig Tx, rowID, columnID uint64) (changed bool, err error) { shard := columnID / ShardWidth From 32ad1db2b210b8ec71766629528c5762b1a18931 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 2 Sep 2021 16:37:52 -0500 Subject: [PATCH 3/6] cleaup api and add keytranslation --- api.go | 305 +++++++++++++++++++++++++++------------ api_test.go | 315 +++++++++++++++++++++++++++++++++++++++++ field_internal_test.go | 2 +- go.mod | 1 + go.sum | 2 + http/handler.go | 4 +- 6 files changed, 536 insertions(+), 93 deletions(-) diff --git a/api.go b/api.go index 4fca1a776..a0074371c 100644 --- a/api.go +++ b/api.go @@ -2179,15 +2179,6 @@ func (api *API) TranslateIndexDB(ctx context.Context, indexName string, partitio _, err := store.ReadFrom(rd) return err } - -// TranslateFieldDB is an internal function to load the field keys database -func (api *API) TranslateFieldDB(ctx context.Context, indexName, fieldName string, rd io.Reader) error { - idx := api.holder.Index(indexName) - field := idx.Field(fieldName) - store := field.TranslateStore() - _, err := store.ReadFrom(rd) - return err -} func (api *API) mutexCheckThisNode(ctx context.Context, qcx *Qcx, indexName string, fieldName string) (map[uint64]map[uint64][]uint64, error) { index := api.holder.Index(indexName) if index == nil { @@ -2200,82 +2191,86 @@ func (api *API) mutexCheckThisNode(ctx context.Context, qcx *Qcx, indexName stri return field.MutexCheck(ctx, qcx) } -// mergeMutexCollisions adds collisions to an existing map if they aren't already -// present. It modifies dst. -func mergeMutexCollisions(dst, src map[uint64][]uint64) { - for k, v := range src { - if len(v) == 0 { - continue +// mergeIDLists merges a list of numeric IDs into another list, removing +// duplicates. +func mergeIDLists(dst []uint64, src []uint64) []uint64 { + dst = append(dst, src...) + sort.Slice(dst, func(i, j int) bool { + return dst[i] < dst[j] + }) + // dedup. + n := 0 + prev := dst[0] + for i := 0; i < len(dst); i++ { + if dst[i] != prev { + dst[n] = dst[i] + n++ } - existing := dst[k] - if len(existing) == 0 { - dst[k] = v - continue - } - // existing and v are both non-empty lists of collisions for this key. - // but if replication is working, both nodes should have the SAME list - // of collisions, so it's probably worth special-casing that check: - if len(existing) == len(v) { - different := false - for i := range existing { - if v[i] != existing[i] { - different = true - break - } - } - // yay, we can just ignore this - if !different { - continue - } - } - // combine... - existing = append(existing, v...) - // sort... - sort.Slice(existing, func(i, j int) bool { - return existing[i] < existing[j] - }) - // dedup. - n := 0 - prev := existing[0] - for i := 0; i < len(existing); i++ { - if existing[i] != prev { - existing[n] = existing[i] - n++ - } - prev = existing[i] - } - dst[k] = existing[:n] + prev = dst[i] } + return dst } -// MutexCheck checks for collisions in a given mutex field. The response is -// a map[shard]map[column]values. -func (api *API) MutexCheck(ctx context.Context, qcx *Qcx, indexName string, fieldName string, remote bool) (map[uint64]map[uint64][]uint64, error) { +// mergeKeyLists merges a list of string IDs into another list, removing +// duplicates. +func mergeKeyLists(dst []string, src []string) []string { + dst = append(dst, src...) + sort.Slice(dst, func(i, j int) bool { + return dst[i] < dst[j] + }) + // dedup. + n := 0 + prev := dst[0] + for i := 0; i < len(dst); i++ { + if dst[i] != prev { + dst[n] = dst[i] + n++ + } + prev = dst[i] + } + return dst +} + +// MutexCheckNode checks for collisions in a given mutex field. The response is +// a map[shard]map[column]values, not translated. +func (api *API) MutexCheckNode(ctx context.Context, qcx *Qcx, indexName string, fieldName string) (map[uint64]map[uint64][]uint64, error) { if err := api.validate(apiMutexCheck); err != nil { return nil, errors.Wrap(err, "validating api method") } - // short path: if this is an internal remote request, only try to solve the - // question for these shards. - if remote { - out, err := api.mutexCheckThisNode(ctx, qcx, indexName, fieldName) - return out, err - } - var nodes []*Node - if !remote { - nodes = Nodes(api.cluster.nodes).Clone() - } else { - nodes = []*Node{api.cluster.nodeByID(api.server.nodeID)} - } + return api.mutexCheckThisNode(ctx, qcx, indexName, fieldName) +} - /* - // request data from other nodes as well - snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN) - */ +// MutexCheck checks a named field for mutex violations, returning a +// map of record IDs to values for records that have multiple values in the +// field. The return will be one of: +// map[uint64][]uint64 // unkeyed index, unkeyed field +// map[uint64][]string // unkeyed index, keyed field +// map[string][]uint64 // keyed index, unkeyed field +// map[string][]string // keyed index, keyed field +func (api *API) MutexCheck(ctx context.Context, qcx *Qcx, indexName string, fieldName string) (result interface{}, err error) { + if err = api.validate(apiMutexCheck); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + index, err := api.Index(ctx, indexName) + if err != nil { + return nil, err + } + field, err := api.Field(ctx, indexName, fieldName) + if err != nil { + return nil, err + } + if field.Type() != FieldTypeMutex { + return nil, errors.New("can only check mutex state for mutex fields") + } + nodes := Nodes(api.cluster.nodes).Clone() eg, _ := errgroup.WithContext(ctx) - myID := api.server.nodeID + results := make([]map[uint64]map[uint64][]uint64, len(nodes)) + myID := api.Node().ID + // vprint.VV("MyID %#v\n", myID) for i, node := range nodes { i := i // loop variable shadowing is a war crime + // vprint.VV("Compare %#v with %v", node.ID, myID) if node.ID != myID { node := node // loop variable shadowing again eg.Go(func() (err error) { @@ -2289,36 +2284,166 @@ func (api *API) MutexCheck(ctx context.Context, qcx *Qcx, indexName string, fiel }) } } - err := eg.Wait() + err = eg.Wait() if err != nil { return nil, err } - var out map[uint64]map[uint64][]uint64 + // We now have a series of maps from shards to maps of record IDs to + // values. But wait! Either the field, or the index, might be using keys, + // and want those translated. So we have to translate those. + useIndexKeys := index.Keys() + useFieldKeys := field.Keys() + var indexKeys = map[uint64]string{} + var fieldKeys = map[uint64]string{} + var indexIDs []uint64 + var fieldIDs []uint64 + // build translation tables, if we need them + untranslated := "untranslated" + // We don't know which of four map types we want to be working with, + // but what we can do is make a function which works with that map type + // given the raw integer values, and is a closure with an already-created + // map which has already been stashed in `result`. Because maps are + // reference-y, this should actually work. + var process func(uint64, []uint64) + if useIndexKeys || useFieldKeys { + for _, nodeResults := range results { + for _, shardResults := range nodeResults { + for record, values := range shardResults { + if useIndexKeys { + if _, ok := indexKeys[record]; !ok { + indexKeys[record] = untranslated + indexIDs = append(indexIDs, record) + } + } + if useFieldKeys { + for _, value := range values { + if _, ok := fieldKeys[value]; !ok { + fieldKeys[value] = untranslated + fieldIDs = append(fieldIDs, value) + } + } + } + } + } + } + // we now have lists of the keys, so... + if useIndexKeys { + indexKeyList, err := api.cluster.translateIndexIDs(ctx, indexName, indexIDs) + if err != nil { + return nil, errors.Wrap(err, "translating index keys") + } + if len(indexKeyList) != len(indexIDs) { + return nil, fmt.Errorf("translating %d record IDs, got %d keys", len(indexIDs), len(indexKeyList)) + } + for i := range indexIDs { + indexKeys[indexIDs[i]] = indexKeyList[i] + } + } + if useFieldKeys { + fieldKeyList, err := api.cluster.translateFieldListIDs(field, fieldIDs) + if err != nil { + return nil, errors.Wrap(err, "translating index keys") + } + if len(fieldKeyList) != len(fieldIDs) { + return nil, fmt.Errorf("translating %d IDs, got %d keys", len(indexIDs), len(fieldKeyList)) + } + for i := range fieldIDs { + fieldKeys[fieldIDs[i]] = fieldKeyList[i] + } + } + + } + // define the process functions. separated from above code just to make + // it easier to follow/compare them. + if useIndexKeys { + if useFieldKeys { + outMap := make(map[string][]string) + var valueKeys []string + result = outMap + process = func(recordID uint64, valueIDs []uint64) { + valueKeys = valueKeys[:0] + for _, id := range valueIDs { + valueKeys = append(valueKeys, fieldKeys[id]) + } + record := indexKeys[recordID] + if existing, ok := outMap[record]; ok { + outMap[record] = mergeKeyLists(existing, valueKeys) + } else { + // The append is so we can reuse this buffer safely, + // which matters if there's replication, because many + // cases won't need to copy the buffer, they'll just + // copy individual things from it. + outMap[record] = append([]string{}, valueKeys...) + } + } + } else { + outMap := make(map[string][]uint64) + result = outMap + process = func(recordID uint64, values []uint64) { + record := indexKeys[recordID] + if existing, ok := outMap[record]; ok { + outMap[record] = mergeIDLists(existing, values) + } else { + outMap[record] = values + } + } + } + } else { + if useFieldKeys { + outMap := make(map[uint64][]string) + var valueKeys []string + result = outMap + process = func(record uint64, valueIDs []uint64) { + valueKeys = valueKeys[:0] + for _, id := range valueIDs { + valueKeys = append(valueKeys, fieldKeys[id]) + } + if existing, ok := outMap[record]; ok { + outMap[record] = mergeKeyLists(existing, valueKeys) + } else { + // The append is so we can reuse this buffer safely, + // which matters if there's replication, because many + // cases won't need to copy the buffer, they'll just + // copy individual things from it. + outMap[record] = append([]string{}, valueKeys...) + } + } + } else { + outMap := make(map[uint64][]uint64) + result = outMap + process = func(record uint64, values []uint64) { + if existing, ok := outMap[record]; ok { + outMap[record] = mergeIDLists(existing, values) + } else { + outMap[record] = values + } + } + } + } + for _, nodeResults := range results { if len(nodeResults) == 0 { continue } - if out == nil { - out = nodeResults - continue - } - for k, v := range nodeResults { + for _, v := range nodeResults { if len(v) == 0 { continue } - existing := out[k] - // results from two nodes. might be normal with replication. - if len(existing) == 0 { - out[k] = v - continue + for record, values := range v { + process(record, values) } - // we have two maps for this shard. whee. - mergeMutexCollisions(existing, v) - // we don't store it back into out[k] because it was already - // a non-empty map, so stores to it update it. yay? } } - return out, nil + return result, nil +} + +// TranslateFieldDB is an internal function to load the field keys database +func (api *API) TranslateFieldDB(ctx context.Context, indexName, fieldName string, rd io.Reader) error { + idx := api.holder.Index(indexName) + field := idx.Field(fieldName) + store := field.TranslateStore() + _, err := store.ReadFrom(rd) + return err } type serverInfo struct { diff --git a/api_test.go b/api_test.go index 3f6369476..bec6c8b1c 100644 --- a/api_test.go +++ b/api_test.go @@ -28,6 +28,7 @@ import ( "github.com/pilosa/pilosa/v2/boltdb" "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/server" + "github.com/pilosa/pilosa/v2/shardwidth" "github.com/pilosa/pilosa/v2/test" ) @@ -622,3 +623,317 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, 0)) } } + +type mutexCheckIndex struct { + index *pilosa.Index + indexName string + createdAt int64 + fields map[bool]mutexCheckField +} + +type mutexCheckField struct { + fieldName string + field *pilosa.Field + createdAt int64 +} + +func TestAPI_MutexCheck(t *testing.T) { + c := test.MustRunCluster(t, 3) + defer c.Close() + + m0 := c.GetNode(0) + nodesByID := make(map[string]*test.Command, 3) + qcxsByID := make(map[string]*pilosa.Qcx, 3) + for i := 0; i < 3; i++ { + node := c.GetNode(i) + id := node.API.Node().ID + nodesByID[id] = node + } + + indexes := make(map[bool]mutexCheckIndex) + + ctx := context.Background() + for _, keyedIndex := range []bool{false, true} { + indexName := fmt.Sprintf("i%t", keyedIndex) + index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: keyedIndex, TrackExistence: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + if index.CreatedAt() == 0 { + t.Fatal("index createdAt is empty") + } + indexData := mutexCheckIndex{indexName: indexName, index: index, fields: make(map[bool]mutexCheckField), createdAt: index.CreatedAt()} + for _, keyedField := range []bool{false, true} { + fieldName := fmt.Sprintf("f%t", keyedField) + var field *pilosa.Field + if keyedField { + field, err = m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeMutex(pilosa.CacheTypeNone, 0), pilosa.OptFieldKeys()) + } else { + field, err = m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeMutex(pilosa.CacheTypeNone, 0)) + } + if err != nil { + t.Fatalf("creating field: %v", err) + } + if field.CreatedAt() == 0 { + t.Fatal("field createdAt is empty") + } + indexData.fields[keyedField] = mutexCheckField{fieldName: fieldName, field: field, createdAt: field.CreatedAt()} + } + indexes[keyedIndex] = indexData + } + + rowIDs := []uint64{0, 1, 2, 3} + colIDs := []uint64{0, 1, 2, 3} + rowKeysBase := []string{"v0", "v1", "v2", "v3"} + colKeysBase := []string{"c0", "c1", "c2", "c3"} + + const nShards = 10 + + // now, try the same thing for each combination of keyed/unkeyed. we + // share code between keyed/unkeyed fields, but for indexes, the logic + // is fundamentally different because we can't know shards in advance. + indexData := indexes[false] + for keyedField, fieldData := range indexData.fields { + fmt.Println("running:", indexData.indexName, fieldData.fieldName) + t.Run(fmt.Sprintf("%s-%s", indexData.indexName, fieldData.fieldName), func(t *testing.T) { + for id, node := range nodesByID { + qcxsByID[id] = node.API.Txf().NewQcx() + } + for shard := uint64(0); shard < nShards; shard++ { + // restore row/col ID values which can get altered by imports + for i := range rowIDs { + rowIDs[i] = uint64(i) + colIDs[i] = (shard << shardwidth.Exponent) + uint64(i) + (shard % 4) + } + req := &pilosa.ImportRequest{ + Index: indexData.indexName, + IndexCreatedAt: indexData.createdAt, + Field: fieldData.fieldName, + FieldCreatedAt: fieldData.createdAt, + Shard: shard, + ColumnIDs: colIDs, + } + if keyedField { + req.RowKeys = rowKeysBase + } else { + req.RowIDs = rowIDs + } + nodesForShard, err := m0.API.ShardNodes(ctx, indexData.indexName, shard) + if err != nil { + t.Fatalf("obtaining shard list: %v", err) + } + if len(nodesForShard) < 1 { + t.Fatalf("no nodes for shard %d", shard) + } + node := nodesByID[nodesForShard[0].ID] + if err := node.API.Import(ctx, qcxsByID[nodesForShard[0].ID], req); err != nil { + t.Fatalf("importing data: %v", err) + } + } + // and then we break the mutex and close the Qcxs + for id, node := range nodesByID { + field, err := node.API.Field(ctx, indexData.indexName, fieldData.fieldName) + if err != nil { + t.Fatalf("requesting field %s from node %s: %v", fieldData.fieldName, id, err) + } + pilosa.CorruptAMutex(t, field, qcxsByID[id]) + err = qcxsByID[id].Finish() + if err != nil { + t.Fatalf("closing out transaction on node %s: %v", id, err) + } + } + qcx := m0.API.Txf().NewQcx() + defer qcx.Abort() + + results, err := m0.API.MutexCheck(ctx, qcx, indexData.indexName, fieldData.fieldName) + if err != nil { + t.Fatalf("checking mutexes: %v", err) + } + // first two shards of each group of 4 should have a collision in + // position 1 + expected := map[uint64]bool{ + (0 << shardwidth.Exponent) + 1: true, + (1 << shardwidth.Exponent) + 1: true, + (4 << shardwidth.Exponent) + 1: true, + (5 << shardwidth.Exponent) + 1: true, + (8 << shardwidth.Exponent) + 1: true, + (9 << shardwidth.Exponent) + 1: true, + } + if keyedField { + mapped, ok := results.(map[uint64][]string) + if !ok { + t.Fatalf("expected map[uint64][]string, got %T", results) + } + seen := 0 + for k, v := range mapped { + seen++ + if !expected[k] { + t.Fatalf("expected all collisions to be 1 shards (s %% 4 in [0,1]), got %d", k) + } + if len(v) != 2 { + t.Fatalf("expected exactly two collisions") + } + } + if seen != len(expected) { + t.Fatalf("expected exactly %d records to have collisions", len(expected)) + } + } else { + mapped, ok := results.(map[uint64][]uint64) + if !ok { + t.Fatalf("expected map[uint64][]uint64, got %T", results) + } + seen := 0 + for k, v := range mapped { + seen++ + if !expected[k] { + t.Fatalf("expected all collisions to be 1 shards (s %% 4 in [0,1]), got %d", k) + } + if len(v) != 2 { + t.Fatalf("expected exactly two collisions") + } + } + if seen != len(expected) { + t.Fatalf("expected exactly %d records to have collisions", len(expected)) + } + } + }) + } + indexData = indexes[true] + for keyedField, fieldData := range indexData.fields { + t.Run(fmt.Sprintf("%s-%s", indexData.indexName, fieldData.fieldName), func(t *testing.T) { + for id, node := range nodesByID { + qcxsByID[id] = node.API.Txf().NewQcx() + } + req := &pilosa.ImportRequest{ + Index: indexData.indexName, + IndexCreatedAt: indexData.createdAt, + Field: fieldData.fieldName, + FieldCreatedAt: fieldData.createdAt, + Shard: 0, // ignored when using keys + } + rowKeys := make([]string, 0, len(rowKeysBase)*nShards) + colKeys := make([]string, 0, len(rowKeysBase)*nShards) + rowIDs = rowIDs[:0] + for shard := uint64(0); shard < nShards; shard++ { + for i := range rowKeysBase { + colKeys = append(colKeys, fmt.Sprintf("s%d-%s", shard, colKeysBase[i])) + if keyedField { + rowKeys = append(rowKeys, rowKeysBase[i]) + } else { + rowIDs = append(rowIDs, uint64(i)) + } + } + } + req.ColumnKeys = colKeys + if keyedField { + req.RowKeys = rowKeys + } else { + req.RowIDs = rowIDs + } + var id string + var node *test.Command + for id, node = range nodesByID { + break + } + if err := node.API.Import(ctx, qcxsByID[id], req); err != nil { + t.Fatalf("importing data: %v", err) + } + expected, err := node.API.FindIndexKeys(ctx, indexData.indexName, colKeys...) + if err != nil { + t.Fatalf("looking up index keys: %v", err) + } + for key, id := range expected { + // CorruptAMutex should only corrupt things in position 1 of their + // shards... + if id%(1< Date: Thu, 2 Sep 2021 16:47:39 -0500 Subject: [PATCH 4/6] go mod tidy --- go.mod | 3 +-- go.sum | 16 ---------------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 4f4ed4c05..60e333795 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 - github.com/glycerine/vprint v0.0.0-20200730000117-76cea49a68ea // indirect github.com/gogo/protobuf v1.2.1 github.com/golang/protobuf v1.4.2 github.com/google/go-cmp v0.5.2 @@ -56,7 +55,7 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/grpc v1.28.0 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect - gopkg.in/yaml.v2 v2.3.0 // indirect + gopkg.in/yaml.v2 v2.3.0 modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible diff --git a/go.sum b/go.sum index 828f9786c..87df86f36 100644 --- a/go.sum +++ b/go.sum @@ -65,7 +65,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -74,8 +73,6 @@ github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E= github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y= -github.com/glycerine/vprint v0.0.0-20200730000117-76cea49a68ea h1:Uiuhuh77mImdrAMjPfw2V8tWw4AF6r9dxbNkECo23SA= -github.com/glycerine/vprint v0.0.0-20200730000117-76cea49a68ea/go.mod h1:q7RHAiHHxYrXtGEkX14OuACg+cHODdKnVvTgpBnOzHk= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -97,7 +94,6 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -112,7 +108,6 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -149,12 +144,10 @@ github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -174,15 +167,12 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -198,7 +188,6 @@ github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -293,11 +282,9 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -403,7 +390,6 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214095126-aec9a390925b h1:tv7/y4pd+sR8bcNb2D6o7BNU6zjWm0VjQLac+w7fNNM= @@ -455,7 +441,6 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= @@ -485,7 +470,6 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From a279dc2b9fb02ef690d1eddc110d0e2a702a1dc3 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 2 Sep 2021 16:58:21 -0500 Subject: [PATCH 5/6] go mod tidy +1 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 60e333795..f0d161c95 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/grpc v1.28.0 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect - gopkg.in/yaml.v2 v2.3.0 + gopkg.in/yaml.v2 v2.3.0 // indirect modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible From 333ff4f4f4e8f4ad0285e65ac82b45572c79b68c Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 2 Sep 2021 17:08:38 -0500 Subject: [PATCH 6/6] rename pilosa to featurebase --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ef6b7ce80..ce623b054 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -67,7 +67,7 @@ jobs: name: golang steps: - run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request - - run: curl https://moleculacorp:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/pilosa/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e + - run: curl https://moleculacorp:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/featurebase/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e test-build-arm: executor: name: golang