From adf3e528f56ed6da5a6b29b644b5c8c0d82adb81 Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 21 Oct 2021 13:09:24 -0500 Subject: [PATCH 01/40] Change time estimation to use avg time per message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In [SUP-75](https://molecula.atlassian.net/browse/SUP-75?atlOrigin=eyJpIjoiYmU5MzdkMmUyZTAyNGQ2Y2IzMDMzYTgzMDU2Y2ZhNmMiLCJwIjoiaiJ9) Allen pointed out that the time estimation is really good for the first couple lines of output, but gets exponentially worse as execution continues. After looking into it, it looks like we’re currently using a heuristic based on the amount of messages processed in the previous second(ish) which is what results in that sort of exponential drop off. To remedy this, I adjusted the time estimation calculation to use the average time per message up to the point of calculating the new estimate to ideally improve estimates over time, with the trade-off of a potentially less accurate estimate to begin with. --- server.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server.go b/server.go index fdca4dc39..ce6babe25 100644 --- a/server.go +++ b/server.go @@ -722,8 +722,10 @@ func (s *Server) Open() error { if now := time.Now(); now.Sub(prevMsg) > time.Second { progressRatio := float64(i+1) / float64(len(toSend)) - remainingRatio := 1 - progressRatio - timeRemaining := time.Duration(float64(now.Sub(prevMsg)) * (remainingRatio / progressRatio)) + numSentMessages := len(toSend) - (i + 1) + messagesLeft := len(toSend) - numSentMessages + avgTimePerMessage := float64(now.Sub(start)) / float64(numSentMessages) + timeRemaining := time.Duration(avgTimePerMessage * float64(messagesLeft)) s.logger.Printf("synced %d/%d messages (%.2f%% complete; %s remaining)", i+1, len(toSend), 100*progressRatio, timeRemaining) prevMsg = now } From 27515a3f98d5518f5d85188fce7995fdc9986617 Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 21 Oct 2021 14:24:27 -0500 Subject: [PATCH 02/40] number of sent messages is just i silly --- server.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server.go b/server.go index ce6babe25..8ee1ba5bc 100644 --- a/server.go +++ b/server.go @@ -722,9 +722,8 @@ func (s *Server) Open() error { if now := time.Now(); now.Sub(prevMsg) > time.Second { progressRatio := float64(i+1) / float64(len(toSend)) - numSentMessages := len(toSend) - (i + 1) - messagesLeft := len(toSend) - numSentMessages - avgTimePerMessage := float64(now.Sub(start)) / float64(numSentMessages) + messagesLeft := len(toSend) - i + avgTimePerMessage := float64(now.Sub(start)) / float64(i) timeRemaining := time.Duration(avgTimePerMessage * float64(messagesLeft)) s.logger.Printf("synced %d/%d messages (%.2f%% complete; %s remaining)", i+1, len(toSend), 100*progressRatio, timeRemaining) prevMsg = now From 7102de9f605c7e2900ef8589634edebee1d44dd5 Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 21 Oct 2021 16:19:57 -0500 Subject: [PATCH 03/40] off by one error fixed --- server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server.go b/server.go index 8ee1ba5bc..44e9da6f2 100644 --- a/server.go +++ b/server.go @@ -722,8 +722,8 @@ func (s *Server) Open() error { if now := time.Now(); now.Sub(prevMsg) > time.Second { progressRatio := float64(i+1) / float64(len(toSend)) - messagesLeft := len(toSend) - i - avgTimePerMessage := float64(now.Sub(start)) / float64(i) + messagesLeft := len(toSend) - (i + 1) + avgTimePerMessage := float64(now.Sub(start)) / float64(i+1) timeRemaining := time.Duration(avgTimePerMessage * float64(messagesLeft)) s.logger.Printf("synced %d/%d messages (%.2f%% complete; %s remaining)", i+1, len(toSend), 100*progressRatio, timeRemaining) prevMsg = now From 45e36600f3b299edea7544c295975991456f3bac Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 22 Oct 2021 10:49:07 -0500 Subject: [PATCH 04/40] don't include .*.swp --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d53a2e53c..21dbedb4d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ release-pilosa-fsck.*.*.tar.gz pilosa *.dot .idea/ +.*.swp From 0f108a612dc5ff23b11d6248b543795fa0e74239 Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 22 Oct 2021 11:25:03 -0500 Subject: [PATCH 05/40] refactor and add unit tests --- server.go | 8 +++---- util.go | 9 ++++++++ util_test.go | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 util_test.go diff --git a/server.go b/server.go index 44e9da6f2..f6fd1ef5b 100644 --- a/server.go +++ b/server.go @@ -703,6 +703,7 @@ func (s *Server) Open() error { start := time.Now() prevMsg := start + numMsgs := uint(len(toSend)) s.logger.Printf("start initial cluster state sync") for i := range toSend { for { @@ -721,11 +722,8 @@ func (s *Server) Open() error { } if now := time.Now(); now.Sub(prevMsg) > time.Second { - progressRatio := float64(i+1) / float64(len(toSend)) - messagesLeft := len(toSend) - (i + 1) - avgTimePerMessage := float64(now.Sub(start)) / float64(i+1) - timeRemaining := time.Duration(avgTimePerMessage * float64(messagesLeft)) - s.logger.Printf("synced %d/%d messages (%.2f%% complete; %s remaining)", i+1, len(toSend), 100*progressRatio, timeRemaining) + pctDone := (float64(i+1) / float64(numMsgs)) * 100 + s.logger.Printf("synced %d/%d messages (%.2f%% complete; %s remaining)", i+1, numMsgs, pctDone, EstTimeLeft(start, now, uint(i), numMsgs)) prevMsg = now } } diff --git a/util.go b/util.go index 3bbeff8e3..93f5e9878 100644 --- a/util.go +++ b/util.go @@ -25,6 +25,7 @@ import ( "sort" "strings" "syscall" + "time" "unsafe" "github.com/molecula/featurebase/v2/roaring" @@ -343,3 +344,11 @@ func roaringFragmentHasData(path string, index, field, view string, shard uint64 return } + +// EstTimeLeft returns the estimated remaining time to iterate through some items +// given a start time, the current time, the iteration, and the number of items +func EstTimeLeft(start time.Time, now time.Time, i uint, total uint) time.Duration { + msgsLeft := total - (i + 1) + avgMsgTime := float64(now.Sub(start)) / float64(i+1) + return time.Duration(avgMsgTime * float64(msgsLeft)) +} diff --git a/util_test.go b/util_test.go new file mode 100644 index 000000000..824035744 --- /dev/null +++ b/util_test.go @@ -0,0 +1,59 @@ +package pilosa + +// util_test.go has unit tests for utility functions from util.go + +import ( + "testing" + "time" +) + +func TestEstTimeLeft(t *testing.T) { + cases := []struct { + start time.Time + now time.Time + i uint + total uint + }{ + { + time.Date(1969, time.June, 9, 4, 20, 0, 0, time.UTC), + time.Date(1969, time.June, 9, 4, 21, 0, 0, time.UTC), + 10, + 20, + }, + { + time.Date(1969, time.June, 9, 4, 20, 0, 0, time.UTC), + time.Date(1969, time.June, 9, 4, 20, 4, 0, time.UTC), + 10, + 20, + }, + { + time.Date(1969, time.June, 9, 4, 20, 0, 0, time.UTC), + time.Date(1969, time.June, 9, 4, 21, 1, 5, time.UTC), + 10, + 20, + }, + { + time.Date(1969, time.June, 9, 4, 20, 0, 0, time.UTC), + time.Date(1969, time.June, 9, 4, 21, 0, 0, time.UTC), + 1, + 20, + }, + { + time.Date(1969, time.June, 9, 4, 20, 0, 0, time.UTC), + time.Date(1969, time.June, 9, 4, 21, 0, 0, time.UTC), + 10, + 50, + }, + } + + for _, c := range cases { + // we expect that it will be the avg time per message times + // the number of remaining messages + expected := time.Duration((float64(c.now.Sub(c.start)) / float64(c.i+1)) * float64(c.total-(c.i+1))) + + timeLeft := EstTimeLeft(c.start, c.now, c.i, c.total) + if timeLeft != expected { + t.Errorf("Time left was incorrect, expected: %d, but got: %d", expected, timeLeft) + } + } +} From 681ed9923d16804f932963b8cdbf0bf9ec810638 Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 22 Oct 2021 11:29:09 -0500 Subject: [PATCH 06/40] add license header --- util_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/util_test.go b/util_test.go index 824035744..9aa336f31 100644 --- a/util_test.go +++ b/util_test.go @@ -1,3 +1,17 @@ +// Copyright 2021 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa // util_test.go has unit tests for utility functions from util.go From bd0d68b2fd4f3c7d47fb3d270485abba41f1567f Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 22 Oct 2021 12:54:46 -0500 Subject: [PATCH 07/40] rename function, return pctDone --- server.go | 4 ++-- util.go | 10 ++++++---- util_test.go | 8 ++++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/server.go b/server.go index f6fd1ef5b..8912d38bb 100644 --- a/server.go +++ b/server.go @@ -722,8 +722,8 @@ func (s *Server) Open() error { } if now := time.Now(); now.Sub(prevMsg) > time.Second { - pctDone := (float64(i+1) / float64(numMsgs)) * 100 - s.logger.Printf("synced %d/%d messages (%.2f%% complete; %s remaining)", i+1, numMsgs, pctDone, EstTimeLeft(start, now, uint(i), numMsgs)) + estimate, pctDone := GetLoopProgress(start, now, uint(i), numMsgs) + s.logger.Printf("synced %d/%d messages (%.2f%% complete; %s remaining)", i+1, numMsgs, pctDone, estimate) prevMsg = now } } diff --git a/util.go b/util.go index 93f5e9878..d165f634e 100644 --- a/util.go +++ b/util.go @@ -345,10 +345,12 @@ func roaringFragmentHasData(path string, index, field, view string, shard uint64 return } -// EstTimeLeft returns the estimated remaining time to iterate through some items -// given a start time, the current time, the iteration, and the number of items -func EstTimeLeft(start time.Time, now time.Time, i uint, total uint) time.Duration { +// GetLoopProgress returns the estimated remaining time to iterate through some items +// as well as the loop completion percentage with the following parameters: +// the start time, the current time, the iteration, and the number of items +func GetLoopProgress(start time.Time, now time.Time, i uint, total uint) (time.Duration, float64) { msgsLeft := total - (i + 1) avgMsgTime := float64(now.Sub(start)) / float64(i+1) - return time.Duration(avgMsgTime * float64(msgsLeft)) + pctDone := (float64(i+1) / float64(total)) * 100 + return time.Duration(avgMsgTime * float64(msgsLeft)), pctDone } diff --git a/util_test.go b/util_test.go index 9aa336f31..ea27f1c5b 100644 --- a/util_test.go +++ b/util_test.go @@ -21,7 +21,7 @@ import ( "time" ) -func TestEstTimeLeft(t *testing.T) { +func TestGetLoopProgress(t *testing.T) { cases := []struct { start time.Time now time.Time @@ -64,10 +64,14 @@ func TestEstTimeLeft(t *testing.T) { // we expect that it will be the avg time per message times // the number of remaining messages expected := time.Duration((float64(c.now.Sub(c.start)) / float64(c.i+1)) * float64(c.total-(c.i+1))) + expectedPct := 100 * (float64(c.i+1) / float64(c.total)) - timeLeft := EstTimeLeft(c.start, c.now, c.i, c.total) + timeLeft, pctDone := GetLoopProgress(c.start, c.now, c.i, c.total) if timeLeft != expected { t.Errorf("Time left was incorrect, expected: %d, but got: %d", expected, timeLeft) } + if pctDone != expectedPct { + t.Errorf("Percentage done was incorrect, expected: %f, but got: %f", expectedPct, pctDone) + } } } From 5dd4b0e048a6e2f6ab4ac4673639930f857d4a25 Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 22 Oct 2021 14:49:05 -0500 Subject: [PATCH 08/40] rename vars to more sensible names --- util.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/util.go b/util.go index d165f634e..ebbff6e75 100644 --- a/util.go +++ b/util.go @@ -348,9 +348,9 @@ func roaringFragmentHasData(path string, index, field, view string, shard uint64 // GetLoopProgress returns the estimated remaining time to iterate through some items // as well as the loop completion percentage with the following parameters: // the start time, the current time, the iteration, and the number of items -func GetLoopProgress(start time.Time, now time.Time, i uint, total uint) (time.Duration, float64) { - msgsLeft := total - (i + 1) - avgMsgTime := float64(now.Sub(start)) / float64(i+1) - pctDone := (float64(i+1) / float64(total)) * 100 - return time.Duration(avgMsgTime * float64(msgsLeft)), pctDone +func GetLoopProgress(start time.Time, now time.Time, iteration uint, total uint) (remaining time.Duration, pctDone float64) { + itemsLeft := total - (iteration + 1) + avgItemTime := float64(now.Sub(start)) / float64(iteration+1) + pctDone = (float64(iteration+1) / float64(total)) * 100 + return time.Duration(avgItemTime * float64(itemsLeft)), pctDone } From d934d117da74875477b0400460c8103aefe90548 Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 22 Oct 2021 14:52:57 -0500 Subject: [PATCH 09/40] add test case names --- util_test.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/util_test.go b/util_test.go index ea27f1c5b..3416a95a8 100644 --- a/util_test.go +++ b/util_test.go @@ -22,37 +22,44 @@ import ( ) func TestGetLoopProgress(t *testing.T) { + // TODO: try to find more sneaky cases cases := []struct { + name string start time.Time now time.Time i uint total uint }{ { + "one minute, half done", time.Date(1969, time.June, 9, 4, 20, 0, 0, time.UTC), time.Date(1969, time.June, 9, 4, 21, 0, 0, time.UTC), 10, 20, }, { + "four seconds, half done", time.Date(1969, time.June, 9, 4, 20, 0, 0, time.UTC), time.Date(1969, time.June, 9, 4, 20, 4, 0, time.UTC), 10, 20, }, { + "one minute, one second, 5 μs, half done", time.Date(1969, time.June, 9, 4, 20, 0, 0, time.UTC), time.Date(1969, time.June, 9, 4, 21, 1, 5, time.UTC), 10, 20, }, { + "one minute, one done", time.Date(1969, time.June, 9, 4, 20, 0, 0, time.UTC), time.Date(1969, time.June, 9, 4, 21, 0, 0, time.UTC), 1, 20, }, { + "one minute, 1/5 done", time.Date(1969, time.June, 9, 4, 20, 0, 0, time.UTC), time.Date(1969, time.June, 9, 4, 21, 0, 0, time.UTC), 10, @@ -61,17 +68,19 @@ func TestGetLoopProgress(t *testing.T) { } for _, c := range cases { - // we expect that it will be the avg time per message times - // the number of remaining messages - expected := time.Duration((float64(c.now.Sub(c.start)) / float64(c.i+1)) * float64(c.total-(c.i+1))) - expectedPct := 100 * (float64(c.i+1) / float64(c.total)) + t.Run(c.name, func(t *testing.T) { + // we expect that it will be the avg time per message times + // the number of remaining messages + expected := time.Duration((float64(c.now.Sub(c.start)) / float64(c.i+1)) * float64(c.total-(c.i+1))) + expectedPct := 100 * (float64(c.i+1) / float64(c.total)) - timeLeft, pctDone := GetLoopProgress(c.start, c.now, c.i, c.total) - if timeLeft != expected { - t.Errorf("Time left was incorrect, expected: %d, but got: %d", expected, timeLeft) - } - if pctDone != expectedPct { - t.Errorf("Percentage done was incorrect, expected: %f, but got: %f", expectedPct, pctDone) - } + timeLeft, pctDone := GetLoopProgress(c.start, c.now, c.i, c.total) + if timeLeft != expected { + t.Errorf("Time left was incorrect, expected: %d, but got: %d", expected, timeLeft) + } + if pctDone != expectedPct { + t.Errorf("Percentage done was incorrect, expected: %f, but got: %f", expectedPct, pctDone) + } + }) } } From d51c6b950f1cf05253ae481b86b6ea9cfc920c20 Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 25 Oct 2021 15:24:55 -0500 Subject: [PATCH 10/40] meaningless commit to kick off sonarcloud with new rules --- util_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/util_test.go b/util_test.go index 3416a95a8..c659053c5 100644 --- a/util_test.go +++ b/util_test.go @@ -15,6 +15,7 @@ package pilosa // util_test.go has unit tests for utility functions from util.go +// import ( "testing" From dc8702ea3e8df040184dd919c74d6995b3dad4c7 Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 13 Oct 2021 12:38:54 -0500 Subject: [PATCH 11/40] use testhook auditor to track Qcx open/close This also requires doing something to keep the TxGroup in each Qcx from holding its Tx references after the Qcx closes, because otherwise the list of Qcxs that we keep to verify that they all got closed ends up keeping every shared/read-only Tx open forever, resulting in many gigabytes of memory usage when running with the race detector. To avoid having to reason about whether anything would ever access a nil TxGroup, or run through iteratively zeroing maps, we just make a new empty group at that point. --- txfactory.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/txfactory.go b/txfactory.go index 93e9449df..7c0d06add 100644 --- a/txfactory.go +++ b/txfactory.go @@ -31,6 +31,7 @@ import ( "github.com/molecula/featurebase/v2/roaring" txkey "github.com/molecula/featurebase/v2/short_txkey" "github.com/molecula/featurebase/v2/storage" + "github.com/molecula/featurebase/v2/testhook" . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck "github.com/pkg/errors" "github.com/zeebo/blake3" @@ -147,6 +148,11 @@ func (q *Qcx) Finish() (err error) { } } err2 := q.Grp.FinishGroup() + // drop the old group so we aren't holding references to all those Tx + q.Grp = q.Txf.NewTxGroup() + if !q.done { + _ = testhook.Closed(q.Txf.holder.Auditor, q, nil) + } q.done = true if err != nil { @@ -164,7 +170,11 @@ func (q *Qcx) Abort() { (*q.RequiredForAtomicWriteTx).Rollback() } q.Grp.AbortGroup() - + // drop the old group so we aren't holding references to all those Tx + q.Grp = q.Txf.NewTxGroup() + if !q.done { + _ = testhook.Closed(q.Txf.holder.Auditor, q, nil) + } q.done = true } @@ -197,6 +207,7 @@ func (f *TxFactory) NewQcx() (qcx *Qcx) { if f.typeOfTx == "roaring" { qcx.isRoaring = true } + _ = testhook.Opened(f.holder.Auditor, qcx, nil) return } From ecd0ecc6d1b8fcef14a642aca6cbee5998e7f22d Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 13 Oct 2021 12:39:06 -0500 Subject: [PATCH 12/40] add featurebase to .gitignore we ignored pilosa binaries but we've renamed so now the binary is named featurebase. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 21dbedb4d..2082bd5c0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ release-pilosa-fsck.*.*.tar.gz /log.* /tourna.log.* pilosa +/featurebase *.dot .idea/ .*.swp From ad30a926f4014e3685a68e95cf678ab85256383b Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 4 Oct 2021 15:09:51 -0500 Subject: [PATCH 13/40] Giant Commit: drop a bunch of stuff we don't use. These commits are hard to disentagle, and doing them separately means re-modifying the same chunks of code several times before removing it, and similar things. Basically: (1) Drop the bolt backend storage. (2) Drop the blue-green wrapper that compares two backends. (3) Drop unused or barely-used Tx API components from all the remaining backends. (4) Minor related cleanup to simplify things related to these. The boltdb backend existed only to verify RBF. The blue-green wrapper was mostly used to verify RBF, but in practice we had to do a lot of working around that, and it introduced a lot of special cases. Types removed: IteratorFinder: Used only to implement the roaring iterator on top of boltdb, and to complicate the way it worked in roaring. Reverted the complications. Also unexport NewSliceContainers which is used only for that outside of roaring's internals. PortMapper from cluster_internal_test.go: Used only for a test we removed early this year. Never used for anything else. RawRoaringData: Totally unused. TxStore: Totally unused. Functions removed from Tx API, and sometimes corresponding members were removed from structs: * Dump: debugging code, I don't think I found any actually reachable paths to it. * Group: only used for debugging TxGroup stuff * IncrementOpN: only used by fragment, fragment can increment its own opN. * Options: unused? * Pointer: debugging only * Readonly: used only to decide how to handle Tx in a TxGrp, but we never add a non-readonly Tx to a TxGrp. Removed also all the corresponding write-aware stuff. * RoaringBitmapReader: Used exactly once, can just be a bm.WriteTo. * Sn (and OpenSnList): Unused * UnionInPlace: unused and conceptually-invalid; it didn't write to storage and shouldn't have, and was just "create a bitmap then call union-in-place", which we can do directly. * UseRowCache: just checked storage.UseRowCache. Other things removed: The SetRequiredForAtomicWriteTx and ClearRequiredForAtomicWriteTx functions go away, since nothing now seems to be using them? Same for holder_internal_test's `testHasBit` and `testMustNotHaveBit`, which were unused. The DBPerShard "DeleteDBPath" and "HasData" functions and related parts were mostly unused; took out the parts that were never actually being reached. Changed the API of one function to simplify special cases and remove things: * ImportRoaringBits had a special "data" argument which gave it subtly different semantics for RBF and roaring (for roaring, it could produce a roaring bitmap *with ops log*), didn't seem to be adding much. Removed corresponding "readStorageFromArchive" which is not otherwise used. Also took out various debugging/dumping functions that were unused and may have bitrotted. Dropped a test from txfactory_internal_test, and the "pjobs" code, because those two were the only things that needed Barrier and thus idem, which lets us drop two more dependencies. We already have errgroup for grouping things which want to terminate as soon as one of them errors, approximately. To do better we'd have to have context-threading, really. Unbroke the WriteFragment test for non-roaring tests and made it not roaring-only. --- api.go | 12 +- barrier.go | 219 ----- barrier_test.go | 90 -- bluegreentx.go | 1028 ---------------------- bluegreentx_test.go | 100 --- bolt.go | 1612 ----------------------------------- bolt_test.go | 1270 --------------------------- catcher.go | 71 +- cluster_internal_test.go | 50 -- cmd/roaring-migrate/main.go | 2 +- ctl/server.go | 2 +- dbshard.go | 638 ++------------ dbshard_internal_test.go | 22 +- delete_test.go | 1 - executor_test.go | 1 - fragment.go | 91 +- fragment_internal_test.go | 43 +- go.mod | 2 - go.sum | 4 - holder.go | 19 - holder_internal_test.go | 68 -- index.go | 10 - pjobs.go | 115 --- pjobs_test.go | 51 -- rbf.go | 94 +- rbf/cursor_internal_test.go | 110 +-- rbf/cursorx.go | 8 +- rbf/db.go | 2 +- rbf/tx.go | 147 +--- rbf/tx_test.go | 29 - rbf/util.go | 55 +- roaring/roaring.go | 37 +- rrtx.go | 93 +- server.go | 4 +- server/cluster_test.go | 24 - server/config.go | 11 +- stattx.go | 123 +-- storage/cache.go | 2 +- tx.go | 107 +-- txfactory.go | 676 +-------------- txfactory_internal_test.go | 385 +-------- util.go | 230 ----- utils_internal_test.go | 30 - view.go | 16 +- 44 files changed, 223 insertions(+), 7481 deletions(-) delete mode 100644 barrier.go delete mode 100644 barrier_test.go delete mode 100644 bluegreentx.go delete mode 100644 bluegreentx_test.go delete mode 100644 bolt.go delete mode 100644 bolt_test.go delete mode 100644 pjobs.go delete mode 100644 pjobs_test.go diff --git a/api.go b/api.go index ca3506a0e..7bcc0dda7 100644 --- a/api.go +++ b/api.go @@ -1510,9 +1510,7 @@ func addClearToImportOptions(opts []ImportOption) []ImportOption { return append(opts, OptImportOptionsClear(true)) } -// Import avoids re-writing a bajillion tests to be transaction-aware by allowing a nil pQcx. -// It is convenient for some tests, particularly those in loops, to pass a nil qcx and -// treat the Import as having been commited when we return without error. We make it so. +// Import does the top-level importing. func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts ...ImportOption) (err error) { if req.Clear { opts = addClearToImportOptions(opts) @@ -1648,8 +1646,8 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest, return errors.Wrap(err, "committing") } -// ImportValue avoids re-writing a bajillion tests by allowing a nil pQcx. -// Then we will commit before returning. +// ImportValue is a wrapper around the common code in ImportValueWithTx, which +// currently just translates req.Clear into a clear ImportOption. func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error { if req.Clear { opts = addClearToImportOptions(opts) @@ -2654,9 +2652,7 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64 if err != nil { return err } - //need to find the path to the db - //will not work on blue green - db := dbs.W[0] + db := dbs.W finalPath := db.Path() + "/data" tempPath := finalPath + ".tmp" o, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) diff --git a/barrier.go b/barrier.go deleted file mode 100644 index b150dc486..000000000 --- a/barrier.go +++ /dev/null @@ -1,219 +0,0 @@ -// home https://github.com/glycerine/lmdb-go -// Copyright (c) 2020, the lmdb-go authors -// Copyright (c) 2015, Bryan Matsuo -// All rights reserved. - -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: - -// Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. - -// Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. - -// Neither the name of the author nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. - -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package pilosa - -import ( - "github.com/glycerine/idem" -) - -// Barrier allows us to temporarily halt all readers, so that -// a writer can commit alone and thus compact the db. -// The Barrier starts unblocked, alllowing passage to any -// caller of WaitAtGate(). -type Barrier struct { - wait chan *appointment // send upon entering the waiting room. - halt *idem.Halter - blockReqCh chan *blockReq - unblockCh chan *unblock -} - -type blockReq struct { - count int - done chan struct{} -} - -func newBlockReq(count int) *blockReq { - return &blockReq{ - count: count, - done: make(chan struct{}), - } -} - -type appointment struct { - id int - done chan struct{} -} - -func newAppointment(id int) *appointment { - return &appointment{ - id: id, - done: make(chan struct{}), - } -} - -// NewBarrier is either open, allowing immediate passage, -// or blocked, halting all callers at WaitAtGate() -// until the barrier is opened. By default it is open. -// -// Barrier.Close() must be called when the barrier -// is no longer needed to avoid a goroutine leak. -func NewBarrier() (b *Barrier) { - b = &Barrier{ - wait: make(chan *appointment), // waiters indicate they are waiting for the gate by sending here. - halt: idem.NewHalter(), - blockReqCh: make(chan *blockReq), - unblockCh: make(chan *unblock), - } - go func() { - defer b.halt.Done.Close() - - var waitlist []*appointment - var curBlockReq *blockReq - - for { - select { - case br := <-b.blockReqCh: - if br.count == 0 { - close(br.done) - continue - } - if curBlockReq == nil { - // good, changing state from open to closed barrier. - } else { - panic("got 2nd block request atop of first") - } - curBlockReq = br - //vv("barrier: request to block for %v waiters", br.count) - if len(waitlist) != 0 { - panic("had waiters when we were open, internal/client bug") - } - case appt := <-b.wait: - //vv("barrier.wait sees appt = '%#v' and curBlockReq = '%#v'", appt, curBlockReq) - if curBlockReq == nil { - close(appt.done) - continue - } - waitlist = append(waitlist, appt) - n := len(waitlist) - th := curBlockReq.count - if th < 0 { - // infinite waiters. we block everybody until we - // see an unblock request. - continue - } - if n >= th { - close(curBlockReq.done) - curBlockReq = nil - } - case ub := <-b.unblockCh: - for _, appt := range waitlist { - close(appt.done) - } - waitlist = nil - curBlockReq = nil - close(ub.done) - case <-b.halt.ReqStop.Chan: - return - } - } - }() - return -} - -// WaitAtGate will return immediately -// if the barrier is unblocked. Otherwise -// it will not return until another -// goroutine unblocks the barrier. -func (b *Barrier) WaitAtGate(id int) { - appt := newAppointment(id) - select { - case b.wait <- appt: - select { - case <-appt.done: - case <-b.halt.ReqStop.Chan: - } - case <-b.halt.ReqStop.Chan: - } -} - -// Close should be called to stop the -// barrier's background goroutine when -// you are done using the barrier. -func (b *Barrier) Close() { - b.halt.ReqStop.Close() - <-b.halt.Done.Chan -} - -type unblock struct { - done chan struct{} -} - -func newUnblock() *unblock { - return &unblock{ - done: make(chan struct{}), - } -} - -// Unblock lets all waiting goroutines resume execution. -func (b *Barrier) UnblockReaders() { - ub := newUnblock() - select { - case b.unblockCh <- ub: - select { - case <-ub.done: - case <-b.halt.ReqStop.Chan: - } - case <-b.halt.ReqStop.Chan: - } -} - -// BlockUntil is called with a count, the -// number of waiters required to be present and waiting -// at the gate before call returns. -// A count of < 0 will return immediately and raise -// the barrier to any number of arriving readers. -// A count of 0 is a no-op. -// -// Otherwise we raise the barrier -// and wait until we have seen count other goroutines waiting -// on it. -// -// We return without releasing the waiters. Call -// Open when you want them to resume. -func (b *Barrier) BlockUntil(count int) { - if count == 0 { - return - } - req := newBlockReq(count) - b.blockReqCh <- req - if count > 0 { - <-req.done - } -} - -// BlockAllReadersNoWait raises the barrier to -// an infinite number of waiters and returns immediately -// to the caller. -func (b *Barrier) BlockAllReadersNoWait() { - req := newBlockReq(-1) // -1 means block any number of readers. - b.blockReqCh <- req - // don't wait. <-req.done -} diff --git a/barrier_test.go b/barrier_test.go deleted file mode 100644 index 70fd4d311..000000000 --- a/barrier_test.go +++ /dev/null @@ -1,90 +0,0 @@ -// home https://github.com/glycerine/lmdb-go -// Copyright (c) 2020, the lmdb-go authors -// Copyright (c) 2015, Bryan Matsuo -// All rights reserved. - -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: - -// Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. - -// Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. - -// Neither the name of the author nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. - -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package pilosa - -import ( - "fmt" - "sync/atomic" - "testing" - "time" -) - -func TestBarrierHolds(t *testing.T) { - b := NewBarrier() - defer b.Close() - - released := int64(0) - - waiter := func(i int) { - b.WaitAtGate(i) - //vv("goro %v is released", i) - atomic.AddInt64(&released, 1) - } - - for i := 0; i < 3; i++ { - go waiter(i) - } - time.Sleep(time.Second) - r := atomic.SwapInt64(&released, 0) - if r != 3 { - panic("open barrier held back goro") - } - //vv("good: barrier started open") - - //seenAll := make(chan bool) - b.BlockAllReadersNoWait() - for i := 0; i < 3; i++ { - go waiter(i) - } - - time.Sleep(time.Second) - r = atomic.SwapInt64(&released, 0) - if r != 0 { - panic("bad: barrier did not hold back goro") - } - //vv("good: barrier of 4 did not release on 3") - go waiter(4) - - time.Sleep(time.Second) - r = atomic.SwapInt64(&released, 0) - if r != 0 { - panic(fmt.Sprintf("bad: barrier did not hold back goro, should wait for unblock. r = %v", r)) - } - - b.UnblockReaders() - - time.Sleep(time.Second) - r = atomic.SwapInt64(&released, 0) - if r != 4 { - panic("bad: unblock should have released 4 goro") - } - -} diff --git a/bluegreentx.go b/bluegreentx.go deleted file mode 100644 index 07c716437..000000000 --- a/bluegreentx.go +++ /dev/null @@ -1,1028 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "fmt" - "io" - "reflect" - "sync" - - "github.com/molecula/featurebase/v2/roaring" - txkey "github.com/molecula/featurebase/v2/short_txkey" - . "github.com/molecula/featurebase/v2/vprint" -) - -// blueGreenTx runs two Tx together and notices differences in their output. -// By convention, the 'b' Tx is the output that is returned to caller. -// -// Warning: DATA RACES are expected if RoaringTx is one side of the Tx pair. -// The checkDatabase() call will do reads of the fragments at Commit/Rollback, -// while the snapshotqueue may be doing writes. -// -// Do not run with go test -race and expect it to be race free with RoaringTx -// on one arm. -// -// Note: using the dbshard.go DBShard.mut RWMutex to begin and end -// both the A and B transactions atomically, we support a single importer and -// lots of readers running under blue-green transactions. Two writers a.k.a. two -// github ingests at once will deadlock eventually, but I think that may be asking -// for more than we want to test under blue-green, as it would require a bunch of -// test-only internal executor logic that could mess with the production path. -// So, for now, a limitation on blue green tests is that they be single -// writer/single importer going at once. -// -type blueGreenTx struct { - a Tx - b Tx // b's output is returned - - o Txo - as string - bs string - - types []txtype - hasRoaring bool - - // roaring will not create as many Tx (they are - // psuedo Tx anyway), espcially when deleting - // files. Return the non-roaring Sn if - // possible, by referencing useSnA. - useSnA bool - - idx *Index - - checker blueGreenChecker - mu sync.Mutex - rollbackOrCommitDone bool - - txf *TxFactory - - short bool // short Dump or long - - FullDump bool // else quieter, don't attemp Dump() if false. -} - -// blueGreenRegistry is used to force checking of (read) transactions -// before writes happen, if roaring is on one of the A/B branches. -// Because roaring won't have an MVCC view of the world. Writes to -// roaring will show up, while writes to the DB won't show up on -// readTx that have already started. -type blueGreenRegistry struct { - mu sync.Mutex - m map[int64]*blueGreenTx - types []txtype - hasRoaring bool -} - -// if we have raoring in the mix we cannot expect reads -// to match up, but otherwise do. -func newBlueGreenReg(types []txtype) *blueGreenRegistry { - - hasRoaring := false - if types[0] == roaringTxn || types[1] == roaringTxn { - hasRoaring = true - } - return &blueGreenRegistry{ - m: make(map[int64]*blueGreenTx), - types: types, - hasRoaring: hasRoaring, - } -} - -// add remembers the tx so we can check that -// all tx were finished before Close(). -func (b *blueGreenRegistry) add(c *blueGreenTx) { - b.mu.Lock() - defer b.mu.Unlock() - if c.useSnA { - b.m[c.a.Sn()] = c - } else { - b.m[c.b.Sn()] = c - } -} - -func (b *blueGreenRegistry) finishedTx(tx *blueGreenTx) { - b.mu.Lock() - defer b.mu.Unlock() - sn := tx.Sn() - delete(b.m, sn) - //vv("blueGreenRegistry deleted _sn_ %v", sn) - - // Note that a tx.o.dbs.Cleanup(tx) call should not be needed, - // because the individual tx will call cleanup themselves. -} - -func (b *blueGreenRegistry) Close() { - b.mu.Lock() - defer b.mu.Unlock() - if len(b.m) > 0 { - PanicOn(fmt.Sprintf("still have open/unchecked blueGreenTx: '%#v'", b.m)) - //AlwaysPrintf("still have unchecked blueGreenTx: '%#v'", b.m) - } -} - -func (txf *TxFactory) newBlueGreenTx(a, b Tx, idx *Index, o Txo) *blueGreenTx { - as := a.Type() - bs := b.Type() - c := &blueGreenTx{a: a, - b: b, - idx: idx, - as: as, - bs: bs, - txf: txf, - types: txf.types, - hasRoaring: txf.blueGreenReg.hasRoaring, - short: true, - } - - if c.types[1] == roaringTxn { - c.useSnA = true - } - //vv("newBlueGreenTx with a.sn=%v with o.Shard=%v", c.Sn(), int(o.Shard)) - - c.checker.c = c - c.o = o - txf.blueGreenReg.add(c) - return c -} - -var _ Tx = (*blueGreenTx)(nil) - -func (c *blueGreenTx) Type() string { - return c.a.Type() + "_" + c.b.Type() -} - -var blueGreenTxDumpMut sync.Mutex - -func (c *blueGreenTx) Dump(short bool, shard uint64) { - - if !c.FullDump { - return - } - - blueGreenTxDumpMut.Lock() - defer blueGreenTxDumpMut.Unlock() - fmt.Printf("%v blueGreenTx.Dump ============== \n", FileLine(2)) - fmt.Printf("A(%v) Dump:\n", c.as) - c.a.Dump(short, shard) - fmt.Printf("B(%v) Dump:\n", c.bs) - c.b.Dump(short, shard) - - if !short { - fmt.Printf("dbPerShard.DumpAll(): idx=%p\n", c.idx) - c.idx.holder.txf.dbPerShard.DumpAll() - } -} - -func (c *blueGreenTx) Readonly() bool { - a := c.a.Readonly() - b := c.b.Readonly() - if a != b { - PanicOn(fmt.Sprintf("Readonly difference, a=%v, but b =%v", a, b)) - } - return b -} - -// for now we just return B's list, since this is involved in -// holder Open which can happen before any blue-green is done; -// in fact this is instrumental in setting up the sync from -// green to blue. -func (c *blueGreenTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvB []txkey.FieldView, errB error) { - fvB, errB = c.b.GetSortedFieldViewList(idx, shard) - return -} - -func (c *blueGreenTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { - c.checker.see(index, field, view, shard) - // can't really do simultaneous iteration on A and B, so punt and - // just give back B. - return c.b.NewTxIterator(index, field, view, shard) -} - -func (c *blueGreenTx) Pointer() string { - return fmt.Sprintf("%p", c) -} - -func (c *blueGreenTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - c.checker.see(index, field, view, shard) - c.a.IncrementOpN(index, field, view, shard, changedN) - c.b.IncrementOpN(index, field, view, shard, changedN) -} - -// compareTxState is called for the first Commit or Rollback a blueGreenTx sees. -func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) { - if c.o.blueGreenOff { - return - } - here := fmt.Sprintf("%v/%v/%v/%v", index, field, view, shard) - //vv("compareTxState here = '%v', _sn_ %v gid=%v", here, c.Sn(), curGID()) - aIter, aFound, aErr := c.a.ContainerIterator(index, field, view, shard, 0) - bIter, bFound, bErr := c.b.ContainerIterator(index, field, view, shard, 0) - if aErr == nil || aIter != nil { - defer aIter.Close() - } - if bErr == nil || bIter != nil { - defer bIter.Close() - } - - if aFound != bFound { - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) ContainerIterator had aFound=%v, but B(%v) had bFound=%v; at '%v'", here, c.as, aFound, c.bs, bFound, Stack())) - } - - if aErr != nil || bErr != nil { - if aErr != nil && bErr != nil { - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) reported err '%v'; B(%v) reported err '%v' at %v", here, c.as, aErr, c.bs, bErr, Stack())) - } - if aErr != nil { - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) reported err %v at %v; but B(%v) did not", here, c.as, aErr, c.bs, Stack())) - } - if bErr != nil { - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: B(%v) reported err %v at %v; but A(%v) did not", here, c.bs, bErr, c.as, Stack())) - } - } - for aIter.Next() { - aKey, aValue := aIter.Value() - - if !bIter.Next() { - AlwaysPrintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, dump to follow, Stack=\n %v\n\n and here is dump:", here, c.as, aKey, c.bs, Stack()) - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, at %v", here, c.as, aKey, c.bs, Stack())) - } - bKey, bValue := bIter.Value() - if bKey != aKey { - AlwaysPrintf("problem in caller %v", Caller(2)) - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, Stack())) - } - if err := aValue.BitwiseCompare(bValue); err != nil { - c.Dump(c.short, shard) - //vv("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at Stack=%v", here, aKey, err, c.as, c.bs, Stack()) - PanicOn(fmt.Sprintf("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at Stack=%v", here, aKey, err, c.as, c.bs, Stack())) - } - //vv("successfully matched aKey(%v)='%v' and bKey(%v)='%v'", c.as, aKey, c.bs, bKey) - } - // end checking everything in A, but does B have more? - if bIter.Next() { - AlwaysPrintf("bIter has more than it should. problem in caller %v. _sn_ %v", Caller(2), c.Sn()) - c.Dump(c.short, shard) - bKey, _ := bIter.Value() - PanicOn(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, (a.sn=%v) (b.sn=%v) at %v", here, c.bs, bKey, c.as, c.a.Sn(), c.b.Sn(), Stack())) - } - //vv("done without problem. compareTxState here = '%v', _sn_ %v gid=%v", here, c.Sn(), curGID()) -} - -func (c *blueGreenTx) checkDatabase() { - if c.o.blueGreenOff { - return - } - if c.hasRoaring && !c.o.Write { - // With roaring on one arm, we only check the we are A/B - // consistent after every write. - // - // Ideally reads can only see that consitent state, and don't need - // to be checked themselves-- but we do try if both A and B - // are transactional. Sketch of proof by induction that - // write checking should, theoretically, suffice: - // Starting with zero data, if we have agreement in both A/B - // database state after each write, then - // because there is only ever a single - // writer (for LMDB/RBF), we should always have the same - // data state between A and B as long as every prior - // A/B check of the serialized writes suceeded. - // - // This avoids a key problem we discovered when A/B checking reads - // with roaring on one arm. - // The MVCC of the transactional engines means that reads that - // start before a write commit will look very different - // when comparing to roaring's non-transactional state. - return - } - - c.checker.mu.Lock() - defer c.checker.mu.Unlock() - if c.checker.checkDone { - return // idemopotent. checkDatabase can be called twice. Only the first does the checks. - } - c.checker.checkDone = true - - for index, fields := range c.checker.seen() { - for field, views := range fields { - for view, shards := range views { - for shard := range shards { - c.compareTxState(index, field, view, shard) - } - } - } - } -} - -func (c *blueGreenTx) IsDone() bool { - return c.b.IsDone() -} - -func (c *blueGreenTx) Rollback() { - c.mu.Lock() - defer c.mu.Unlock() - if c.rollbackOrCommitDone { - return // avoid using discarded tx for Dump, which will PanicOn. - } - c.rollbackOrCommitDone = true - - if c.o.Write { - c.checkDatabase() - } - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - //vv("blueGreenTx.Rollback() about to call (%v) a.Rollback()", c.as) - c.a.Rollback() - //vv("blueGreenTx.Rollback() about to call (%v) b.Rollback()", c.bs) - c.b.Rollback() - //vv("blueGreenTx.Rollback() done. bgtx p=%p", c) - - c.txf.blueGreenReg.finishedTx(c) -} - -func (c *blueGreenTx) Commit() error { - c.mu.Lock() - defer c.mu.Unlock() - - if c.rollbackOrCommitDone { - return nil - } - //vv("blueGreenTx.Commit() called. bgtx p=%p", c) - c.rollbackOrCommitDone = true - if c.o.Write { - if !c.o.blueGreenOff { - c.checkDatabase() - } - } - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - errA := c.a.Commit() - _ = errA - errB := c.b.Commit() - - compareErrors(errA, errB) - c.txf.blueGreenReg.finishedTx(c) - return errB -} - -func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.RoaringBitmap(index, field, view, shard) - _, _ = a, errA - b, errB := c.b.RoaringBitmap(index, field, view, shard) - if !c.o.blueGreenOff { - compareErrors(errA, errB) - - slcA := a.Slice() - slcB := b.Slice() - if !reflect.DeepEqual(slcA, slcB) { - PanicOn("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!") - } - } - return b, errB -} - -func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.Container(index, field, view, shard, key) - b, errB := c.b.Container(index, field, view, shard, key) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - err = a.BitwiseCompare(b) - PanicOn(err) - } - return b, errB -} - -func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - errA := c.a.PutContainer(index, field, view, shard, key, rc) - errB := c.b.PutContainer(index, field, view, shard, key, rc) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return errB -} - -func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { - c.checker.see(index, field, view, shard) - - // these are the first port of call for debugging, so we leave them in. - // ================== begin save comments. - //c.checkDatabase() - ////vv("got past database check at TOP of ImportRoaringBits") - //c.Dump(c.short, shard) - ////vv("done with top dump; clear=%v", clear) - // ================== end save comments. - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - - // remember where the iterator started, so we can replay it a second time. - rit2 := rit.Clone() - PanicOn(err) - - changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) - changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize, data) - - if !c.o.blueGreenOff { - - if len(data) == 0 { - // okay to check! otherwise we are in the fragment.fillFragmentFromArchive - // case where we know that RoaringTx.ImportRoaringBits changed and rowSet will - // be inaccurate. - if changedA != changedB { - PanicOn(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB)) - } - if len(rowSetA) != len(rowSetB) { - PanicOn(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB)) - } - for k, va := range rowSetA { - vb, ok := rowSetB[k] - if !ok { - PanicOn(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB)) - } - if va != vb { - PanicOn(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb)) - } - } - } - compareErrors(errA, errB) - c.checkDatabase() - } - return changedB, rowSetB, errB -} - -func (c *blueGreenTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - errA := c.a.RemoveContainer(index, field, view, shard, key) - errB := c.b.RemoveContainer(index, field, view, shard, key) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return errB -} - -func (c *blueGreenTx) UseRowCache() bool { - // avoid cross-talk between our two implementations - // by never allowing either to use the row cache. - return false -} - -var _ = (&blueGreenTx{}).isIn // happy linter - -func (c *blueGreenTx) isIn(index, field, view string, shard uint64, ckey uint64) (r []bool) { - r = make([]bool, 2) - inA, errA := c.a.Contains(index, field, view, shard, ckey) - PanicOn(errA) - inB, errB := c.b.Contains(index, field, view, shard, ckey) - PanicOn(errB) - r[0] = inA - r[1] = inB - return -} - -func (c *blueGreenTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - c.checker.see(index, field, view, shard) - //vv("blueGreenTx) Add(index=%v, field=%v, view=%v, shard=%v", index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Add() PanicOn '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, Stack()) - PanicOn(r) - } - }() - - // must copy a before calling Add(), since RoaringTx.Add() uses roaring.DirectAddN() - // which modifies the input array a. - a2 := make([]uint64, len(a)) - copy(a2, a) - - ach, errA := c.a.Add(index, field, view, shard, a...) - _, _ = ach, errA - - bch, errB := c.b.Add(index, field, view, shard, a2...) - - if !c.o.blueGreenOff { - - if ach != bch { - PanicOn(fmt.Sprintf("Add() difference, ach=%v, but bch=%v; errA='%v'; errB='%v'", ach, bch, errA, errB)) - } - compareErrors(errA, errB) - } - - return bch, errB -} - -func compareErrors(errA, errB error) { - switch { - case errA == nil && errB == nil: - // OK - case errA == nil: - PanicOn(fmt.Sprintf("errA is nil, but errB = %#v", errB)) - case errB == nil: - PanicOn(fmt.Sprintf("errB is nil, but errA = %#v", errA)) - default: - ae := errA.Error() - be := errB.Error() - if ae != be { - PanicOn(fmt.Sprintf("errA is '%v', but errB is '%v'", ae, be)) - } - } -} - -func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - ach, errA := c.a.Remove(index, field, view, shard, a...) - _, _ = ach, errA - bch, errB := c.b.Remove(index, field, view, shard, a...) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return bch, errB -} - -func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - ax, errA := c.a.Contains(index, field, view, shard, key) - _, _ = ax, errA - bx, errB := c.b.Contains(index, field, view, shard, key) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return bx, errB -} - -func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - - ait, afound, errA := c.a.ContainerIterator(index, field, view, shard, firstRoaringContainerKey) - _, _, _ = ait, afound, errA - - bit, bfound, errB := c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - - if errB != nil { - // RoaringTx can return an iterator and an error, so be sure Close it we have it. - if ait != nil { - ait.Close() - } - if bit != nil { - bit.Close() - } - return nil, bfound, errB - } - if errA != nil { - // RoaringTx can return an iterator and an error, so be sure Close it we have it. - if ait != nil { - ait.Close() - } - } - - // INVAR: errA == errB == nil - bgi := NewBlueGreenIterator(c, ait, bit) - return bgi, bfound, errB -} - -func (tx *blueGreenTx) GetFieldSizeBytes(index, field string) (uint64, error) { - return 0, nil -} - -func NewBlueGreenIterator(tx *blueGreenTx, ait, bit roaring.ContainerIterator) *blueGreenIterator { - return &blueGreenIterator{ - tx: tx, - as: tx.as, - bs: tx.bs, - ait: ait, - bit: bit, - } -} - -type blueGreenIterator struct { - tx *blueGreenTx - as string - bs string - - ait roaring.ContainerIterator - bit roaring.ContainerIterator -} - -func (bgi *blueGreenIterator) Next() bool { - na := bgi.ait.Next() - nb := bgi.bit.Next() - if na != nb { - PanicOn(fmt.Sprintf("na=%v(%v) != nb(%v)=%v", na, bgi.as, bgi.bs, nb)) - } - return nb -} - -func (bgi *blueGreenIterator) Value() (uint64, *roaring.Container) { - ka, ca := bgi.ait.Value() - kb, cb := bgi.bit.Value() - - if !bgi.tx.o.blueGreenOff { - if ka != kb { - PanicOn(fmt.Sprintf("ka=%v != kb=%v", ka, kb)) - } - err := ca.BitwiseCompare(cb) - PanicOn(err) - } - return kb, cb -} -func (bgi *blueGreenIterator) Close() { - bgi.ait.Close() - bgi.bit.Close() -} - -// ForEach is read-only on the database, and so we only pass through to B. -// Avoids the side-effects of calling fn too many times, which can cause serious false alarms. -func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.ForEach(index, field, view, shard, fn) - -} - -// ForEachRange cannot change the database, and we also can't control -// the side effects of the fn() calls. So we only pass through to B, not A. -// No checker.see() is needed as well, because we are read-only. -func (c *blueGreenTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { - - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - - // calling fn will have side effects; can only call it the right number of times. - // so can't do this. - // errA := c.a.ForEachRange(index, field, view, shard, start, end, fn) - return c.b.ForEachRange(index, field, view, shard, start, end, fn) -} - -func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.Count(index, field, view, shard) - _, _ = a, errA - b, errB := c.b.Count(index, field, view, shard) - _, _ = b, errB - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return b, errB -} - -func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.Max(index, field, view, shard) - _, _ = a, errA - b, errB := c.b.Max(index, field, view, shard) - _, _ = b, errB - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return b, errB -} - -func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - amin, afound, errA := c.a.Min(index, field, view, shard) - _, _, _ = amin, afound, errA - bmin, bfound, errB := c.b.Min(index, field, view, shard) - _, _, _ = bmin, bfound, errB - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return bmin, bfound, errB -} - -func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - errA := c.a.UnionInPlace(index, field, view, shard, others...) - errB := c.b.UnionInPlace(index, field, view, shard, others...) - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return errB -} - -func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - c.Dump(c.short, shard) - AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.CountRange(index, field, view, shard, start, end) - b, errB := c.b.CountRange(index, field, view, shard, start, end) - - if !c.o.blueGreenOff { - if a != b { - PanicOn(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b)) - } - - compareErrors(errA, errB) - } - return b, errB -} - -func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see OffsetRange() on _sn_ %v, PanicOn '%v' at '%v'", c.Sn(), r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.OffsetRange(index, field, view, shard, offset, start, end) - b, errB := c.b.OffsetRange(index, field, view, shard, offset, start, end) - - if !c.o.blueGreenOff { - - err = roaringBitmapDiff(a, b) - if err != nil { - c.Dump(false, shard) - PanicOn(fmt.Errorf("on _sn_ %v OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) err: %v", c.Sn(), index, field, view, int(shard), offset, start, end, err)) - } - compareErrors(errA, errB) - } - - return b, errB -} - -func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - c.Dump(c.short, shard) - AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - - rcA, szA, errA := c.a.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) - rcB, szB, errB := c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - - // We are seeing Roaring vs Badger size differences on - // server/ test TestClusterResize_AddNode/ContinuousShards, - // so turn off the szA vs szB checks and MutliReaderB use. But keep them if we want to - // check RBF vs Badger for byte-for-byte compatiblity (we - // suspect the ops log or optimized bitmaps are accounting for the difference). - sizeMustMatch := false // !c.hasRoaring - if c.o.blueGreenOff { - sizeMustMatch = false - } - if sizeMustMatch { - if szA != szB { - PanicOn(fmt.Sprintf("szA(%v) = %v, but szB(%v) = %v; fragmentPathForRoaring='%v'", c.as, szA, c.bs, szB, fragmentPathForRoaring)) - } - return &MultiReaderB{a: rcA, b: rcB}, szB, errB - } else { - // one db won't get data if we do - //return &MultiReaderB{a: rcA, b: rcB, allowSizeVariation: true}, szB, errB - _, _ = szA, errA - rcA.Close() - return rcB, szB, errB - } -} - -func (c *blueGreenTx) Group() *TxGroup { - return c.b.Group() -} - -func (c *blueGreenTx) Options() Txo { - return c.b.Options() -} - -// Sn retreives the serial number of the Tx. -func (c *blueGreenTx) Sn() int64 { - asn := c.a.Sn() - bsn := c.b.Sn() - - if c.useSnA { - return asn - } - return bsn -} - -func (c *blueGreenTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { - return GenericApplyFilter(c, index, field, view, shard, ckey, filter) -} - -// MultiReaderB is returned by RoaringBitmapReader. It verifies -// that identical byte streams are read from its two members. -type MultiReaderB struct { - a io.ReadCloser - b io.ReadCloser - - allowSizeVariation bool -} - -// Read implements the standard io.Reader method. It panics -// if "a" and "b" have even one byte different in their reads. -func (m *MultiReaderB) Read(p []byte) (nB int, errB error) { - nB, errB = m.b.Read(p) - p2 := make([]byte, nB) - - // read (and discard after comparing for equality) the exact same amount from A. - - // ReadAtLeast reads from r into buf until it has read at least - // min bytes. It returns the number of bytes copied and an error - // if fewer bytes were read. The error is EOF only if no bytes - // were read. If an EOF happens after reading fewer than min bytes, - // ReadAtLeast returns ErrUnexpectedEOF. If min is greater than - // the length of buf, ReadAtLeast returns ErrShortBuffer. On - // return, n >= min if and only if err == nil. If r returns - // an error having read at least min bytes, the error is dropped. - nA, errA := io.ReadAtLeast(m.a, p2, nB) - - if !m.allowSizeVariation { - if errA == io.ErrUnexpectedEOF { - PanicOn(fmt.Sprintf("MultiReaderB got ErrUnexpectedEOF: read %v bytes from B, but could only read %v bytes for A", nB, nA)) - } - if nA != nB { - PanicOn(fmt.Sprintf("MultiReaderB read %v bytes from B, but could only read %v bytes for A", nB, nA)) - } - cmp := bytes.Compare(p[:nB], p2[:nB]) - if cmp != 0 { - PanicOn(fmt.Sprintf("MultiReaderB reads p and p2 (cmp= %v) differed.", cmp)) - } - } - return -} - -func (m *MultiReaderB) Close() error { - m.a.Close() - return m.b.Close() -} - -// blueGreenChecker is used -type blueGreenChecker struct { - visited map[string]map[string]map[string]map[uint64]struct{} - - c *blueGreenTx - - // lock mu when using visited. - // otherwise concurrent map writes on TestAPI_Import/RowIDColumnKey - mu sync.Mutex - - checkDone bool -} - -// see would mark a thing as seen. -func (b *blueGreenChecker) see(index, field, view string, shard uint64) { - // keep this next Printf. Useful to see the sequence of Tx operations. - //fmt.Printf("blueGreenTx.%v on index='%v' shard=%v\n", Caller(1), index, shard) - - if !b.c.o.Write { - return - } - - b.mu.Lock() - defer b.mu.Unlock() - - if b.visited == nil { - b.visited = make(map[string]map[string]map[string]map[uint64]struct{}) - } - var visitedIdx map[string]map[string]map[uint64]struct{} - var visitedField map[string]map[uint64]struct{} - var visitedView map[uint64]struct{} - - if visitedIdx = b.visited[index]; visitedIdx == nil { - visitedIdx = make(map[string]map[string]map[uint64]struct{}) - b.visited[index] = visitedIdx - } - if visitedField = visitedIdx[field]; visitedField == nil { - visitedField = make(map[string]map[uint64]struct{}) - visitedIdx[field] = visitedField - } - if visitedView = visitedField[view]; visitedView == nil { - visitedView = make(map[uint64]struct{}) - visitedField[view] = visitedView - } - visitedView[shard] = struct{}{} -} - -// seen reports the things it has seen, exactly once so -// that Rollback can be called after Commit without repeating -// the check. -func (b *blueGreenChecker) seen() map[string]map[string]map[string]map[uint64]struct{} { - return b.visited -} diff --git a/bluegreentx_test.go b/bluegreentx_test.go deleted file mode 100644 index 3a365800b..000000000 --- a/bluegreentx_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "context" - "io" - "io/ioutil" - "os" - "strings" - "testing" - - cryrand "crypto/rand" - - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck -) - -var _ = context.Background -var _ = os.Open -var _ = strings.Split - -func TestMultiReaderB(t *testing.T) { - // MultiReaderB should read identical chunks of bytes from both its "a" and "b" - // member io.Readers, else it should panic. This should hold for - // varying sizes of inputs. - - for n := 1 << 5; n < (1 << 18); n = n*2 - 13 { - src := io.LimitReader(cryrand.Reader, int64(n)) - - a := make([]byte, n) - nr := 0 - for nr < n { - na, err := src.Read(a) - PanicOn(err) - nr += na - } - if nr != n { - panic("short read") - } - - b := make([]byte, n) - copy(b, a) - if !bytes.Equal(a, b) { - panic("test prep failed") - } - - m := &MultiReaderB{ - a: ioutil.NopCloser(bytes.NewBuffer(a)), - b: ioutil.NopCloser(bytes.NewBuffer(b)), - } - - // should not trigger the internal panic of MultiReadB - ncp, err := io.Copy(ioutil.Discard, m) - PanicOn(err) - if ncp != int64(n) { - panic("short copy") - } - - for victim := 0; victim < n; victim += 7 { - - copy(b, a) - if victim%2 == 0 { - // corrupt b - b[victim] = (b[victim] + 1) % 255 - } else { - // corrupt a - a[victim] = (a[victim] + 1) % 255 - } - m = &MultiReaderB{ - a: ioutil.NopCloser(bytes.NewBuffer(a)), - b: ioutil.NopCloser(bytes.NewBuffer(b)), - } - helperShouldPanicOnCopy(m) - } - } -} - -func helperShouldPanicOnCopy(m *MultiReaderB) { - // differences in bytes read should be noticed - defer func() { - r := recover() - if r == nil { - panic("expected panic on byte difference but didn't see it") - } - }() - _, _ = io.Copy(ioutil.Discard, m) -} diff --git a/bolt.go b/bolt.go deleted file mode 100644 index 12435abef..000000000 --- a/bolt.go +++ /dev/null @@ -1,1612 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "math" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/molecula/featurebase/v2/hash" - "github.com/molecula/featurebase/v2/roaring" - "github.com/molecula/featurebase/v2/storage" - . "github.com/molecula/featurebase/v2/vprint" - - // On Bolt only, we still use the long txkey, because - // this allows Max() to work readily. - // - "github.com/molecula/featurebase/v2/short_txkey" - txkey "github.com/molecula/featurebase/v2/txkey" - "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" -) - -const isDebugRun = false - -// boltRegistrar facilitates shutdown -// of all the bolt databases started under -// tests. Its needed because most tests don't cleanup -// the *Index(es) they create. But we still -// want to shutdown boltDB goroutines -// after tests run. -// -// It also allows opening the same path twice to -// result in sharing the same open database handle, and -// thus the same transactional guarantees. -// -type boltRegistrar struct { - mu sync.Mutex - mp map[*BoltWrapper]bool - - path2db map[string]*BoltWrapper -} - -func (r *boltRegistrar) Size() int { - r.mu.Lock() - defer r.mu.Unlock() - nmp := len(r.mp) - npa := len(r.path2db) - if nmp != npa { - panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa)) - } - return nmp -} - -var globalBoltReg *boltRegistrar = newBoltTestRegistrar() - -var globalNextTxSnBolt int64 - -func newBoltTestRegistrar() *boltRegistrar { - - return &boltRegistrar{ - mp: make(map[*BoltWrapper]bool), - path2db: make(map[string]*BoltWrapper), - } -} - -// register each bolt created under tests, so we -// can clean them up. This is called by openBoltWrapper() while -// holding the r.mu.Lock, since it needs to atomically -// check the registry and make a new instance only -// if one does not exist for its path, and otherwise -// return the existing instance. -func (r *boltRegistrar) unprotectedRegister(w *BoltWrapper) { - r.mp[w] = true - r.path2db[w.path] = w -} - -// unregister removes w from r -func (r *boltRegistrar) unregister(w *BoltWrapper) { - r.mu.Lock() - delete(r.mp, w) - delete(r.path2db, w.path) - r.mu.Unlock() -} - -func DumpAllBolt() { - short := true - globalBoltReg.mu.Lock() - defer globalBoltReg.mu.Unlock() - for w := range globalBoltReg.mp { - AlwaysPrintf("this bolt path='%v' has: \n%v\n", w.path, w.StringifiedBoltKeys(nil, short)) - } -} - -// openBoltDB opens the database in the bpath directoy -// without deleting any prior content. Any BoltDB -// database directory will have the "-bolt" suffix. -// -// openBoltDB will check the registry and make a new instance only -// if one does not exist for its bpath. Otherwise it returns -// the existing instance. This insures only one boltDB -// per bpath in this pilosa node. -func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error) { - r.mu.Lock() - defer r.mu.Unlock() - w, ok := r.path2db[path] - if ok { - // creates the effect of having only one bolt open per pilosa node. - return w, nil - } - // otherwise, make a new bolt and store it in globalBoltReg - - dir := filepath.Dir(path) - if !DirExists(path) { - PanicOn(os.MkdirAll(dir, 0755)) - } - fsyncEnabled := true - if cfg != nil { - fsyncEnabled = cfg.FsyncEnabled - } - - db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !fsyncEnabled}) - if err != nil { - return nil, errors.Wrapf(err, fmt.Sprintf("open bolt path '%v'", path)) - } - - // docs on fsync from https://godoc.org/github.com/etcd-io/bbolt - // - // Setting the NoSync flag will cause the database to skip fsync() - // calls after each commit. This can be useful when bulk loading data - // into a database and you can restart the bulk load in the event of - // a system failure or database corruption. Do not set this flag for - // normal use. - // - // If the package global IgnoreNoSync constant is true, this value is - // ignored. See the comment on that constant for more details. - // - // THIS IS UNSAFE. PLEASE USE WITH CAUTION. - // NoSync bool - - // When true, skips syncing freelist to disk. This improves the database - // write performance under normal operation, but requires a full database - // re-sync during recovery. - // NoFreelistSync bool - - if cfg != nil && !cfg.FsyncEnabled { - db.NoSync = true - db.NoFreelistSync = true - } else { - // default to using fsync on bolt. - db.NoSync = false - db.NoFreelistSync = false - } - - err = db.Update(func(tx *bolt.Tx) (err error) { - _, err = tx.CreateBucketIfNotExists(bucketCT) - return - }) - if err != nil { - return nil, errors.Wrapf(err, fmt.Sprintf("create bolt bucket '%v' in path '%v'", string(bucketCT), path)) - } - - name := filepath.Base(path) - w = &BoltWrapper{ - name: name, - db: db, - reg: r, - path: path, - doAllocZero: doAllocZero, - openTx: make(map[*BoltTx]bool), - - DeleteEmptyContainer: true, - fsyncEnabled: cfg.FsyncEnabled, - } - r.unprotectedRegister(w) - - return w, nil -} - -func (w *BoltWrapper) Path() string { - return w.path -} - -func (w *BoltWrapper) HasData() (has bool, err error) { - - tx, err := w.NewTx(!writable, "", Txo{}) - if err != nil { - return false, errors.Wrap(err, "HasData NewTx") - } - defer tx.Rollback() - - bi := NewBoltIterator(tx.(*BoltTx), nil) - defer bi.Close() - - for bi.Next() { - return true, nil - } - return false, nil -} - -func (w *BoltWrapper) CleanupTx(tx Tx) { - // inlined into Rollback and Commit, so this is a no-op, just here to satisfy the interface. -} -func (w *BoltWrapper) CloseDB() error { - w.muDb.Lock() - defer w.muDb.Unlock() - w.closed = true - return w.db.Close() -} -func (w *BoltWrapper) OpenDB() error { - w.muDb.Lock() - defer w.muDb.Unlock() - db, err := bolt.Open(w.path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !w.fsyncEnabled}) - if err != nil { - return err - } - w.db = db - w.closed = false - return nil -} - -func (tx *BoltTx) IsDone() (done bool) { - return atomic.LoadInt64(&tx.unlocked) == 1 -} - -func (w *BoltWrapper) OpenListString() (r string) { - - list := w.listopen() - if len(list) == 0 { - return "" - } - for i, ltx := range list { - if ltx.o.Write { - r += fmt.Sprintf("[%v]write: _sn_ %v %v, \n", i, ltx.sn, ltx.o) - } else { - r += fmt.Sprintf("[%v]read : _sn_ %v %v, \n", i, ltx.sn, ltx.o) - } - } - return -} - -func (w *BoltWrapper) listopen() (slc []*BoltTx) { - w.muDb.Lock() - for v := range w.openTx { - slc = append(slc, v) - } - w.muDb.Unlock() - return -} - -func (w *BoltWrapper) OpenSnList() (slc []int64) { - w.muDb.Lock() - for v := range w.openTx { - slc = append(slc, v.sn) - } - w.muDb.Unlock() - return -} - -// DeleteIndex deletes all the containers associated with -// the named index from the bolt database. -func (w *BoltWrapper) DeleteIndex(indexName string) error { - - // We use the apostrophie rune `'` to locate the end of the - // index name in the key prefix, so we cannot allow indexNames - // themselves to contain apostrophies. - if strings.Contains(indexName, "/") { - return fmt.Errorf("error: bad indexName `%v` in BoltWrapper.DeleteIndex() call: indexName cannot contain '/'", indexName) - } - prefix := txkey.IndexOnlyPrefix(indexName) - return w.DeletePrefix(prefix) -} - -// statically confirm that BoltTx satisfies the Tx interface. -var _ Tx = (*BoltTx)(nil) - -// BoltWrapper provides the NewTx() method. -type BoltWrapper struct { - db *bolt.DB - - muDb sync.Mutex - - path string - name string - - // track our registrar for Close / goro leak reporting purposes. - reg *boltRegistrar - - // make BoltWrapper.Close() idempotent, avoiding panic on double Close() - closed bool - - // doAllocZero sets the corresponding flag on all new BoltTx. - // When doAllocZero is true, we zero out any data from bolt - // after transcation commit and rollback. This simulates - // what would happen if we were to use the mmap-ed data - // from bolt directly. Currently we copy by default for - // safety because otherwise TestAPI_ImportColumnAttrs sees - // corrupted data. - doAllocZero bool - - DeleteEmptyContainer bool - fsyncEnabled bool // for tracking whether our initial config wanted fsync on - - openTx map[*BoltTx]bool -} - -func (w *BoltWrapper) SetHolder(h *Holder) { - // don't need it at the moment - //w.h = h -} - -// NewTxWRITE lets us see in the callstack dumps where the WRITE tx are. -// Can't have more than one active write per database, so the -// 2nd one will block until the first finishes. -func (w *BoltWrapper) NewTxWRITE() (*bolt.Tx, error) { - boltTxn, err := w.db.Begin(true) - if err != nil { - if w.db == nil || w.IsClosed() { - return nil, fmt.Errorf("cannot call NewTxWRITE() on closed Bolt database: '%v'", err) - } - return nil, err - } - return boltTxn, nil -} - -// NewTxREAD lets us see in the callstack dumps where the READ tx are. -func (w *BoltWrapper) NewTxREAD() (*bolt.Tx, error) { - boltTxn, err := w.db.Begin(false) - if err != nil { - if w.db == nil || w.IsClosed() { - return nil, fmt.Errorf("cannot call NewTxREAD() on closed Bolt database: '%v'", err) - } - return nil, err - } - return boltTxn, nil -} - -// NewTx produces Bolt based transactions. If -// the transaction will modify data, then the write flag must be true. -// Read-only queries should set write to false, to allow more concurrency. -// Methods on a BoltTx are thread-safe, and can be called from -// different goroutines. -// -// initialIndexName is optional. It is set by the TxFactory from the Txo -// options provided at the Tx creation point. It allows us to recognize -// and isolate cross-index queries more quickly. It can always be empty "" -// but when set is highly useful for debugging. It has no impact -// on transaction behavior. -// -func (w *BoltWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { - - sn := atomic.AddInt64(&globalNextTxSnBolt, 1) - - ////vv("bolt new tx _sn_ %v; openTx='%v', stack \n%v", sn, w.OpenListString(), stack()) - ////vv("bolt new (write=%v, shard=%v) tx _sn_ %v; openTx='%v'", write, o.Shard, sn, w.OpenListString()) - - var boltTxn *bolt.Tx - if write { - // see the WRITE tx on the callstack. - boltTxn, err = w.NewTxWRITE() - if err != nil { - return nil, err - } - } else { - // see the READ tx on the callstack. - boltTxn, err = w.NewTxREAD() - if err != nil { - return nil, err - } - } - - ltx := &BoltTx{ - sn: sn, - write: write, - tx: boltTxn, - Db: w, - frag: o.Fragment, - doAllocZero: w.doAllocZero, - initialIndexName: initialIndexName, - DeleteEmptyContainer: w.DeleteEmptyContainer, - o: o, - gid: curGID(), - } - tx = ltx - - if isDebugRun { - w.muDb.Lock() - w.openTx[ltx] = true - w.muDb.Unlock() - } - return -} - -// Close shuts down the Bolt database. -func (w *BoltWrapper) Close() (err error) { - w.muDb.Lock() - defer w.muDb.Unlock() - if !w.closed { - if isDebugRun { - // complain if there are still Tx in flight, b/c otherwise we will see - // the somewhat mysterious 'panic: should not be in ReadSlot.free() with slot still owned by gid=107043; refCount=1' - if len(w.openTx) > 0 { - AlwaysPrintf("error: cannot close BoltWrapper with Tx still in flight.") - return - } - } - w.reg.unregister(w) - w.closed = true - return w.db.Close() - } - return nil -} - -func (w *BoltWrapper) IsClosed() (closed bool) { - w.muDb.Lock() - closed = w.closed - w.muDb.Unlock() - return -} - -// BoltTx wraps a bolt.Tx and provides the Tx interface -// method implementations. -// The methods on BoltTx are thread-safe, and can be called -// from different goroutines. -type BoltTx struct { - - // mu serializes bolt operations on this single txn instance. - mu sync.Mutex - sn int64 // serial number - - write bool - Db *BoltWrapper - tx *bolt.Tx - frag *fragment - - opcount int - - //initloc string // stack trace of where we were initially created. - - doAllocZero bool - - initialIndexName string - - DeleteEmptyContainer bool - - unlocked int64 - - o Txo - - // NewTx, write operations, Commit and/or Rollback must all take place on - // the same gid and it must the runtime.LockOSThreaded first. Verify - // that we are using the right goroutine in a debug build using the - // gid, stored here, used for NewTx(). - gid uint64 -} - -// sanity check that database is open. -func (tx *BoltTx) sanity() { - if tx.Db.IsClosed() { - panic("cannot operate on closed Bolt") - } -} - -func (tx *BoltTx) Group() *TxGroup { - return tx.o.Group -} - -func (tx *BoltTx) Type() string { - return BoltTxn -} - -func (tx *BoltTx) UseRowCache() bool { - return storage.EnableRowCache() -} - -// Pointer gives us a memory address for the underlying transaction for debugging. -// It is public because we use it in roaring to report invalid container memory access -// outside of a transaction. -func (tx *BoltTx) Pointer() string { - return fmt.Sprintf("%p", tx) -} - -// Rollback rolls back the transaction. -func (tx *BoltTx) Rollback() { - notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) - if !notDone { - return - } - ////vv("bolt rollback tx _sn_ %v; stack \n%v", tx.sn) // , stack()) - if isDebugRun { - tx.sanity() - tx.Db.muDb.Lock() - delete(tx.Db.openTx, tx) - tx.Db.muDb.Unlock() - } - - tx.mu.Lock() - defer tx.mu.Unlock() - - //tx.debugOnlyGidcheck() - _ = tx.tx.Rollback() // must hold tx.mu mutex lock - - tx.o.dbs.Cleanup(tx) -} - -// Commit commits the transaction to permanent storage. -// Commits can handle up to 100k updates to fragments -// at once, but not more. This is a BoltDB imposed limit. -func (tx *BoltTx) Commit() error { - notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) - if !notDone { - ////vv("Commit already done") - return nil - } - ////vv("bolt commit tx _sn_ %v; path = '%v'; stack \n%v", tx.sn, tx.Db.path, stack()) - //DumpAllBolt() - - if isDebugRun { - tx.sanity() - tx.Db.muDb.Lock() - delete(tx.Db.openTx, tx) - tx.Db.muDb.Unlock() - } - tx.mu.Lock() - defer tx.mu.Unlock() - - err := tx.tx.Commit() - PanicOn(err) - - tx.o.dbs.Cleanup(tx) - return err -} - -// Readonly returns true iff the BoltTx is read-only. -func (tx *BoltTx) Readonly() bool { - return !tx.write -} - -// RoaringBitmap returns the roaring.Bitmap for all bits in the fragment. -func (tx *BoltTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { - - return tx.OffsetRange(index, field, view, shard, 0, 0, LeftShifted16MaxContainerKey) -} - -// Container returns the requested roaring.Container, selected by fragment and ckey -func (tx *BoltTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) { - - // values returned from Get() are only valid while the transaction - // is open. If you need to use a value outside of the transaction then - // you must use copy() to copy it to another byte slice. - // BUT here we are already inside the Txn. - - bkey := txkey.Key(index, field, view, shard, ckey) - tx.mu.Lock() - - //tx.debugOnlyGidcheck() - - bkt := tx.tx.Bucket(bucketCT) - v := bkt.Get(bkey) - tx.mu.Unlock() - - if v == nil { - // not found - return nil, nil - } - n := len(v) - if n > 0 { - c = tx.toContainer(v[n-1], v[0:(n-1)]) - } - return -} - -var bucketCT = []byte("ct") - -// PutContainer stores rc under the specified fragment and container ckey. -func (tx *BoltTx) PutContainer(index, field, view string, shard uint64, ckey uint64, rc *roaring.Container) error { - - bkey := txkey.Key(index, field, view, shard, ckey) - var by []byte - - ct := roaring.ContainerType(rc) - - switch ct { - case roaring.ContainerArray: - by = fromArray16(roaring.AsArray(rc)) - case roaring.ContainerBitmap: - by = fromArray64(roaring.AsBitmap(rc)) - case roaring.ContainerRun: - by = fromInterval16(roaring.AsRuns(rc)) - case roaring.ContainerNil: - panic("wat? nil container is unexpected, no?!?") - default: - panic(fmt.Sprintf("unknown container type: %v", ct)) - } - tx.mu.Lock() - - bkt := tx.tx.Bucket(bucketCT) - err := bkt.Put(bkey, append(by, ct)) - ////vv("err on put bkey = '%v' was %v", string(bkey), err) - tx.mu.Unlock() - - return err -} - -// RemoveContainer deletes the container specified by the shard and container key ckey -func (tx *BoltTx) RemoveContainer(index, field, view string, shard uint64, ckey uint64) error { - bkey := txkey.Key(index, field, view, shard, ckey) - tx.mu.Lock() - //tx.debugOnlyGidcheck() - - bkt := tx.tx.Bucket(bucketCT) - err := bkt.Delete(bkey) - - tx.mu.Unlock() - return err -} - -// Add sets all the a bits hot in the specified fragment. -func (tx *BoltTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - return tx.addOrRemove(index, field, view, shard, false, a...) -} - -// Remove clears all the specified a bits in the chosen fragment. -func (tx *BoltTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - return tx.addOrRemove(index, field, view, shard, true, a...) -} - -func (tx *BoltTx) addOrRemove(index, field, view string, shard uint64, remove bool, a ...uint64) (changeCount int, err error) { - if len(a) == 0 { - return 0, nil - } - - // have to sort, b/c input is not always sorted. - sort.Slice(a, func(i, j int) bool { return a[i] < a[j] }) - - var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter. - var rc *roaring.Container - var hi uint64 - var lo uint16 - - for i, v := range a { - - hi, lo = highbits(v), lowbits(v) - if hi != lastHi { - // either first time through, or changed to a different container. - // do we need put the last updated container now? - if i > 0 { - // not first time through, write what we got. - if remove && (rc == nil || rc.N() == 0) { - err = tx.RemoveContainer(index, field, view, shard, lastHi) - PanicOn(err) - } else { - err = tx.PutContainer(index, field, view, shard, lastHi, rc) - PanicOn(err) - } - } - // get the next container - rc, err = tx.Container(index, field, view, shard, hi) - PanicOn(err) - } // else same container, keep adding bits to rct. - chng := false - // rc can be nil before, and nil after, in both Remove/Add below. - // The roaring container add() and remove() methods handle this. - if remove { - rc, chng = rc.Remove(lo) - } else { - rc, chng = rc.Add(lo) - } - if chng { - changeCount++ - } - lastHi = hi - } - // write the last updates. - if remove { - if rc == nil || rc.N() == 0 { - err = tx.RemoveContainer(index, field, view, shard, hi) - PanicOn(err) - } else { - err = tx.PutContainer(index, field, view, shard, hi, rc) - PanicOn(err) - } - } else { - if rc == nil || rc.N() == 0 { - panic("there should be no way to have an empty bitmap AFTER an Add() operation") - } - err = tx.PutContainer(index, field, view, shard, hi, rc) - PanicOn(err) - } - return -} - -// Contains returns exists true iff the bit chosen by key is -// hot (set to 1) in specified fragment. -func (tx *BoltTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { - - lo, hi := lowbits(key), highbits(key) - bkey := txkey.Key(index, field, view, shard, hi) - tx.mu.Lock() - - bkt := tx.tx.Bucket(bucketCT) - v := bkt.Get(bkey) - - tx.mu.Unlock() - if v == nil { - return false, nil - } - n := len(v) - if n > 0 { - c := tx.toContainer(v[n-1], v[0:(n-1)]) - exists = c.Contains(lo) - } - return exists, err -} - -// key is the container key for the first roaring Container -// roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will -// return the first container at or after key. found will be true if a -// container is found at key. -// -// BoltTx notes: We auto-stop at the end of this shard, not going beyond. -func (tx *BoltTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { - - // needle example: "idx:'i';fld:'f';vw:'v';shd:'00000000000000000000';key@00000000000000000000" - needle := txkey.Key(index, field, view, shard, firstRoaringContainerKey) - - // prefix example: "idx:'i';fld:'f';vw:'v';shard:'00000000000000000000';key@" - prefix := txkey.Prefix(index, field, view, shard) - - bi := NewBoltIterator(tx, prefix) - ok := bi.Seek(needle) - if !ok { - return bi, false, nil - } - - // have to compare b/c bolt might give us valid iterator - // that is past our needle if needle isn't present. - return bi, bytes.Equal(bi.lastKey, needle), nil -} - -func (tx *BoltTx) GetFieldSizeBytes(index, field string) (uint64, error) { - return 0, nil -} - -// BoltIterator is the iterator returned from a BoltTx.ContainerIterator() call. -// It implements the roaring.ContainerIterator interface. -type BoltIterator struct { - tx *BoltTx - cur *bolt.Cursor - - prefix []byte - seekto []byte - - // seen counts how many Next() calls we have seen. - // It is used to match roaring.ContainerIterator semantics. - // Also useful for testing. - seen int - - lastKey []byte - lastVal []byte // *roaring.Container - lastOK bool - lastConsumed bool -} - -// NewBoltIterator creates an iterator on tx that will -// only return boltKeys that start with prefix. -func NewBoltIterator(tx *BoltTx, prefix []byte) (bi *BoltIterator) { - - tx.mu.Lock() - - bkt := tx.tx.Bucket(bucketCT) - cur := bkt.Cursor() - tx.mu.Unlock() - - bi = &BoltIterator{ - tx: tx, - cur: cur, - prefix: prefix, - } - - return -} - -// Close tells the database and transaction that the user is done -// with the iterator. -func (bi *BoltIterator) Close() { - // no-op -} - -// Valid returns false if there are no more values in the iterator's range. -func (bi *BoltIterator) Valid() bool { - return bi.lastOK -} - -// Seek allows the iterator to start at needle instead of the global begining. -func (bi *BoltIterator) Seek(needle []byte) (ok bool) { - bi.tx.mu.Lock() - defer bi.tx.mu.Unlock() - - bi.seen++ // if ommited, red TestBolt_ContainerIterator_empty_iteration_loop() - - k, v := bi.cur.Seek(needle) - ////vv("seek to needle '%v' gives key '%v'", string(needle), txkey.ToString(k)) - - if len(k) == 0 { - // not found, no keys after needle. - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - if len(bi.prefix) > 0 { - ok = bytes.HasPrefix(k, bi.prefix) - if !ok { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - } - - bi.lastKey = k - bi.lastVal = v - bi.lastOK = true - bi.lastConsumed = false - - return true -} - -func (bi *BoltIterator) ValidForPrefix(prefix []byte) bool { - if !bi.lastOK { - return false - } - if len(bi.prefix) == 0 { - return true - } - return bytes.HasPrefix(bi.lastKey, bi.prefix) -} - -func (bi *BoltIterator) String() (r string) { - return fmt.Sprintf("BoltIterator{prefix: '%v', seekto: '%v', seen:%v, lastKey:'%v', lastOK:%v, lastConsumed:%v}", string(bi.prefix), string(bi.seekto), bi.seen, string(bi.lastKey), bi.lastOK, bi.lastConsumed) -} - -// Next advances the iterator. -func (bi *BoltIterator) Next() (ok bool) { - //vv("Next; bi.prefix='%v'", string(bi.prefix)) - if bi.lastOK && !bi.lastConsumed { - //vv("lastOk and not consumed, returning wo doing anything") - bi.seen++ - bi.lastConsumed = true - if len(bi.lastVal) == 0 { - panic("bi.lastVal should not have len 0 if lastOK true") - } - return true - } - - var k, v []byte - if bi.seen == 0 { - if len(bi.prefix) == 0 { - bi.tx.mu.Lock() - k, v = bi.cur.First() - bi.tx.mu.Unlock() - } else { - found := bi.Seek(bi.prefix) - // increments bi.seen for us. - if !found { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - // ready to go - return true - } - } - - bi.seen++ -skipEmpty: - if bi.seen > 1 { - bi.tx.mu.Lock() - k, v = bi.cur.Next() - bi.tx.mu.Unlock() - } - - if len(k) == 0 { - // no more - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - if len(bi.prefix) > 0 { - ok = bytes.HasPrefix(k, bi.prefix) - if !ok { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - } - bi.lastKey = k - bi.lastVal = v - if len(v) == 0 { - // actually under !tx.DeleteEmptyContainer, we can have empty containers! - goto skipEmpty - } - bi.lastOK = true - bi.lastConsumed = true - - return true -} - -// Value retrieves what is pointed at currently by the iterator. -func (bi *BoltIterator) Value() (containerKey uint64, c *roaring.Container) { - if !bi.lastOK { - panic("bi.cur not valid") - } - containerKey = txkey.KeyExtractContainerKey(bi.lastKey) - - v := bi.lastVal - n := len(v) - if n > 0 { - c = bi.tx.toContainer(v[n-1], v[0:(n-1)]) - } else { - panic("v should not be empty!") - } - return -} - -// boltFinder implements roaring.IteratorFinder. -// It is used by BoltTx.ForEach() -type boltFinder struct { - tx *BoltTx - index string - field string - view string - shard uint64 - needClose []Closer -} - -// FindIterator lets boltFinder implement the roaring.FindIterator interface. -func (bf *boltFinder) FindIterator(seek uint64) (roaring.ContainerIterator, bool) { - a, found, err := bf.tx.ContainerIterator(bf.index, bf.field, bf.view, bf.shard, seek) - PanicOn(err) - bf.needClose = append(bf.needClose, a) - return a, found -} - -// Close closes all bf.needClose listed Closers. -func (bf *boltFinder) Close() { - for _, i := range bf.needClose { - i.Close() - } -} - -// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE -// the transaction Commits or Rollsback. -func (tx *BoltTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { - - bf := &boltFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} - itr := roaring.NewIterator(bf) - return itr -} - -// ForEach applies fn to each bitmap in the fragment. -func (tx *BoltTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { - - itr := tx.NewTxIterator(index, field, view, shard) - defer itr.Close() - - // Seek can create many container iterators, thus bf.Close() needClose list. - itr.Seek(0) - // v is the bit we are operating on. - for v, eof := itr.Next(); !eof; v, eof = itr.Next() { - if err := fn(v); err != nil { - return err - } - } - return nil -} - -// ForEachRange applies fn on the selected range of bits on the chosen fragment. -func (tx *BoltTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { - - itr := tx.NewTxIterator(index, field, view, shard) - defer itr.Close() - - itr.Seek(start) - - // v is the bit we are operating on. - for v, eof := itr.Next(); !eof && v < end; v, eof = itr.Next() { - if err := fn(v); err != nil { - return err - } - } - return nil -} - -// Count operates on the full bitmap level, so it sums over all the containers -// in the bitmap. -func (tx *BoltTx) Count(index, field, view string, shard uint64) (uint64, error) { - - a, found, err := tx.ContainerIterator(index, field, view, shard, 0) - PanicOn(err) - defer a.Close() - if !found { - return 0, nil - } - result := int32(0) - for a.Next() { - ckey, cont := a.Value() - _ = ckey - result += cont.N() - } - return uint64(result), nil -} - -// Max is the maximum bit-value in your bitmap. -// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does. -func (tx *BoltTx) Max(index, field, view string, shard uint64) (uint64, error) { - - prefix := txkey.Prefix(index, field, view, shard) - seekto := txkey.Prefix(index, field, view, shard+1) - - bkt := tx.tx.Bucket(bucketCT) - cur := bkt.Cursor() - - var k, v []byte - k, _ = cur.Seek(seekto) - if k == nil { - // we have nothing >= seekto, but we might have stuff before it, and we'll wrap backwards. - k, v = cur.Prev() - if k == nil { - // empty database - return 0, nil - } - } else { - // we found something >= seekto, so backup by 1. - k, v = cur.Prev() - if k == nil { - // nothing before seekto - return 0, nil - } - } - - // have something, are we in [prefix, seekto) ? - cmp := bytes.Compare(k, prefix) - if cmp >= 0 { - // good, got max in k, v - } else { - return 0, nil // nothing in [prefix, seekto). - } - - n := len(v) - if n == 0 { - return 0, nil - } - - hb := txkey.KeyExtractContainerKey(k) - rc := tx.toContainer(v[n-1], v[0:(n-1)]) - - lb := rc.Max() - return hb<<16 | uint64(lb), nil -} - -// Min returns the smallest bit set in the fragment. If no bit is hot, -// the second return argument is false. -func (tx *BoltTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { - - // Seek can create many container iterators, thus the bf.Close() needClose list. - bf := &boltFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} - defer bf.Close() - itr := roaring.NewIterator(bf) - - itr.Seek(0) - - // v is the bit we are operating on. - v, eof := itr.Next() - if eof { - return 0, false, nil - } - return v, true, nil -} - -// UnionInPlace unions all the others Bitmaps into a new Bitmap, and then writes it to the -// specified fragment. -func (tx *BoltTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - PanicOn(err) - - rbm.UnionInPlace(others...) - // iterate over the containers that changed within rbm, and write them back to disk. - - it, found := rbm.Containers.Iterator(0) - _ = found // don't care about the value of found, because first containerKey might be > 0 - - for it.Next() { - containerKey, rc := it.Value() - - // TODO: only write the changed ones back, as optimization? - // Compare to ImportRoaringBits. - err := tx.PutContainer(index, field, view, shard, containerKey, rc) - PanicOn(err) - } - return nil -} - -// CountRange returns the count of hot bits in the start, end range on the fragment. -// roaring.countRange counts the number of bits set between [start, end). -func (tx *BoltTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { - - if start >= end { - return 0, nil - } - - skey := highbits(start) - ekey := highbits(end) - - citer, found, err := tx.ContainerIterator(index, field, view, shard, skey) - _ = found - PanicOn(err) - - defer citer.Close() - - // If range is entirely in one container then just count that range. - if skey == ekey { - citer.Next() - _, c := citer.Value() - return uint64(c.CountRange(int32(lowbits(start)), int32(lowbits(end)))), nil - } - - for citer.Next() { - k, c := citer.Value() - if k < skey { - citer.Close() - panic(fmt.Sprintf("should be impossible for k(%v) to be less than skey(%v). tx p=%p", k, skey, tx)) - } - - // k > ekey handles the case when start > end and where start and end - // are in different containers. Same container case is already handled above. - if k > ekey { - break - } - if k == skey { - n += uint64(c.CountRange(int32(lowbits(start)), roaring.MaxContainerVal+1)) - continue - } - if k < ekey { - n += uint64(c.N()) - continue - } - if k == ekey { - n += uint64(c.CountRange(0, int32(lowbits(end)))) - break - } - } - - return n, nil -} - -// OffsetRange creates a new roaring.Bitmap to return in other. For all the -// hot bits in [start, endx) of the chosen fragment, it stores -// them into other but with offset added to their bit position. -// The primary client is doing this, using ShardWidth, already; see -// fragment.rowFromStorage() in fragment.go. For example: -// -// data, err := tx.OffsetRange(f.index, f.field, f.view, f.shard, -// -// f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) -// ^ offset ^ start ^ endx -// -// The start and endx arguments are container keys that have been shifted left by 16 bits; -// their highbits() will be taken to determine the actual container keys. This -// is done to conform to the roaring.OffsetRange() argument convention. -// -func (tx *BoltTx) OffsetRange(index, field, view string, shard, offset, start, endx uint64) (other *roaring.Bitmap, err error) { - ////vv("top of BoltTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v)", index, field, view, int(shard), int(offset), int(start), int(endx)) - //defer func() { - ////vv("returning from BoltTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) other returning is: '%#v' stack=\n%v", index, field, view, int(shard), int(offset), int(start), int(endx), asInts(other.Slice()), stack()) - //}() - - // roaring does these three checks in its OffsetRange - if lowbits(offset) != 0 { - panic("offset must not contain low bits") - } - if lowbits(start) != 0 { - panic("range start must not contain low bits") - } - if lowbits(endx) != 0 { - panic("range end must not contain low bits") - } - - other = roaring.NewSliceBitmap() - off := highbits(offset) - hi0, hi1 := highbits(start), highbits(endx) - - needle := txkey.Key(index, field, view, shard, hi0) - prefix := txkey.Prefix(index, field, view, shard) - - it := NewBoltIterator(tx, prefix) - defer it.Close() - it.Seek(needle) - for ; it.ValidForPrefix(prefix); it.Next() { - bkey := it.lastKey - k := txkey.KeyExtractContainerKey(bkey) - - // >= hi1 is correct b/c endx cannot have any lowbits set. - if uint64(k) >= hi1 { - break - } - destCkey := off + (k - hi0) - - v := it.lastVal - n := len(v) - if n == 0 { - continue - } - c := tx.toContainer(v[n-1], v[0:(n-1)]) - other.Containers.Put(destCkey, c.Freeze()) - } - return other, nil -} - -// IncrementOpN increments the tx opcount by changedN -func (tx *BoltTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - tx.opcount += changedN -} - -// ImportRoaringBits handles deletes by setting clear=true. -// rowSet[rowID] returns the number of bit changed on that rowID. -func (tx *BoltTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { - - n := itr.Len() - if n == 0 { - return - } - rowSet = make(map[uint64]int) - - var currRow uint64 - - var oldC *roaring.Container - for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { - if rowSize != 0 { - currRow = itrKey / rowSize - } - nsynth := int(synthC.N()) - if nsynth == 0 { - continue - } - // INVAR: nsynth > 0 - - oldC, err = tx.Container(index, field, view, shard, itrKey) - PanicOn(err) - if err != nil { - return - } - - if oldC == nil || oldC.N() == 0 { - // no container at the itrKey in bolt (or all zero container). - if clear { - // changed of 0 and empty rowSet is perfect, no need to change the defaults. - continue - } else { - - changed += nsynth - rowSet[currRow] += nsynth - - err = tx.PutContainer(index, field, view, shard, itrKey, synthC) - if err != nil { - return - } - continue - } - } - - if clear { - existN := oldC.N() // number of bits set in the old container - newC := oldC.Difference(synthC) - - // update rowSet and changes - if newC.N() == existN { - // INVAR: do changed need adjusting? nope. same bit count, - // so no change could have happened. - continue - } else { - changes := int(existN - newC.N()) - changed += changes - rowSet[currRow] -= changes - - if tx.DeleteEmptyContainer && newC.N() == 0 { - err = tx.RemoveContainer(index, field, view, shard, itrKey) - if err != nil { - return - } - continue - } - err = tx.PutContainer(index, field, view, shard, itrKey, newC) - if err != nil { - return - } - continue - } - } else { - // setting bits - - existN := oldC.N() - if existN == roaring.MaxContainerVal+1 { - // completely full container already, set will do nothing. so changed of 0 default is perfect. - continue - } - if existN == 0 { - // can nsynth be zero? No, because of the continue/invariant above where nsynth > 0 - changed += nsynth - rowSet[currRow] += nsynth - err = tx.PutContainer(index, field, view, shard, itrKey, synthC) - if err != nil { - return - } - continue - } - - newC := roaring.Union(oldC, synthC) // UnionInPlace was giving us crashes on overly large containers. - - if roaring.ContainerType(newC) == roaring.ContainerBitmap { - newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it. - } - if newC.N() != existN { - changes := int(newC.N() - existN) - changed += changes - rowSet[currRow] += changes - - err = tx.PutContainer(index, field, view, shard, itrKey, newC) - if err != nil { - PanicOn(err) - return - } - continue - } - } - } - return -} - -func (tx *BoltTx) toContainer(typ byte, v []byte) (r *roaring.Container) { - - //tx.debugOnlyGidcheck() - - if len(v) == 0 { - return nil - } - - var w []byte - useRowCache := tx.UseRowCache() - if tx.doAllocZero || useRowCache { - // Do electric fence-inspired bad-memory read detection. - // - // The v []byte lives in BoltDB's memory-mapped vlog-file, - // and Bolt will recycle it after tx ends with rollback or commit. - // - // Problem is, at least some operations were not respecting transaction boundaries. - // This technique helped us find them. The rowCache was an example. - // - // See the global const DetectMemAccessPastTx - // at the top of txfactory.go to activate/deactivate this. - // - // Seebs suggested this nice variation: we could use individual mmaps for these - // copies, which would be unusable in production, but workable for testing, and then unmap them, - // which would get us probable segfaults on future accesses to them. - // - // The go runtime also has an -efence flag which may be similarly useful if really pressed. - // - w = make([]byte, len(v)) - copy(w, v) - } else { - w = v - } - return ToContainer(typ, w) -} - -func ToContainer(typ byte, w []byte) (c *roaring.Container) { - switch typ { - case roaring.ContainerArray: - c = roaring.NewContainerArray(toArray16(w)) - case roaring.ContainerBitmap: - c = roaring.NewContainerBitmap(-1, toArray64(w)) - case roaring.ContainerRun: - c = roaring.NewContainerRun(toInterval16(w)) - default: - panic(fmt.Sprintf("unknown container: %v", typ)) - } - c.SetMapped(true) - return c -} - -// StringifiedBoltKeys returns a string with all the container -// keys available in bolt. -func (w *BoltWrapper) StringifiedBoltKeys(optionalUseThisTx Tx, short bool) (r string) { - if optionalUseThisTx == nil { - tx, _ := w.NewTx(!writable, "", Txo{}) - defer tx.Rollback() - r = stringifiedBoltKeysTx(tx.(*BoltTx), short) - return - } - - btx, ok := optionalUseThisTx.(*BoltTx) - if !ok { - return fmt.Sprintf("", optionalUseThisTx) - } - r = stringifiedBoltKeysTx(btx, short) - return -} - -// countBitsSet returns the number of bits set (or "hot") in -// the roaring container value found by the txkey.Key() -// formatted bkey. -func (tx *BoltTx) countBitsSet(bkey []byte) (n int) { - - //tx.debugOnlyGidcheck() - - bkt := tx.tx.Bucket(bucketCT) - v := bkt.Get(bkey) - - if v == nil { - // some queries bkey may not be present! don't panic. - return 0 - } - - n = len(v) - if n > 0 { - rc := tx.toContainer(v[n-1], v[0:(n-1)]) - n = int(rc.N()) - } - return -} - -func (tx *BoltTx) Dump(short bool, shard uint64) { - fmt.Printf("%v\n", stringifiedBoltKeysTx(tx, short)) -} - -func (tx *BoltTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []short_txkey.FieldView, err error) { - bkt := tx.tx.Bucket(bucketCT) - err = bkt.ForEach(func(bkey, v []byte) error { - fv := txkey.FieldViewFromFullKey(bkey) - var shortFV short_txkey.FieldView - shortFV.Field = fv.Field - shortFV.View = fv.View - fvs = append(fvs, shortFV) - return nil - }) - return -} - -// stringifiedBoltKeysTx reports all the bolt keys and a -// corresponding blake3 hash viewable by txn within the entire -// bolt database. -// It also reports how many bits are hot in the roaring container -// (how many bits are set, or 1 rather than 0). -// -// By convention, we must return the empty string if there -// are no keys present. The tests use this to confirm -// an empty database. -func stringifiedBoltKeysTx(tx *BoltTx, short bool) (r string) { - - r = "allkeys:[\n" - it := NewBoltIterator(tx, nil) - defer it.Close() - any := false - for it.Next() { - any = true - - bkey := it.lastKey - key := txkey.ToString(bkey) - ckey := txkey.KeyExtractContainerKey(bkey) - h := "" - srbm := "" - v := it.lastVal - n := len(v) - if n == 0 { - panic("should not have empty v here") - } - h = hash.Blake3sum16(v[0:(n - 1)]) - ct := tx.toContainer(v[n-1], v[0:(n-1)]) - cts := roaring.NewSliceContainers() - cts.Put(ckey, ct) - rbm := &roaring.Bitmap{Containers: cts} - srbm = BitmapAsString(rbm) - - r += fmt.Sprintf("%v -> %v (%v hot)\n", key, h, tx.countBitsSet(bkey)) - if !short { - r += " ......." + srbm + "\n" - } - } - r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) - - if !any { - return "" - } - return "bolt-" + r -} - -func (w *BoltWrapper) DeleteDBPath(dbs *DBShard) (err error) { - path := dbs.pathForType(boltTxn) - err = os.RemoveAll(path) - if err != nil { - return errors.Wrap(err, "DeleteDBPath") - } - return -} - -func (w *BoltWrapper) DeleteField(index, field, fieldPath string) (err error) { - - // TODO(jea) cleanup: I think this fieldPath delete just goes away now. - // remove this commented stuff once we are sure. - // - // under blue-green roaring_bolt, the directory will not be found, b/c roaring will have - // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: - // "If the path does not exist, RemoveAll returns nil (no error)" - - err = os.RemoveAll(fieldPath) - if err != nil { - return errors.Wrap(err, "removing directory") - } - prefix := txkey.FieldPrefix(index, field) - return w.DeletePrefix(prefix) -} - -func (w *BoltWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { - prefix := txkey.Prefix(index, field, view, shard) - return w.DeletePrefix(prefix) -} - -func (w *BoltWrapper) DeletePrefix(prefix []byte) error { - - tx, _ := w.NewTx(writable, w.name, Txo{}) - - // NewTx will grab these, so don't lock until after it. - w.muDb.Lock() - - bi := NewBoltIterator(tx.(*BoltTx), prefix) - - for bi.Next() { - //vv("deleting next in cur") - err := bi.cur.Delete() - if err != nil { - w.muDb.Unlock() - panic(err) - } - } - bi.Close() - - // Commit will grab the w.muDb lock, so we must release it first. - w.muDb.Unlock() - - err := tx.Commit() - PanicOn(err) - - return nil -} - -func (tx *BoltTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") - } - var buf bytes.Buffer - sz, err = rbm.WriteTo(&buf) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") - } - return ioutil.NopCloser(&buf), sz, err -} - -func (tx *BoltTx) Options() Txo { - return tx.o -} - -// Sn retreives the serial number of the Tx. -func (tx *BoltTx) Sn() int64 { - return tx.sn -} - -func (c *BoltTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { - return GenericApplyFilter(c, index, field, view, shard, ckey, filter) -} diff --git a/bolt_test.go b/bolt_test.go deleted file mode 100644 index 1dc03b7d0..000000000 --- a/bolt_test.go +++ /dev/null @@ -1,1270 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "fmt" - "math" - "os" - "testing" - - "github.com/molecula/featurebase/v2/roaring" - "github.com/molecula/featurebase/v2/storage" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck -) - -// helpers, each runs their own new txn, and commits if a change/delete -// was made. The txn is rolled back if it is just viewing the data. - -func BoltMustHaveBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, bitvalue uint64) { - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue '%v' was NOT SET!!!", bitvalue)) - } - - tx.Rollback() -} - -func BoltMustNotHaveBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, bitvalue uint64) { - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if exists { - panic(fmt.Sprintf("ARG bitvalue '%v' WAS SET but should not have been.!!!", bitvalue)) - } - tx.Rollback() -} - -func BoltMustSetBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - // add a bit - changed, err := tx.Add(index, field, view, shard, putme) - if changed != 1 { - panic("should have 1 bit changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic("ARG putme was NOT SET!!!") - } - PanicOn(tx.Commit()) -} - -func BoltMustDeleteBitvalueContainer(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - hi := highbits(putme) - PanicOn(tx.RemoveContainer(index, field, view, shard, hi)) - PanicOn(tx.Commit()) -} - -func BoltMustDeleteBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - _, err := tx.Remove(index, field, view, shard, putme) - PanicOn(err) - PanicOn(tx.Commit()) -} - -func mustOpenEmptyBoltWrapper(path string) (w *BoltWrapper, cleaner func()) { - var err error - fn := path - PanicOn(os.RemoveAll(fn)) - ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, &storage.Config{FsyncEnabled: false}) - PanicOn(err) - w = ww.(*BoltWrapper) - - // verify it is empty - allkeys := w.StringifiedBoltKeys(nil, false) - if allkeys != "" { - panic(fmt.Sprintf("freshly created database was not empty! had keys:'%v'", allkeys)) - } - - return w, func() { - w.Close() - PanicOn(os.RemoveAll(fn)) - } -} - -// end of helper utilities -////////////////////////// - -////////////////////////// -// begin Tx method tests - -func TestBolt_DeleteFragment(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_DeleteFragment") - defer clean() - defer dbwrap.Close() - index, field, shard := "i", "f", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16} - views := []string{"v1", "v2"} - for _, view := range views { - for _, v := range bits { - changed, err := tx.Add(index, field, view, shard, v) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - } - } - - for _, view := range views { - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - } - } - err := tx.Commit() - PanicOn(err) - - // end of setup - - victim := "v1" - survivor := "v2" - err = dbwrap.DeleteFragment(index, field, victim, shard, nil) - PanicOn(err) - - tx, _ = dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - for _, view := range views { - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if view == survivor { - if !exists { - panic(fmt.Sprintf("ARG survivor died : bit %v", v)) - } - } else if view == victim { // victim, should have been deleted - if exists { - panic(fmt.Sprintf("ARG victim lived : bit %v", v)) - } - } - } - } -} - -func TestBolt_Max_on_many_containers(t *testing.T) { - path := "TestBolt_Max_on_many_containers" - dbwrap, clean := mustOpenEmptyBoltWrapper(path) - - defer clean() - defer dbwrap.Close() - index, field, view := "i", "f", "v" - - // 099 - // 101 - // 199 - // 300 - // 399 - // - // find max in [300,400) and get 399 - // find max in [000,100) and get 099 - // find max in [100,200) and get 199 - // find max in [400,500) and get nothing back - // find max in [200,300) and get nothing back - - shards := []int{99, 101, 199, 300, 399} - - for _, sh := range shards { - shard := uint64(sh) - for _, pm := range shards { - putme := uint64(pm) - if putme > shard { - continue - } - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - } - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - for _, shard := range shards { - max, err := tx.Max(index, field, view, uint64(shard)) - PanicOn(err) - if max != uint64(shard) { - panic(fmt.Sprintf("expected max (%v) to be == shard = %v", max, shard)) - } - } - - // check for not found - max, err := tx.Max(index, field, view, uint64(200)) - PanicOn(err) - if max != 0 { - panic("expected not found to give 0 max back with nil err") - } - max, err = tx.Max(index, field, view, uint64(400)) - PanicOn(err) - if max != 0 { - panic("expected not found to give 0 max back with nil err") - } - -} - -// and the rest - -func TestBolt_SetBitmap(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_SetBitmap") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - bitvalue := uint64(0) - changed, err := tx.Add(index, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - - err = tx.Commit() - PanicOn(err) - - // - // commited, so should be visible outside the txn - // - - tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) - exists, err = tx2.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!! on tx2") - } - - n, err := tx2.Count(index, field, view, shard) - PanicOn(err) - if n != 1 { - panic(fmt.Sprintf("should have Count 1; instead n = %v", n)) - } - tx2.Rollback() -} - -func TestBolt_OffsetRange(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_OffsetRange") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - bitvalue := uint64(1 << 20) - changed, err := tx.Add(index, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - bitvalue2 := uint64(1<<20 + 1) - changed, err = tx.Add(index, field, view, shard, bitvalue2) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - exists, err = tx.Contains(index, field, view, shard, bitvalue2) - PanicOn(err) - if !exists { - panic("ARG bitvalue2 was NOT SET!!!") - } - - err = tx.Commit() - PanicOn(err) - - offset := uint64(0 << 20) - start := uint64(0 << 16) - endx := bitvalue + 1<<16 - - tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) - rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx) - PanicOn(err) - tx2.Rollback() - - // should see our 1M value - s2 := BitmapAsString(rbm2) - expect2 := "c(1048576, 1048577)" - if s2 != expect2 { - panic(fmt.Sprintf("s2='%v', but expected '%v'", s2, expect2)) - } - - // now offset by 2M - offset = uint64(2 << 20) - tx3, _ := dbwrap.NewTx(!writable, index, Txo{}) - rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx) - PanicOn(err) - tx3.Rollback() - - //expect to see 3M == 3145728 - s3 := BitmapAsString(rbm3) - expect3 := "c(3145728, 3145729)" - - if s3 != expect3 { - panic(fmt.Sprintf("s3='%v', but expected '%v'", s3, expect3)) - } -} - -func TestBolt_Count_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Count_on_many_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{0, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - n, err := tx.Count(index, field, view, shard) - PanicOn(err) - if int(n) != len(putmeValues) { - panic(fmt.Sprintf("expected Count of %v but got n=%v", len(putmeValues), n)) - } -} - -func TestBolt_Count_dense_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Count_dense_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - expected := 0 - for i := uint64(0); i < (1<<16)+2; i += 2 { - changed, err := tx.Add(index, field, view, shard, i) - PanicOn(err) - if changed <= 0 { - panic("wat? should have changed") - } - expected++ - } - defer tx.Rollback() - - n, err := tx.Count(index, field, view, shard) - PanicOn(err) - if int(n) != expected { - panic(fmt.Sprintf("expected Count of %v but got n=%v", expected, n)) - } -} - -func TestBolt_ContainerIterator_on_empty(t *testing.T) { - // iterate on empty container, should not find anything. - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - bitvalue := uint64(0) - citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue) - PanicOn(err) - defer citer.Close() - if found { - panic("should not have found anything") - } - PanicOn(err) -} - -func TestBolt_ContainerIterator_on_one_bit(t *testing.T) { - // set one bit, iterate. - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator_on_one_bit") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - bitvalue := uint64(42) - - // add a bit - changed, err := tx.Add(index, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - - // same Tx, continues in use. - - citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(bitvalue)) - if !found { - panic("ContainerIterator did not find the 42 bit") - } - PanicOn(err) - defer citer.Close() - - loopCount := 0 - for citer.Next() { - key, container := citer.Value() - if key != 0 { - panic("42 should have had key 0") - } - if container == nil { - panic("container was nil") - } - if container.N() != 1 { - panic("put a bit in, but size of container was not 1") - } - if !container.Contains(lowbits(bitvalue)) { - panic("container did not have our bitvalue!") - } - loopCount++ - if loopCount > 0 { // happier linter - break - } - } - if loopCount != 1 { - panic("ContainerIterator did not return a citer that scanned our set bit") - } -} - -func TestBolt_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator_on_one_bit_fail_to_find") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - putme := uint64(1<<16) + 3 // in the key:1 container - searchme := putme + 1 - - // add a bit - changed, err := tx.Add(index, field, view, shard, putme) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic("ARG putme was NOT SET!!!") - } - - // same Tx, continues in use. - - citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(searchme)) - if !found { - panic("ContainerIterator did not find the searchme") - } - defer citer.Close() - loopCount := 0 - for citer.Next() { - key, container := citer.Value() - if key != 1 { - panic("Containeriterator searching for highbits(searchme) should not have had a bit") - } - if container == nil { - panic("container was nil") - } - if container.N() != 1 { - panic("put a bit in, but size of container was not 1") - } - if container.Contains(lowbits(searchme)) { - panic("container should have putme but not our searchme!") - } - loopCount++ - // only want first pass. keep linter happy by avoiding raw break - if loopCount > 0 { - break - } - } - PanicOn(err) -} - -func TestBolt_ContainerIterator_empty_iteration_loop(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator_empty_iteration_loop") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - putme := uint64(1<<16) + 3 // in the key:1 container - searchme := uint64(1 << 17) // in the next container, key:2 - - // add a bit - changed, err := tx.Add(index, field, view, shard, putme) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic("ARG putme was NOT SET!!!") - } - - // same Tx, continues in use. - - citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(searchme)) - PanicOn(err) - if found { - panic("ContainerIterator found the searchme, when it should not have") - } - defer citer.Close() - if citer.Next() { - panic("expected no looping, 0 iterations, b/c started searchme past our data in putme") - } - - // expect to see a blow up from the citer.Value() call, verify that we do. - func() { - defer func() { - r := recover() - if r == nil { - panic("expected a panic from citer.Value() in this case") - } - }() - citer.Value() // should panic - }() - -} - -func TestBolt_ForEach_on_one_bit(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ForEach_on_one_bit") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - bitvalue := uint64(42) - - // add a bit - changed, err := tx.Add(index, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - - // same Tx, continues in use. - count := 0 - err = tx.ForEach(index, field, view, shard, func(v uint64) error { - if v != bitvalue { - panic(fmt.Sprintf("bitvalue corrupt got %v want %v", v, bitvalue)) - } - count += 1 - return nil - }) - PanicOn(err) - if count != 1 { - panic(fmt.Sprintf("Expected single iteration got %v ", count)) - } -} - -func TestBolt_RemoveContainer_one_bit_test(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_RemoveContainer_one_bit_test") - defer clean() - defer dbwrap.Close() - - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{0, 13, 77, 1511} - - for _, putme := range putmeValues { - - // a) delete of whole container in a seperate txn. Commit should establish the deletion. - - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // b) deletion + rollback on the txn should restore the deleted bit - - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // delete, but rollback instead of commit - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - hi := highbits(putme) - PanicOn(tx.RemoveContainer(index, field, view, shard, hi)) - tx.Rollback() - - // verify that the rollback undid the deletion. - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // c) within one Tx, after delete it should be gone as viewed within the txn. - tx, _ = dbwrap.NewTx(writable, index, Txo{}) - hi = highbits(putme) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme)) - } - - PanicOn(tx.RemoveContainer(index, field, view, shard, hi)) - - exists, err = tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if exists { - panic(fmt.Sprintf("ARG putme '%v' was SET even after RemoveContiner in this txn.", putme)) - } - - tx.Rollback() - - // verify that the rollback undid the deletion. - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - // leave with clean slate - BoltMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) - } -} - -func TestBolt_Remove_one_bit_test(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Remove_one_bit_test") - defer clean() - defer dbwrap.Close() - - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{0, 13, 77, 1511} - - for _, putme := range putmeValues { - - // a) delete of whole container in a seperate txn. Commit should establish the deletion. - - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustDeleteBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // b) deletion + rollback on the txn should restore the deleted bit - - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // delete, but rollback instead of commit - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - hi, lo := highbits(putme), lowbits(putme) - _, _ = hi, lo - _, err := tx.Remove(index, field, view, shard, hi) - PanicOn(err) - tx.Rollback() - - // verify that the rollback undid the deletion. - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // c) within one Tx, after delete it should be gone as viewed within the txn. - tx, _ = dbwrap.NewTx(writable, index, Txo{}) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme)) - } - - mustRemove(tx.Remove(index, field, view, shard, putme)) - - exists, err = tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if exists { - panic(fmt.Sprintf("ARG putme '%v' was SET even after Remove in this txn.", putme)) - } - - tx.Rollback() - - // verify that the rollback undid the deletion. - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - // leave with clean slate - BoltMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) - } -} - -func TestBolt_Min_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Min_on_many_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - // verify no containers flag works - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - min, containersExist, err := tx.Min(index, field, view, shard) - _ = min - PanicOn(err) - if containersExist { - panic("no containers should exist") - } - tx.Rollback() - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ = dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - min, containersExist, err = tx.Min(index, field, view, shard) - PanicOn(err) - if !containersExist { - panic("containers should exist") - } - expected := putmeValues[0] - if min != expected { - panic(fmt.Sprintf("expected Min() of %v but got min=%v", expected, min)) - } -} - -func TestBolt_CountRange_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_CountRange_on_many_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - // verify no containers flag works - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64) - PanicOn(err) - if n != 0 { - panic("no containers should exist") - } - tx.Rollback() - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ = dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64) - PanicOn(err) - if n == 0 { - panic("containers should exist") - } - expected := uint64(len(putmeValues)) - if n != expected { - panic(fmt.Sprintf("expected CountRange() of %v but got n=%v", expected, n)) - } -} - -func TestBolt_CountRange_middle_container(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_CountRange_middle_container") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - // pick out just the middle container with the 1 bit set on it. - n, err := tx.CountRange(index, field, view, shard, 4, (2<<16)+1) - PanicOn(err) - if n != 1 { - panic("middle 1 bit container should exist") - } -} - -func TestBolt_CountRange_many_middle_container(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_CountRange_many_middle_container") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - // get them all - n, err := tx.CountRange(index, field, view, shard, 0, (4<<16)+1) - PanicOn(err) - if n != 3 { - panic("count should have been all 3 bits") - } -} - -func TestBolt_UnionInPlace(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_UnionInPlace") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{3, 2 << 16} - - others := roaring.NewBitmap() - others2 := roaring.NewBitmap() - others3 := roaring.NewBitmap() - // populate others with putmeValues +1 into others - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) - n, err := tx2.Count(index, field, view, shard) - PanicOn(err) - if n != 2 { - panic("should have 2 bits set") - } - tx2.Rollback() - - for _, putme := range putmeValues { - mustAddR(others.Add(putme)) // should not change count, b/c putme already in the rbm - mustAddR(others.Add(putme + 1)) - mustAddR(others2.Add(putme + 2)) - } - mustAddR(others3.Add(4 << 16)) // outside the 2<<16 container - - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - err = tx.UnionInPlace(index, field, view, shard, others, others2, others3) - PanicOn(err) - - // end game, check we got the union. - rbm, err := tx.RoaringBitmap(index, field, view, shard) - PanicOn(err) - n = rbm.Count() - if n != 7 { - panic("should have a total 3 + 3 +1 = 7 bits set on the containers") - } -} - -func TestBolt_RoaringBitmap(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_RoaringBitmap") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - expected := uint64(3) - putme := expected - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - PanicOn(err) - - slc := rbm.Slice() - if slc[0] != uint64(expected) { - panic(fmt.Sprintf("should have gotten %v back", expected)) - } -} - -func TestBolt_ImportRoaringBits(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ImportRoaringBits") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - tx.(*BoltTx).DeleteEmptyContainer = true // traditional lmdb Tx behavior, but not Roaring. - - //bitvalue := uint64(42) - - // get some roaring bits, get an itr RoaringIterator from them - rowSize := uint64(0) - //bits := []uint64{0} - bits := []uint64{0, 2, 5, 1<<16 + 1, 2 << 16} - data := getTestBitmapAsRawRoaring(bits...) - itr, err := roaring.NewRoaringIterator(data) - PanicOn(err) - clear := false - logme := false - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != len(bits) { - panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now test the union in place with the same set gives no change. - - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != 0 { - panic(fmt.Sprintf("should have not changed any bits on the second import, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now test the clear path - clear = true - - for _, v := range bits { - // clear 1 bit at a time - data := getTestBitmapAsRawRoaring(v) - itr, err := roaring.NewRoaringIterator(data) - PanicOn(err) - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != 1 { - panic(fmt.Sprintf("should have changed 1 bit: '%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - PanicOn(err) - } - n, err := tx.Count(index, field, view, shard) - PanicOn(err) - if n != 0 { - panic(fmt.Sprintf("n = %v not zero so the clearbits didn't happen!", n)) - } - allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) - - // should have no keys - if allkeys != "" { - panic("bolt should have no keys now") - } -} - -func TestBolt_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ImportRoaringBits_set_nonoverlapping_bits") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - // get some roaring bits, get an itr RoaringIterator from them - rowSize := uint64(0) - //bits := []uint64{0} - bits := []uint64{0, 2, 1 << 16, 1<<16 + 2} - data := getTestBitmapAsRawRoaring(bits...) - itr, err := roaring.NewRoaringIterator(data) - PanicOn(err) - - bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16} - data2 := getTestBitmapAsRawRoaring(bits2...) - itr2, err := roaring.NewRoaringIterator(data2) - PanicOn(err) - - clear := false - logme := false - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != len(bits) { - panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now import the 2nd, overlapping set and set them. - - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil) - _ = rowSet - if changed != 4 { - panic(fmt.Sprintf("should have changed 2 bits: the 1 and the 3, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - PanicOn(err) -} - -func TestBolt_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ImportRoaringBits_clear_nonoverlapping_bits") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - // get some roaring bits, get an itr RoaringIterator from them - rowSize := uint64(0) - //bits := []uint64{0} - bits := []uint64{0, 2, 1 << 16, 1<<16 + 2} //, 5, 1<<16 + 1, 2 << 16} - data := getTestBitmapAsRawRoaring(bits...) - itr, err := roaring.NewRoaringIterator(data) - PanicOn(err) - - bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16} - data2 := getTestBitmapAsRawRoaring(bits2...) - itr2, err := roaring.NewRoaringIterator(data2) - PanicOn(err) - - clear := false - logme := false - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != len(bits) { - panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now import the 2nd overlapping set and clear them. - clear = true - - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil) - _ = rowSet - if changed != 2 { - panic(fmt.Sprintf("should have changed 1 bit: the 2, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - PanicOn(err) - - n, err := tx.Count(index, field, view, shard) - PanicOn(err) - if n != 2 { // just the 0 and the 1<<16 bits should be left set. - panic(fmt.Sprintf("n = %v not 2 so the clearbits didn't happen!", n)) - } - -} - -func TestBolt_DeleteIndex(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_DeleteIndex") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - bitvalue := uint64(777) - bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16} - for _, v := range bits { - changed, err := tx.Add(index, field, view, shard, v) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - } - - index2 := "i2" // should not be deleted, even though it shares a prefix with 'i' - changed, err := tx.Add(index2, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - } - exists, err := tx.Contains(index2, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!! on index2") - } - err = tx.Commit() - PanicOn(err) - - // end of setup - err = dbwrap.DeleteIndex(index) - PanicOn(err) - - tx, _ = dbwrap.NewTx(!writable, index2, Txo{}) - defer tx.Rollback() - exists, err = tx.Contains(index2, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2)) - } - - for _, v := range bits { - exists, err = tx.Contains(index, field, view, shard, v) - PanicOn(err) - if exists { - allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) - panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) - } - } -} - -func TestBolt_DeleteIndex_over100k(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_DeleteIndex_over100k") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - bitvalue := uint64(777) - limit := uint64(100002) // default batch size in DeleteIndex is 100k keys per delete transaction. - //limit := uint64(101) - for v := uint64(1); v < limit; v++ { - // shift by << 16 to get into a different shard - changed, err := tx.Add(index, field, view, shard, v<<16) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - if v%100000 == 0 { - PanicOn(tx.Commit()) - tx, _ = dbwrap.NewTx(writable, index, Txo{}) - } - } - - index2 := "i2" // should not be deleted, even though it shares a prefix with 'i' - changed, err := tx.Add(index2, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - err = tx.Commit() - PanicOn(err) - - // end of setup - err = dbwrap.DeleteIndex(index) - PanicOn(err) - - tx, _ = dbwrap.NewTx(!writable, index2, Txo{}) - defer tx.Rollback() - exists, err := tx.Contains(index2, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2)) - } - - for v := uint64(0); v < limit; v++ { - exists, err = tx.Contains(index, field, view, shard, v<<16) - PanicOn(err) - if exists { - allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) - panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) - } - } -} - -func TestBolt_HasData(t *testing.T) { - - db, clean := mustOpenEmptyBoltWrapper("TestBolt_SliceOfShards") - defer clean() - defer db.Close() - - // HasData should start out false. - hasAnything, err := db.HasData() - if err != nil { - t.Fatal(err) - } - if hasAnything { - t.Fatalf("HasData reported existing data on an empty database") - } - - // check that HasData sees a committed record. - - index, field, view, shard, putme := "i", "f", "v", uint64(123), uint64(42) - BoltMustSetBitvalue(db, index, field, view, shard, putme) - - // HasData(false) should now report data - hasAnything, err = db.HasData() - if err != nil { - t.Fatal(err) - } - if !hasAnything { - t.Fatalf("HasData() reported no data on a database that has bits written to it") - } -} diff --git a/catcher.go b/catcher.go index b807b8ea9..92ce5b2b9 100644 --- a/catcher.go +++ b/catcher.go @@ -15,9 +15,6 @@ package pilosa import ( - "fmt" - "io" - "github.com/molecula/featurebase/v2/roaring" txkey "github.com/molecula/featurebase/v2/short_txkey" . "github.com/molecula/featurebase/v2/vprint" @@ -42,40 +39,18 @@ func init() { var _ Tx = (*catcherTx)(nil) -func (c *catcherTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - c.b.IncrementOpN(index, field, view, shard, changedN) -} - func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { return c.b.NewTxIterator(index, field, view, shard) } -func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { +func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { defer func() { if r := recover(); r != nil { AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, Stack()) PanicOn(r) } }() - return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) -} - -func (c *catcherTx) Dump(short bool, shard uint64) { - c.b.Dump(short, shard) -} - -func (c *catcherTx) Readonly() bool { - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Readonly() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.Readonly() -} - -func (tx *catcherTx) Pointer() string { - return fmt.Sprintf("%p", tx) + return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize) } func (c *catcherTx) Rollback() { @@ -143,14 +118,6 @@ func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key return c.b.RemoveContainer(index, field, view, shard, key) } -func (c *catcherTx) UseRowCache() bool { - return c.b.UseRowCache() -} - -func (c *catcherTx) IsDone() bool { - return c.b.IsDone() -} - func (c *catcherTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { defer func() { @@ -250,17 +217,6 @@ func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, return c.b.Min(index, field, view, shard) } -func (c *catcherTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.UnionInPlace(index, field, view, shard, others...) -} - func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { defer func() { @@ -283,33 +239,10 @@ func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, return c.b.OffsetRange(index, field, view, shard, offset, start, end) } -func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) -} - func (c *catcherTx) Type() string { return c.b.Type() } -func (c *catcherTx) Group() *TxGroup { - return c.b.Group() -} - -func (c *catcherTx) Options() Txo { - return c.b.Options() -} - -// Sn retreives the serial number of the Tx. -func (c *catcherTx) Sn() int64 { - return c.b.Sn() -} - func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { return GenericApplyFilter(c, index, field, view, shard, ckey, filter) } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index cc55ae9aa..3d204ae92 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -17,7 +17,6 @@ package pilosa import ( "fmt" "math/rand" - "net" "reflect" "strings" "testing" @@ -30,57 +29,8 @@ import ( "github.com/molecula/featurebase/v2/testhook" "github.com/molecula/featurebase/v2/topology" . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck - "github.com/pkg/errors" ) -// GlobalPortMap avoids many races and port conflicts when setting -// up ports for test clusters. Used for tests only. -var globalPortMap *GlobalPortMapper - -func init() { - globalPortMap = NewGlobalPortMapper(300) -} - -// GlobalPortMapper maintains a pool of available ports by -// holding them open until GetPort() is called. -type GlobalPortMapper struct { - availPorts map[int]net.Listener -} - -// reserve n ports -func NewGlobalPortMapper(n int) (pm *GlobalPortMapper) { - - pm = &GlobalPortMapper{ - availPorts: make(map[int]net.Listener), - } - for i := 0; i < n; i++ { - lsn, err := net.Listen("tcp", ":0") - if err != nil { - panic(errors.Wrap(err, "trying to listen on ephemeral port")) - } - r := lsn.Addr() - port := r.(*net.TCPAddr).Port - pm.availPorts[port] = lsn - } - return -} - -func (pm *GlobalPortMapper) GetPort() (port int, err error) { - for port, lsn := range pm.availPorts { - lsn.Close() - return port, nil - } - return -1, fmt.Errorf("no more ports available") -} - -func (pm *GlobalPortMapper) MustGetPort() int { - port, err := pm.GetPort() - if err != nil { - panic(err) - } - return port -} - // Ensure that fragCombos creates the correct fragment mapping. func TestFragCombos(t *testing.T) { uri0, err := pnet.NewURIFromAddress("host0") diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go index 8b5b62d6c..627d56ef2 100644 --- a/cmd/roaring-migrate/main.go +++ b/cmd/roaring-migrate/main.go @@ -310,7 +310,7 @@ func Migrate(dataDir, backupPath string) error { return err } key := string(txkey.Prefix(index, field, view, shard)) - _, _, err = tx.ImportRoaringBits(key, itr, clear, log, rowSize, nil) + _, _, err = tx.ImportRoaringBits(key, itr, clear, log, rowSize) if err != nil { tx.Rollback() return err diff --git a/ctl/server.go b/ctl/server.go index 6c4405135..71c26bb3c 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -94,7 +94,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // over-ride. // TODO: the comment above was carried over from the PILOSA_TXSRC flag, but // we should confirm that this still applies. - flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, fmt.Sprintf("transaction/storage to use: one of roaring, rbf, bolt, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc. The default is: %v. The env var PILOSA_STORAGE_BACKEND is over-ridden by --storage.backend option on the command line.", storage.DefaultBackend)) + flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, fmt.Sprintf("transaction/storage to use: one of roaring or rbf. The default is: %v. The env var PILOSA_STORAGE_BACKEND is over-ridden by --storage.backend option on the command line.", storage.DefaultBackend)) flags.BoolVar(&srv.Config.Storage.FsyncEnabled, "storage.fsync", true, "enable fsync fully safe flush-to-disk") // RowcacheOn diff --git a/dbshard.go b/dbshard.go index a86cc4ab7..15bd41915 100644 --- a/dbshard.go +++ b/dbshard.go @@ -57,12 +57,10 @@ type DBIndex struct { type DBWrapper interface { NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) - DeleteDBPath(dbs *DBShard) error Close() error DeleteFragment(index, field, view string, shard uint64, frag interface{}) error DeleteField(index, field, fieldPath string) error OpenListString() string - OpenSnList() (sns []int64) Path() string HasData() (has bool, err error) SetHolder(h *Holder) @@ -82,134 +80,52 @@ type DBShard struct { Shard uint64 Open bool - // With RWMutex, the blue-green Tx can start and commit - // atomically. - mut sync.RWMutex - - types []txtype - stypes []string + typ txtype + styp string hasRoaring bool // if either of the types is roaringTxn - W []DBWrapper + W DBWrapper ParentDBIndex *DBIndex idx *Index per *DBPerShard - useOpenList int - closed bool - - isBlueGreen bool + closed bool } func (dbs *DBShard) DeleteFragment(index, field, view string, shard uint64, frag interface{}) (err error) { - for _, w := range dbs.W { - err = w.DeleteFragment(index, field, view, shard, frag) - if err != nil { - return err - } + if index != dbs.Index { + return fmt.Errorf("DeleteFragment called on DBShard for %q with index %q", dbs.Index, index) } - return + if shard != dbs.Shard { + return fmt.Errorf("DeleteFragment called on DBShard for %d with shard %d", dbs.Shard, shard) + } + return dbs.W.DeleteFragment(index, field, view, shard, frag) } func (dbs *DBShard) DeleteFieldFromStore(index, field, fieldPath string) (err error) { - for _, w := range dbs.W { - err = w.DeleteField(index, field, fieldPath) - if err != nil { - return err - } + if index != dbs.Index { + return fmt.Errorf("DeleteFieldFromStore called on DBShard for %q with index %q", dbs.Index, index) } - return + return dbs.W.DeleteField(index, field, fieldPath) } func (dbs *DBShard) Close() (err error) { - for _, w := range dbs.W { - err = w.Close() - if err != nil { - return err - } - } dbs.closed = true - return -} - -// Cleanup must be called at every commit/rollback of a Tx, in -// order to release the read-write mutex that guarantees a single -// writer at a time. Each tx must take care to call cleanup() -// exactly once. examples: -// tx.o.dbs.Cleanup(tx) -// tx.Options().dbs.Cleanup(tx) -// -func (dbs *DBShard) Cleanup(tx Tx) { - if dbs == nil { - return // some tests are using Tx only, no dbs available. - } - //vv("gid %v top of DBShard %v Cleanup for tx.Sn = %v; dbs=%p; is 2nd: %v; type='%v'; dbs.stypes='%#v'", curGID(), dbs.Shard, tx.Sn(), dbs, tx.Type() == dbs.stypes[1], tx.Type(), dbs.stypes) - if !dbs.hasRoaring { - if dbs.isBlueGreen { - // only release on the 2nd Tx's cleanup - if tx.Type() == dbs.stypes[1] { - if tx.Readonly() { - dbs.mut.RUnlock() - //vv("gid %v released read-lock on shard %v", curGID(), dbs.Shard) - } else { - dbs.mut.Unlock() - //vv("gid %v released write-lock on shard %v", curGID(), dbs.Shard) - } - } - } - } + return dbs.W.Close() } func (dbs *DBShard) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { - - if dbs.isBlueGreen { - // enforce only one writer at a time. The dbs.mut is held until - // the Tx finishes. This makes the two Tx in the blue-green Tx atomic. - if !dbs.hasRoaring { - if write { - //vv("shard %v about to write lock by gid %v; stack =\n%v", dbs.Shard, curGID(), stack()) - dbs.mut.Lock() - //vv("shard %v was write locked by gid %v; stack =\n%v", dbs.Shard, curGID(), stack()) - } else { - //vv("shard %v about to be read locked by gid %v; stack=\n%v", dbs.Shard, curGID(), stack()) - dbs.mut.RLock() - //vv("shard %v was read locked by gid %v; stack=\n%v", dbs.Shard, curGID(), stack()) - } - } + if initialIndexName != dbs.Index { + return nil, fmt.Errorf("NewTx called on DBShard for %q with index %q", dbs.Index, initialIndexName) } if o.dbs != dbs { - PanicOn(fmt.Sprintf("TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs)) + return nil, fmt.Errorf("dbs mismatch: TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs) } if o.Shard != dbs.Shard { - PanicOn(fmt.Sprintf("shard disagreement! o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard))) + return nil, fmt.Errorf("shard disagreement: o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard)) } - var txns []Tx - - for _, w := range dbs.W { - tx, err = w.NewTx(write, initialIndexName, o) - if err != nil { - return nil, err - } - txns = append(txns, tx) - } - if len(txns) == 1 { - return - } - // blue green - tx, err = dbs.per.txf.newBlueGreenTx(txns[0], txns[1], o.Index, o), nil - //vv("dbshard returning blue-green tx sn %v", tx.Sn()) - return -} - -func (dbs *DBShard) DeleteDBPath() (err error) { - for _, w := range dbs.W { - err = w.DeleteDBPath(dbs) - if err != nil { - return err - } - } - return + return dbs.W.NewTx(write, initialIndexName, o) } type flatkey struct { @@ -228,34 +144,26 @@ type DBPerShard struct { // Easily see how many we have. Flatmap map[flatkey]*DBShard - types []txtype + typ txtype hasRoaring bool txf *TxFactory holder *Holder - // which of our types is not-roaring, since - // roaring doesn't keep a list of open Tx sn. - // or default to the 2nd. - useOpenList int - // cache the shards per index to avoid excessive - // directory scans of the index directory. Keep per - // txtype to allow blue-green migrate open to be fast too. + // directory scans of the index directory. // Keep it up-to-date as we add shards to avoid doing // a filesystem rescan on new shard creation. // - // txtype -> index -> *shardSet - index2shards map[txtype]map[string]*shardSet - - isBlueGreen bool + // index -> *shardSet + index2shards map[string]*shardSet StorageConfig *storage.Config RBFConfig *rbfcfg.Config } -func newIndex2Shards() (r map[txtype]map[string]*shardSet) { - r = make(map[txtype]map[string]*shardSet) +func newIndex2Shards() (r map[string]*shardSet) { + r = make(map[string]*shardSet) return } @@ -350,51 +258,6 @@ func newShardSetFromMap(m map[uint64]bool) *shardSet { } } -// HasData returns true if the database has at least one key. -// For roaring it returns true if we a fragment stored. -// The `which` argument is the index into the per.W slice. 0 for blue, 1 for green. -// If you pass 1, be sure you have a blue-green configuration. -func (per *DBPerShard) HasData(which int) (hasData bool, err error) { - // has to aggregate across all available DBShard for each index and shard. - - if per.types[which] == roaringTxn { - return per.RoaringHasData() // this needs to be made accurate - } - - for _, v := range per.Flatmap { - hasData, err = v.W[which].HasData() - if err != nil { - return - } - if hasData { - return - } - } - return -} - -func (per *DBPerShard) RoaringHasData() (bool, error) { - idxs := per.holder.Indexes() - const requireData = true - for _, idx := range idxs { - shards, err := per.TypedDBPerShardGetShardsForIndex(roaringTxn, idx, "", requireData) - if err != nil { - return false, err - } - if len(shards) > 0 { - return true, nil - } - } - return false, nil -} - -func (per *DBPerShard) ListOpenString() (r string) { - for _, v := range per.Flatmap { - r += v.HolderPath + " -> " + v.W[per.useOpenList].OpenListString() + "\n" - } - return -} - func (per *DBPerShard) LoadExistingDBs() (err error) { idxs := per.holder.Indexes() @@ -414,37 +277,24 @@ func (per *DBPerShard) LoadExistingDBs() (err error) { return } -func (txf *TxFactory) NewDBPerShard(types []txtype, holderDir string, holder *Holder) (d *DBPerShard) { +func (txf *TxFactory) NewDBPerShard(typ txtype, holderDir string, holder *Holder) (d *DBPerShard) { if holder.cfg == nil || holder.cfg.RBFConfig == nil || holder.cfg.StorageConfig == nil { PanicOn("must have holder.cfg.RBFConfig and holder.cfg.StorageConfig set here") } - useOpenList := 0 hasRoaring := false - if types[0] == roaringTxn { + if typ == roaringTxn { hasRoaring = true } - if len(types) == 2 { - // blue-green, avoid the empty roaring Tx open list. - // Prefer B's open list if neither is roaring. - if types[0] == roaringTxn || types[1] != roaringTxn { - useOpenList = 1 - } - if types[1] == roaringTxn { - hasRoaring = true - } - } d = &DBPerShard{ - types: types, + typ: typ, HolderDir: holderDir, holder: holder, dbh: NewDBHolder(), Flatmap: make(map[flatkey]*DBShard), txf: txf, - useOpenList: useOpenList, hasRoaring: hasRoaring, - isBlueGreen: len(types) > 1, index2shards: newIndex2Shards(), StorageConfig: holder.cfg.StorageConfig, RBFConfig: holder.cfg.RBFConfig, @@ -469,14 +319,12 @@ func (per *DBPerShard) DeleteIndex(index string) (err error) { if err != nil { return errors.Wrap(err, "DBPerShard.DeleteIndex dbs.Close()") } - for _, ty := range per.types { - path := dbs.pathForType(ty) - err = os.RemoveAll(path) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("DBPerShard.DeleteIndex os.RemoveAll('%v')", path)) - } - delete(per.index2shards[ty], index) + path := dbs.pathForType(per.typ) + err = os.RemoveAll(path) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("DBPerShard.DeleteIndex os.RemoveAll('%v')", path)) } + delete(per.index2shards, index) } // allow the index to be created again anew. @@ -502,10 +350,8 @@ func (per *DBPerShard) DeleteFieldFromStore(index, field, fieldPath string) (err return nil } for _, dbs := range dbi.Shard { - for _, w := range dbs.W { - if e := w.DeleteField(index, field, fieldPath); e != nil && err == nil { - err = errors.Wrap(e, "DeleteFieldFromStore()") - } + if e := dbs.W.DeleteField(index, field, fieldPath); e != nil && err == nil { + err = errors.Wrap(e, "DeleteFieldFromStore()") } } return err @@ -521,46 +367,6 @@ func (per *DBPerShard) DeleteFragment(index, field, view string, shard uint64, f return dbs.DeleteFragment(index, field, view, shard, frag) } -func (dbs *DBShard) DumpAll() { - short := false - fmt.Printf("\n============= begin DumpAll dbs=%p index='%v', shard=%v ========\n", dbs, dbs.Index, int(dbs.Shard)) - for i, ty := range dbs.types { - _ = i - tx, err := dbs.W[i].NewTx(!writable, "", Txo{Index: dbs.idx}) - PanicOn(err) - defer tx.Rollback() - fmt.Printf("\n============= dumping dbs.W[%v] %v ========\n", i, ty) - tx.Dump(short, dbs.Shard) - - switch ty { - case roaringTxn: - case rbfTxn: - case boltTxn: - default: - PanicOn(fmt.Sprintf("unknown txtyp: '%v'", ty)) - } - } - fmt.Printf("\n============= end of DumpAll index='%v', shard=%v ========\n", dbs.Index, int(dbs.Shard)) -} - -func (per *DBPerShard) DumpAll() { - per.Mu.Lock() - defer per.Mu.Unlock() - - found1 := false - for _, dbi := range per.dbh.Index { - for _, dbs := range dbi.Shard { - if dbs.Open { - found1 = true - dbs.DumpAll() - } - } - } - if !found1 { - AlwaysPrintf("DBPerShard.DumpAll() sees no databases. dir='%v'", per.HolderDir) - } -} - // if you know the shard, you can use this // pathForType and prefixForType must be kept in sync! func (dbs *DBShard) pathForType(ty txtype) string { @@ -570,11 +376,6 @@ func (dbs *DBShard) pathForType(ty txtype) string { // is a no-op anyhow. so doesn't need to be correct atm. path := dbs.HolderPath + sep + dbs.Index + sep + backendsDir + sep + ty.DirectoryName() + sep + fmt.Sprintf("shard.%04v", dbs.Shard) - if ty == boltTxn { - // special case: - // bolt doesn't use a directory like the others, just a direct path. - path += sep + "bolt.db" - } return path } @@ -593,23 +394,13 @@ var ErrNoData = fmt.Errorf("no data") // // Caller must hold per.Mu.Lock() already. func (per *DBPerShard) updateIndex2ShardCacheWithNewShard(dbs *DBShard) { - - for _, ty := range dbs.types { - mapIndex2shardSet, ok := per.index2shards[ty] - if !ok { - mapIndex2shardSet = make(map[string]*shardSet) - per.index2shards[ty] = mapIndex2shardSet - } - // INVAR: mapIndex2shardSet is good, but may be an empty map - - shardset, ok := mapIndex2shardSet[dbs.Index] - if !ok { - shardset = newShardSet() - mapIndex2shardSet[dbs.Index] = shardset - } - // INVAR: shardset is present, not nil; a map that can be added to. - shardset.add(dbs.Shard) + shardset, ok := per.index2shards[dbs.Index] + if !ok { + shardset = newShardSet() + per.index2shards[dbs.Index] = shardset } + // INVAR: shardset is present, not nil; a map that can be added to. + shardset.add(dbs.Shard) } func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *DBShard, err error) { @@ -629,76 +420,47 @@ func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *In } dbs, ok = dbi.Shard[shard] if dbs != nil && dbs.closed { - if len(per.types) == 1 && per.types[0] == roaringTxn { - // roaring txn are nil/fake anyway. Don't freak out. - } else { - PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.types[0]='%v'; len(per.types)=%v", dbs, per.types[0], len(per.types))) + // roaring txn are nil/fake anyway. Don't freak out. + if per.typ != roaringTxn { + PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.typ='%v'", dbs, per.typ)) } } if !ok { dbs = &DBShard{ - types: per.types, + typ: per.typ, ParentDBIndex: dbi, Index: index, Shard: shard, HolderPath: per.HolderDir, idx: idx, per: per, - useOpenList: per.useOpenList, hasRoaring: per.hasRoaring, - isBlueGreen: len(per.types) > 1, } - dbs.stypes = make([]string, len(per.types)) - for i, ty := range per.types { - dbs.stypes[i] = ty.String() - } - + dbs.styp = per.typ.String() dbi.Shard[shard] = dbs per.updateIndex2ShardCacheWithNewShard(dbs) } if !dbs.Open { var registry DBRegistry - for _, ty := range dbs.types { - switch ty { - case roaringTxn: - registry = globalRoaringReg - case rbfTxn: - registry = globalRbfDBReg - registry.(*rbfDBRegistrar).SetRBFConfig(per.RBFConfig) - case boltTxn: - registry = globalBoltReg - default: - PanicOn(fmt.Sprintf("unknown txtyp: '%v'", ty)) - } - path := dbs.pathForType(ty) - w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx, per.StorageConfig) - PanicOn(err) - h := idx.Holder() - w.SetHolder(h) - dbs.Open = true - if w != nil && len(dbs.W) == 0 { - per.Flatmap[flatkey{index: index, shard: shard}] = dbs - } - dbs.W = append(dbs.W, w) + switch dbs.typ { + case roaringTxn: + registry = globalRoaringReg + case rbfTxn: + registry = globalRbfDBReg + registry.(*rbfDBRegistrar).SetRBFConfig(per.RBFConfig) + default: + PanicOn(fmt.Sprintf("unknown txtyp: '%v'", dbs.typ)) } + path := dbs.pathForType(dbs.typ) + w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx, per.StorageConfig) + PanicOn(err) + h := idx.Holder() + w.SetHolder(h) + dbs.Open = true + per.Flatmap[flatkey{index: index, shard: shard}] = dbs + dbs.W = w } - return -} - -func (per *DBPerShard) Del(dbs *DBShard) (err error) { - per.Mu.Lock() - defer per.Mu.Unlock() - - err = dbs.Close() - if err != nil { - return - } - PanicOn(dbs.DeleteDBPath()) - delete(per.Flatmap, flatkey{index: dbs.Index, shard: dbs.Shard}) - - // delete from the heirarchy - delete(dbs.ParentDBIndex.Shard, dbs.Shard) - return nil + return dbs, nil } func (per *DBPerShard) Close() (err error) { @@ -718,28 +480,7 @@ func (per *DBPerShard) Close() (err error) { // If requireData, we open the database and see that it has a key, rather // than assume that the database file presence is enough. func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requireData bool) (map[uint64]bool, error) { - - n := len(f.types) - if n != 1 && n != 2 { - PanicOn(fmt.Sprintf("internal error. only green or blue/green supported. we see types len %v", n)) - } - - var shards []map[uint64]bool - for _, ty := range f.types { - ss, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(ty, idx, roaringViewPath, requireData) - if err != nil { - return nil, err - } - shards = append(shards, ss) - } - - // Note: we don't actually know when the blue call and when the green call comes - // through here. So if we are deleting a shard, we will see a difference earlier - // in one than the other. TestAPI_ClearFlagForImportAndImportValues for example. - // Hence we cannot do a blue-green check here for matching shards. - - // If we are populating blue from green, it does matter that we return green. - return shards[n-1], nil + return f.dbPerShard.TypedDBPerShardGetShardsForIndex(f.typ, idx, roaringViewPath, requireData) } // if roaringViewPath is "" then for ty == roaringTxn we go to disk to discover @@ -751,10 +492,6 @@ func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requir // when a new DBShard is made, we will update the list of shards then. Thus // the per.index2shard should always be up to date AFTER the first call here. // -// Note: we cannot here call GetView2ShardsMapForIndex() because that only ever -// returns the green data and we are used during migration for both blue -// and green. -// func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, roaringViewPath string, requireData bool) (shardMap map[uint64]bool, err error) { // use the cache, always @@ -769,13 +506,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r return shardMap, nil } - i2ss, ok := per.index2shards[ty] - if !ok { - // index -> shardSet - i2ss = make(map[string]*shardSet) - per.index2shards[ty] = i2ss - } - // INVAR: i2ss is good, but may be an empty map + i2ss := per.index2shards ss, ok := i2ss[idx.name] if ok { @@ -785,7 +516,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r // gotta read shards from disk directory layout. setOfShards := newShardSet() - per.index2shards[ty][idx.name] = setOfShards + per.index2shards[idx.name] = setOfShards // Upon return, cache the setOfShards value and reuse it next time @@ -854,13 +585,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r } func (per *DBPerShard) unprotectedTypedIndexShardHasData(ty txtype, idx *Index, shard uint64) (hasData bool, err error) { - whichty := 0 - if len(per.types) == 2 { - if ty == per.types[1] { - whichty = 1 - } - } - if ty != per.types[whichty] { + if ty != per.typ { return } @@ -871,7 +596,7 @@ func (per *DBPerShard) unprotectedTypedIndexShardHasData(ty txtype, idx *Index, "per.GetDBShard(index='%v', shard='%v', ty='%v')", idx.name, shard, ty.String())) } - return dbs.W[whichty].HasData() + return dbs.W.HasData() } func listDirUnderDir(root string, includeRoot bool, ignoreEmpty bool) (files []string, err error) { @@ -906,224 +631,6 @@ func listDirUnderDir(root string, includeRoot bool, ignoreEmpty bool) (files []s return } -// populateBlueFromGreen prepares for a blue_green run at startup time. -// -// It is called at the end of Holder.Open(). This allows the application -// of blue-green checking to pilosa instances that -// were previously run only with a single (solo) backend. -// -// PRE: This operation requires, at its start, either: -// -// (1) an empty blue database -- this allows transitioning from -// a solo database to blue_green checking where the solo -// becomes the green; or -// -// (2) that the blue data, if present, be logically -// identical to the green data -- this allows one to restart -// a pilosa that was already running in blue_green mode -// and remain in blue_green mode. -// -// In either case, the goal to to finish populateBlueFromGreen() -// and have the exact same logical set of data in both backends. -// -// Why must the data be identical after Holder.Open() finishes? -// Otherwise subsequent blue-green checks have no hope of -// being accurate. -// -// The blue is the destination -- this is always types[0]. -// The green source is always types[1]. The mnemonic is blue_geen. -// The blue is first, so it is in types[0]. The green -// is second, in types[1]. For example, with PILOSA_STORAGE_BACKEND=bolt_roaring -// we have bolt as blue, and roaring as green. The contents of -// bolt must be empty or exactly match roaring. If bolt -// starts empty, it will be populated from roaring by -// populateBlueFromGreen(). -// -func (dbs *DBShard) populateBlueFromGreen() (err error) { - - n := len(dbs.W) - if n != 2 { - PanicOn(fmt.Sprintf("populateBlueFromGreen did not find 2 open DBs: have %v", n)) - } - - dest := dbs.W[0] // blue - src := dbs.W[1] // green - - // copy all the key/container pairs. - // Since a shard is fairly small, we think one Tx will suffice. - - readtx, err := src.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard}) - PanicOn(err) - defer readtx.Rollback() - - writetx, err := dest.NewTx(writable, dbs.Index, Txo{Write: writable, Index: dbs.idx, Shard: dbs.Shard}) - PanicOn(err) - defer writetx.Rollback() - - ctWriteCount := 0 - - for _, fld := range dbs.idx.Fields() { - field := fld.Name() - for _, vw := range fld.views() { - view := vw.name - citer, _, err := readtx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0) - if err != nil { - // might be an empty fragment. If so, let's not freak out. - if strings.Contains(err.Error(), "fragment not found") { - continue - } else { - writetx.Rollback() - return errors.Wrap(err, "DBShard.populateBlueFromGreen readtx.ContainerIterator") - } - } - - for citer.Next() { - ckey, rc := citer.Value() - err := writetx.PutContainer(dbs.Index, field, view, dbs.Shard, ckey, rc) - if err != nil { - citer.Close() - writetx.Rollback() - return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.PutContainer") - } - - ctWriteCount++ - if ctWriteCount%1000 == 1 { - - // regularly commiting smaller batches and the first batch as soon as - // possible massively speeds up writing to bolt. - // - // reference: https://github.com/boltdb/bolt/issues/94 - // - // benbjohnson commented on Mar 25, 2014 - // "Bulk loading more than 1000 items at a time is very slow. This is because nodes - // are not splitting before commit which causes large memmove() operations during insertion." - // runtime.memmove is taking all of the time in our pprof profile, when copying rbf to bolt, so we suspect it is this. - // - err = writetx.Commit() - if err != nil { - citer.Close() - writetx.Rollback() - return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.Commit") - } - writetx, err = dest.NewTx(writable, dbs.Index, Txo{Write: writable, Index: dbs.idx, Shard: dbs.Shard}) - if err != nil { - citer.Close() - writetx.Rollback() - return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.NewTx inside citer.Next() loop") - } - } - - } - citer.Close() - } - } - err = writetx.Commit() - if err != nil { - return errors.Wrap(err, "writetx.Commit()") - } - return nil -} - -// verifyBlueEqualsGreen checks that blue and green are identical. -func (dbs *DBShard) verifyBlueEqualsGreen() (err error) { - - n := len(dbs.W) - if n != 2 { - PanicOn(fmt.Sprintf("verifyBlueEqualsGreen did not find 2 open DBs: have %v", n)) - } - - blue := dbs.W[0] - green := dbs.W[1] - - greentx, err := green.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard}) - PanicOn(err) - defer greentx.Rollback() - - bluetx, err := blue.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard}) - PanicOn(err) - defer bluetx.Rollback() - - for _, fld := range dbs.idx.Fields() { - field := fld.Name() - for _, vw := range fld.views() { - - view := vw.name - gCiter, _, err := greentx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0) - if err != nil { - if strings.Contains(err.Error(), "fragment not found") { - continue - } else { - return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen greentx.ContainerIterator") - } - } - - bCiter, _, err := bluetx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0) - if err != nil { - gCiter.Close() - if bCiter != nil { - bCiter.Close() - } - return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen bluetx.ContainerIterator") - } - - for gCiter.Next() { - greenCkey, greenc := gCiter.Value() - - if !bCiter.Next() { - bCiter.Close() - gCiter.Close() - return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ - "sees missing blue container at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v' the greenCkey: '%v'", - dbs.Index, field, view, dbs.Shard, greenCkey)) - } - blueCkey, bluec := bCiter.Value() - - if blueCkey != greenCkey { - bCiter.Close() - gCiter.Close() - return fmt.Errorf("DBShard.verifyBlueEqualsGreen sees sequence-of-ckey "+ - "difference: blueCkey %v not equal to greenCkey %v at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v'", - blueCkey, greenCkey, dbs.Index, field, view, dbs.Shard) - } - nGreen := greenc.N() - nBlue := bluec.N() - if nBlue != nGreen { - bCiter.Close() - gCiter.Close() - return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ - "sees variation in blue at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v", - dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue)) - } - err = bluec.BitwiseCompare(greenc) - if err != nil { - bCiter.Close() - gCiter.Close() - return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ - "sees variation in blue at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v ; BitwiseCompare response: '%v'", - dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue, err)) - } - } - if bCiter.Next() { - blueCkey, _ := bCiter.Value() - bCiter.Close() - gCiter.Close() - return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ - "sees extra blue container (not present in green) at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v' the ckey: '%v'", - dbs.Index, field, view, dbs.Shard, blueCkey)) - } - bCiter.Close() - gCiter.Close() - } - } - - return nil -} - type FieldView2Shards struct { // field -> view -> *shardSet m map[string]map[string]*shardSet @@ -1232,15 +739,8 @@ func (vs *FieldView2Shards) removeField(name string) { delete(vs.m, name) } -// Note: cannot call this during migration, because -// it only ever returns the green shards if we are in blue-green. func (per *DBPerShard) GetFieldView2ShardsMapForIndex(idx *Index) (vs *FieldView2Shards, err error) { - - // for blue-green, it does matter that we return green, so we can migrate from it. - ty := per.types[0] - if per.isBlueGreen { - ty = per.types[1] - } + ty := per.typ switch ty { case roaringTxn: diff --git a/dbshard_internal_test.go b/dbshard_internal_test.go index 9482d1504..13c561810 100644 --- a/dbshard_internal_test.go +++ b/dbshard_internal_test.go @@ -84,7 +84,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { v2s.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet) } - for _, src := range []string{"roaring", "bolt", "rbf"} { + for _, src := range []string{"roaring", "rbf"} { cfg := mustHolderConfig() cfg.StorageConfig.Backend = src holder := NewHolder(tmpdir, cfg) @@ -145,7 +145,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { tx.Rollback() } } else { - // non-roaring: rbf, bolt + // non-roaring: rbf for _, shard := range []uint64{93, 223, 221, 215, 219, 217} { tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard}) @@ -191,14 +191,6 @@ rick/fields/_exists/views/standard/fragments/217 rick/fields/_exists/views/standard/fragments/93 rick/fields/_exists/views/standard/fragments/219 rick/fields/_exists/views/standard/fragments/223 -`, - "bolt": ` -rick/backends/backend-boltdb/shard.0093-bolt/bolt.db -rick/backends/backend-boltdb/shard.0215-bolt/bolt.db -rick/backends/backend-boltdb/shard.0217-bolt/bolt.db -rick/backends/backend-boltdb/shard.0219-bolt/bolt.db -rick/backends/backend-boltdb/shard.0221-bolt/bolt.db -rick/backends/backend-boltdb/shard.0223-bolt/bolt.db `, "rbf": ` rick/backends/backend-rbf/shard.0093-rbf @@ -225,7 +217,7 @@ func makeSampleRoaringDir(t *testing.T, root, index, backend string, minBytes in } var shard uint64 switch backend { - case "bolt", "rbf": + case "rbf": shard = shards[i] idx = helperCreateDBShard(h, index, shard) @@ -266,16 +258,8 @@ func helperCreateDBShard(h *Holder, index string, shard uint64) *Index { } // keep the ocd linter happy -var _ = makeBolttestDB var _ = makeRBFtestDB -func makeBolttestDB(path string, h *Holder, shard uint64) { - i := uint64(1) - w, _ := mustOpenEmptyBoltWrapper(path) - BoltMustSetBitvalue(w, "index", "field", "view", shard, i) - w.Close() -} - func makeRBFtestDB(path string, h *Holder, shard uint64) { i := uint64(1) diff --git a/delete_test.go b/delete_test.go index 061b72d1d..886b71e81 100644 --- a/delete_test.go +++ b/delete_test.go @@ -27,7 +27,6 @@ import ( ) func TestExecutor_DeleteRecords(t *testing.T) { - pilosa.NotBlueGreenTest(t) indexName := "i" setup := func(t *testing.T, r *require.Assertions, c *test.Cluster) { t.Helper() diff --git a/executor_test.go b/executor_test.go index adcf64dbb..ba357d724 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3843,7 +3843,6 @@ func TestExecutor_Execute_Existence(t *testing.T) { hldr2 := c.GetHolder(0) index2 := hldr2.Index("i") _ = index2 - //index2.Dump("after reopen") if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil { t.Fatal(err) diff --git a/fragment.go b/fragment.go index daab8bb34..d3c52c7dc 100644 --- a/fragment.go +++ b/fragment.go @@ -47,6 +47,7 @@ import ( "github.com/molecula/featurebase/v2/roaring" "github.com/molecula/featurebase/v2/shardwidth" "github.com/molecula/featurebase/v2/stats" + "github.com/molecula/featurebase/v2/storage" "github.com/molecula/featurebase/v2/testhook" "github.com/molecula/featurebase/v2/topology" "github.com/molecula/featurebase/v2/tracing" @@ -75,9 +76,6 @@ const ( // snapshotExt is the file extension used for an in-process snapshot. snapshotExt = ".snapshotting" - // copyExt is the file extension used for the temp file used while copying. - copyExt = ".copying" - // cacheExt is the file extension for persisted cache ids. cacheExt = ".cache" @@ -441,7 +439,7 @@ func (f *fragment) inspectStorage(data []byte, file *os.File, newGen generation, // (remapping an existing bitmap to match a new backing store). func (f *fragment) openStorage(unmarshalData bool) error { - useRowCache := f.idx.Txf().UseRowCache() + useRowCache := storage.RowCacheEnabled() if !f.idx.NeedsSnapshot() { f.gen = &NopGeneration{} if useRowCache { @@ -626,7 +624,7 @@ func (f *fragment) mustRow(tx Tx, rowID uint64) *Row { // (updating the cache). func (f *fragment) unprotectedRow(tx Tx, rowID uint64) (*Row, error) { - useRowCache := tx.UseRowCache() + useRowCache := storage.RowCacheEnabled() if useRowCache { if f.rowCache == nil { f.rowCache = newSimpleCache() @@ -699,7 +697,7 @@ func (f *fragment) setBit(tx Tx, rowID, columnID uint64) (changed bool, err erro if tx.Type() == RoaringTxn { return changed, errors.New("internal error: f.gen was nil and tx.Type is RoaringTxn - should never happen under roaring b/c storage should be open") } - // else blue green or transactional backend. Just do it. + // else transactional backend. Just do it. err = doSetFunc() } return changed, err @@ -744,7 +742,7 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo delete(f.checksums, int(rowID/HashBlockSize)) // Increment number of operations until snapshot is required. - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, 1) + f.incrementOpN(1) // If we're using a cache, update it. Otherwise skip the // possibly-expensive count operation. @@ -757,7 +755,7 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo } // Drop the rowCache entry; it's wrong, and we don't want to force // a new copy if no one's reading it. - if tx.UseRowCache() && f.rowCache != nil { + if storage.RowCacheEnabled() && f.rowCache != nil { f.rowCache.Add(rowID, nil) } @@ -809,7 +807,7 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b delete(f.checksums, int(rowID/HashBlockSize)) // Increment number of operations until snapshot is required. - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, 1) + f.incrementOpN(1) // If we're using a cache, update it. Otherwise skip the // possibly-expensive count operation. @@ -822,7 +820,7 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b } // Drop the rowCache entry; it's wrong, and we don't want to force // a new copy if no one's reading it. - if tx.UseRowCache() && f.rowCache != nil { + if storage.RowCacheEnabled() && f.rowCache != nil { f.rowCache.Add(rowID, nil) } @@ -891,7 +889,7 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo } // invalidate rowCache for this row. - if tx.UseRowCache() && f.rowCache != nil { + if storage.RowCacheEnabled() && f.rowCache != nil { f.rowCache.Add(rowID, nil) } @@ -942,7 +940,7 @@ func (f *fragment) unprotectedClearRow(tx Tx, rowID uint64) (changed bool, err e // Clear the row in cache. f.cache.Add(rowID, 0) - if tx.UseRowCache() && f.rowCache != nil { + if storage.RowCacheEnabled() && f.rowCache != nil { f.rowCache.Add(rowID, nil) } @@ -2426,7 +2424,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64 if f.storage != nil { wp = &f.storage.OpWriter } - useRowCache := tx.UseRowCache() + useRowCache := storage.RowCacheEnabled() doFunc := func() error { if len(set) > 0 { @@ -2438,7 +2436,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64 return errors.Wrap(err, "adding positions") } f.stats.Count(MetricImportedN, int64(changedN), 1) - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, changedN) + f.incrementOpN(changedN) } if len(clear) > 0 { @@ -2448,7 +2446,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64 return errors.Wrap(err, "clearing positions") } f.stats.Count(MetricClearedN, int64(changedN), 1) - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, changedN) + f.incrementOpN(changedN) } // Update cache counts for all affected rows. @@ -2735,7 +2733,7 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b rowSize := uint64(1 << shardVsContainerExponent) span, ctx := tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits") - useRowCache := tx.UseRowCache() + useRowCache := storage.RowCacheEnabled() var changed int var rowSet map[uint64]int var wp *io.Writer @@ -2749,7 +2747,7 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b return err } - changed, rowSet, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, rit, clear, true, rowSize, nil) + changed, rowSet, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, rit, clear, true, rowSize) return err }) @@ -2789,7 +2787,7 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b span, _ = tracing.StartSpanFromContext(ctx, "importRoaring.incrementOpN") - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, changed) + f.incrementOpN(changed) span.Finish() return nil @@ -2815,6 +2813,10 @@ func (f *fragment) incrementOpN(changed int) { if changed <= 0 { return } + // don't count opN or ops if our index doesn't want snapshots + if !f.idx.NeedsSnapshot() { + return + } f.opN += changed f.ops++ if f.opN > f.MaxOpN { @@ -3015,11 +3017,15 @@ func (f *fragment) writeStorageToArchive(tw *tar.Writer) error { tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Shard: f.shard}) defer tx.Rollback() - file, sz, err := tx.RoaringBitmapReader(f.index(), f.field(), f.view(), f.shard, f.path()) + rbm, err := tx.RoaringBitmap(f.index(), f.field(), f.view(), f.shard) if err != nil { - return err + return errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") + } + var buf bytes.Buffer + sz, err := rbm.WriteTo(&buf) + if err != nil { + return errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") } - defer file.Close() // Write archive header. if err := tw.WriteHeader(&tar.Header{ @@ -3033,7 +3039,7 @@ func (f *fragment) writeStorageToArchive(tw *tar.Writer) error { // Copy the file up to the last known size. // This is done outside the lock because the storage format is append-only. - if _, err := io.CopyN(tw, file, sz); err != nil { + if _, err := io.CopyN(tw, &buf, sz); err != nil { return errors.Wrap(err, "copying") } return nil @@ -3086,8 +3092,7 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) { // Process file based on file name. switch hdr.Name { case "data": - idx := f.holder.Index(f.index()) - tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx := f.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() if err := f.fillFragmentFromArchive(tx, tr); err != nil { return 0, errors.Wrap(err, "reading storage") @@ -3131,7 +3136,7 @@ func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error { if err != nil { return errors.Wrap(err, "fillFragmentFromArchive NewRoaringIterator") } - changed, rowSet, err := tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, clear, log, rowSize, data) + changed, rowSet, err := tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, clear, log, rowSize) _, _ = changed, rowSet if err != nil { return errors.Wrap(err, "fillFragmentFromArchive ImportRoaringBits") @@ -3139,40 +3144,6 @@ func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error { return nil } -func (f *fragment) readStorageFromArchive(r io.Reader) error { - - // Create a temporary file to copy into. - path := f.path() + copyExt - file, err := os.Create(path) - if err != nil { - return errors.Wrap(err, "creating directory") - } - defer file.Close() - - // Copy reader into temporary path. - if _, err = io.Copy(file, r); err != nil { - return errors.Wrap(err, "copying") - } - - // TODO(jea): isn't this next Rename a file handle leak? - // try closing first - if err := f.closeStorage(); err != nil { - return errors.Wrap(err, "closeStorage-prior-to-Rename-and-openStorage") - } - - // Move snapshot to data file location. - if err := os.Rename(path, f.path()); err != nil { - return errors.Wrap(err, "renaming") - } - - // Reopen storage. - if err := f.openStorage(true); err != nil { - return errors.Wrap(err, "opening") - } - - return nil -} - func (f *fragment) readCacheFromArchive(r io.Reader) error { // Slurp data from reader and write to disk. buf, err := ioutil.ReadAll(r) @@ -3369,7 +3340,7 @@ func (f *fragment) intRowIterator(tx Tx, wrap bool, filters ...roaring.BitmapFil // accumulator [column ID] -> [int value] acc := make(map[uint64]int64) - if tx.UseRowCache() { + if storage.RowCacheEnabled() { // needs a write lock since it will update the f.rowCache f.mu.Lock() defer f.mu.Unlock() diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 9b4ac461a..23c2063f1 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -183,7 +183,6 @@ func TestFragment_RowcacheMap(t *testing.T) { // Ensure a fragment can clear a row. func TestFragment_ClearRow(t *testing.T) { - NotBlueGreenTest(t) f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) @@ -215,7 +214,6 @@ func TestFragment_ClearRow(t *testing.T) { // Ensure a fragment can set a row. func TestFragment_SetRow(t *testing.T) { - NotBlueGreenTest(t) f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 7, "") _ = idx defer f.Clean(t) @@ -1728,7 +1726,7 @@ func roaringOnlyBenchmark(b *testing.B) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { - roaringOnlyTest(t) + // roaringOnlyTest(t) f0, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") defer f0.Clean(t) @@ -1741,6 +1739,10 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } else if _, err := f0.clearBit(tx, 1000, 1); err != nil { t.Fatal(err) } + err := tx.Commit() + if err != nil { + t.Fatalf("committing write: %v", err) + } // Verify cache is populated. if n := f0.cache.Len(); n != 1 { @@ -1755,7 +1757,9 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Read into another fragment. - f1, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") + f1, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") + tx.Rollback() + defer f1.Clean(t) if rn, err := f1.ReadFrom(&buf); err != nil { // eventually calls fragment.fillFragmentFromArchive @@ -1763,6 +1767,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } else if wn != rn { t.Fatalf("read/write byte count mismatch: wn=%d, rn=%d", wn, rn) } + // make a read-only Tx after ReadFrom has committed. + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f1, Shard: f1.shard}) // Verify cache is in other fragment. if n := f1.cache.Len(); n != 1 { @@ -3049,7 +3055,7 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { // to generate an op log and/or snapshot. itr, err := roaring.NewRoaringIterator(data) PanicOn(err) - _, _, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, false, false, 0, nil) + _, _, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, false, false, 0) if err != nil { b.Errorf("import error: %v", err) } @@ -5215,15 +5221,13 @@ func TestImportValueConcurrent(t *testing.T) { // we will be making a new Tx each time, so we can rollback the default provided one. tx.Rollback() - types := idx.holder.txf.TxTypes() - for _, ty := range types { - switch ty { - case roaringTxn: - t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " + - "blueGreenTx because the lack of transactional consistency " + - "from Roaring-per-file will create false comparison " + - "failures.")) - } + ty := idx.holder.txf.TxTyp() + switch ty { + case roaringTxn: + t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " + + "roaring because the lack of transactional consistency " + + "from Roaring-per-file will create false comparison " + + "failures.")) } eg := &errgroup.Group{} @@ -5364,11 +5368,6 @@ func TestImportValueRowCache(t *testing.T) { // do we see races/corruption around concurrent read/write. // especially on writes to the row cache. func TestFragmentConcurrentReadWrite(t *testing.T) { - // actual transaction backends, there won't be any - // data, and in particular, the blue-green tests will - // note this and fire a false-positive. - NotBlueGreenTest(t) - f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked) defer f.Clean(t) tx.Rollback() @@ -5528,12 +5527,6 @@ func TestFragment_Bug_Q2DoubleDelete(t *testing.T) { } } -func NotBlueGreenTest(t *testing.T) { - if strings.Contains(CurrentBackend(), "_") { - t.Skip("skip under blue green") - } -} - var mutexSamplesPrepared sync.Once func requireMutexSampleData(tb testing.TB) { diff --git a/go.mod b/go.mod index ff7c1d715..67d9c30bd 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,6 @@ require ( github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect - github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 github.com/go-test/deep v1.0.7 github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.3.3 diff --git a/go.sum b/go.sum index 0959b010c..60db445b5 100644 --- a/go.sum +++ b/go.sum @@ -86,10 +86,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E= -github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= diff --git a/holder.go b/holder.go index dea18e3e7..756d75443 100644 --- a/holder.go +++ b/holder.go @@ -302,7 +302,6 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { txf, err := NewTxFactory(cfg.StorageConfig.Backend, h.IndexesPath(), h) PanicOn(err) h.txf = txf - h.txf.blueGreenOffIfRunningBlueGreen() _ = testhook.Created(h.Auditor, h, nil) return h @@ -605,8 +604,6 @@ func (h *Holder) Open() error { h.txf = txf } - h.txf.blueGreenOffIfRunningBlueGreen() - // Reset closing in case Holder is being reopened. h.closing = make(chan struct{}) @@ -713,13 +710,6 @@ func (h *Holder) Open() error { return errors.Wrap(err, "Holder.Open h.txf.Open()") } - // under blue_green, we must sync blue from green before we turn on checking. - if err := h.txf.green2blue(h); err != nil { - return errors.Wrap(err, "Holder.Open h.txf.green2blue(h)") - } - - h.txf.blueGreenOnIfRunningBlueGreen() - if h.cfg.LookupDBDSN != "" { h.Logger.Printf("connecting to lookup database") @@ -814,9 +804,6 @@ func (h *Holder) Close() error { if globalUseStatTx { fmt.Printf("%v\n", globalCallStats.report()) } - if h.txf != nil && h.txf.blueGreenReg != nil { - h.txf.blueGreenReg.Close() - } h.Stats.Close() @@ -2131,12 +2118,6 @@ func (h *Holder) addIndex(idx *Index) { h.imu.Unlock() } -func (h *Holder) DumpAllShards() { - h.mu.RLock() - defer h.mu.RUnlock() - h.txf.dbPerShard.DumpAll() -} - func (h *Holder) Txf() *TxFactory { h.mu.Lock() defer h.mu.Unlock() diff --git a/holder_internal_test.go b/holder_internal_test.go index bd35f0b03..7cd8ad630 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -22,7 +22,6 @@ import ( "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/testhook" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck ) var _ = fmt.Printf @@ -109,73 +108,6 @@ func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID ui } } -func testMustHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) { - - //shard := columnID / ShardWidth - - // hmm... if its a new holder, meta data isn't there, so ask for it. - idx, err := h.CreateIndexIfNotExists(index, IndexOptions{}) - PanicOn(err) - - f := idx.Field(field) - if f == nil { - t.Fatalf("no such field '%v'", field) - } - - row, err := f.Row(nil, rowID) - if err != nil { - t.Fatalf("error getting field.Row(rowID=%v): %v", rowID, err) - } - - cols := row.Columns() - if len(cols) == 0 { - t.Fatalf("error getting field.Row().Columns(): empty columns, colID %v bit was not hot", columnID) - } - - for _, c := range cols { - if c == columnID { - return // ok, found it. - } - } - t.Fatalf("error getting field.Row().Columns(): colID %v bit was not hot", columnID) -} - -func testMustNotHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) { - if testHasBit(t, h, index, field, rowID, columnID) { - t.Fatalf("error, expected no bit but this bit was hot: index='%v', field='%v', rowID='%v', columnID='%v'", index, field, rowID, columnID) - } -} - -func testHasBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) bool { - - idx := h.Index(index) - if idx == nil { - return false // not even an index by this name. Obviously no hot bits either. - } - - f := idx.Field(field) - if f == nil { - return false - } - - row, err := f.Row(nil, rowID) - if err != nil { - return false - } - - cols := row.Columns() - if len(cols) == 0 { - return false - } - - for _, c := range cols { - if c == columnID { - return true // ok, found it. - } - } - return false -} - func TestHolderOperatorProcess(t *testing.T) { h, path, err := makeHolder(t, "") if err != nil { diff --git a/index.go b/index.go index 4cfd301b2..8506184f5 100644 --- a/index.go +++ b/index.go @@ -27,7 +27,6 @@ import ( "github.com/molecula/featurebase/v2/roaring" "github.com/molecula/featurebase/v2/stats" "github.com/molecula/featurebase/v2/testhook" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) @@ -464,7 +463,6 @@ func (i *Index) Close() error { // make it clear what the Index.AvailableShards() calls are trying to obtain. const includeRemote = false -const localOnly = true // AvailableShards returns a bitmap of all shards with data in the index. func (i *Index) AvailableShards(localOnly bool) *roaring.Bitmap { @@ -853,14 +851,6 @@ func FormatQualifiedIndexName(index string) string { return fmt.Sprintf("%s\x00", index) } -// Dump prints to stdout the contents of the roaring Containers -// stored in idx. Mostly for debugging. -func (i *Index) Dump(label string) { - fileline := FileLine(2) - fmt.Printf("\n%v Dump: %v\n\n", fileline, label) - i.holder.txf.dbPerShard.DumpAll() -} - func (i *Index) Txf() *TxFactory { return i.holder.txf } diff --git a/pjobs.go b/pjobs.go deleted file mode 100644 index 0636fbd36..000000000 --- a/pjobs.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "sync" - - "github.com/glycerine/idem" -) - -// parallelJobs runs functions in parallel on a goroutine -// pool that has nGoro goroutines. -type parallelJobs struct { - nGoro int - - jobQ chan func(worker int) error - halters []*idem.Halter - - // err is protected by errmu - err error - errmu sync.Mutex -} - -func newParallelJobs(nGoro int) (p *parallelJobs) { - if nGoro < 1 { - // 0 really means, - // "turn it up to 11". - // same for negative. - nGoro = 10000 - } - // maximum 10K goroutines - if nGoro > 10000 { - nGoro = 10000 - } - - p = ¶llelJobs{ - nGoro: nGoro, - jobQ: make(chan func(worker int) error, 10000), - halters: make([]*idem.Halter, nGoro), - } - - for j := 0; j < nGoro; j++ { - h := idem.NewHalter() - p.halters[j] = h - } - - for i, h := range p.halters { - go func(h *idem.Halter, worker int) { - defer h.MarkDone() - for { - select { - case <-h.ReqStop.Chan: - return - case f, ok := <-p.jobQ: - if !ok { - // channel closed, finish up - return - } - - err1 := f(worker) - if err1 != nil { - p.errmu.Lock() - if p.err == nil { - p.err = err1 - } - p.errmu.Unlock() - // an error occurred, tell everyone to stop - for _, h2 := range p.halters { - h2.RequestStop() - } - return - } - } - } - }(h, i) - } - return -} - -// return value accepted will be false if we are shutting down -// due to an error. -func (p *parallelJobs) run(fun func(worker int) error) (accepted bool) { - select { - case <-p.halters[0].ReqStop.Chan: - return false - case p.jobQ <- fun: - return true - } -} - -func (p *parallelJobs) waitForFinish() error { - - // tell the workers no more jobs. - close(p.jobQ) - - // wait for everyone to finish - for i, h := range p.halters { - _ = i - <-h.Done.Chan - } - - return p.err -} diff --git a/pjobs_test.go b/pjobs_test.go deleted file mode 100644 index 71192b278..000000000 --- a/pjobs_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "fmt" - "sync/atomic" - "testing" -) - -func Test_ParallelJobs_EarlyShutdown_WaitsForAllGoro(t *testing.T) { - const n = 10000 // total jobs to run - - var errLastOne = fmt.Errorf("the last job has run, and returned this error") - pj := newParallelJobs(100) - nTotal := int64(0) - for i := 0; i < n; i++ { - accepted := pj.run(func(worker int) error { - highpoint := atomic.AddInt64(&nTotal, 1) - switch int(highpoint) { - case n - 1: - return errLastOne - } - return nil - }) - if !accepted { - panic("should have been accepted") - } - } - err := pj.waitForFinish() - tot := atomic.LoadInt64(&nTotal) - if int(tot) != n { - panic(fmt.Sprintf("We didn't run them all? tot=%v, n=%v; pj.jobQ len %v; err='%v'", tot, n, len(pj.jobQ), err)) - } - if err != errLastOne { - panic("expected to see errLastOne") - } - // good: finished cleanly. -} diff --git a/rbf.go b/rbf.go index 64a268281..cee25f35c 100644 --- a/rbf.go +++ b/rbf.go @@ -15,15 +15,12 @@ package pilosa import ( - "bytes" "fmt" "io" - "io/ioutil" "math" "os" "strings" "sync" - "sync/atomic" "github.com/molecula/featurebase/v2/rbf" rbfcfg "github.com/molecula/featurebase/v2/rbf/cfg" @@ -76,8 +73,6 @@ func (w *RbfDBWrapper) CleanupTx(tx Tx) { w.muDb.Lock() delete(w.openTx, r) - //vv("rbf CleanupTx gid %v about to call r.o.dbs.Cleanup(tx.Sn=%v)", curGID(), tx.Sn()) - r.o.dbs.Cleanup(tx) // release the read/write lock. w.muDb.Unlock() } @@ -186,20 +181,12 @@ type RBFTx struct { initialIndex string tx *rbf.Tx o Txo - sn int64 // serial number Db *RbfDBWrapper done bool mu sync.Mutex // protect done as it changes state } -func (tx *RBFTx) IsDone() (done bool) { - tx.mu.Lock() - done = tx.done - tx.mu.Unlock() - return -} - func (tx *RBFTx) DBPath() string { return tx.tx.DBPath() } @@ -210,17 +197,13 @@ func (tx *RBFTx) Type() string { func (tx *RBFTx) Rollback() { tx.tx.Rollback() - - // must happen after actual rollback tx.Db.CleanupTx(tx) } func (tx *RBFTx) Commit() (err error) { err = tx.tx.Commit() - - // must happen after actual commit tx.Db.CleanupTx(tx) - return + return err } func (tx *RBFTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { @@ -412,10 +395,6 @@ func (tx *RBFTx) Min(index, field, view string, shard uint64) (uint64, bool, err return tx.tx.Min(rbfName(index, field, view, shard)) } -func (tx *RBFTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - return tx.tx.UnionInPlace(rbfName(index, field, view, shard), others...) -} - // CountRange returns the count of hot bits in the start, end range on the fragment. // roaring.countRange counts the number of bits set between [start, end). func (tx *RBFTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { @@ -426,24 +405,8 @@ func (tx *RBFTx) OffsetRange(index, field, view string, shard uint64, offset, st return tx.tx.OffsetRange(rbfName(index, field, view, shard), offset, start, end) } -func (tx *RBFTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {} - -func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { - return tx.tx.ImportRoaringBits(rbfName(index, field, view, shard), rit, clear, log, rowSize, data) -} - -func (tx *RBFTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") - } - var buf bytes.Buffer - sz, err = rbm.WriteTo(&buf) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") - } - return ioutil.NopCloser(&buf), sz, err +func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { + return tx.tx.ImportRoaringBits(rbfName(index, field, view, shard), rit, clear, log, rowSize) } func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { @@ -452,39 +415,6 @@ func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring return b.Iterator() } -func (tx *RBFTx) Pointer() string { - return fmt.Sprintf("%p", tx) -} - -func (tx *RBFTx) Dump(short bool, shard uint64) { - tx.tx.Dump(short, shard) -} - -// Readonly is true if the transaction is not read-and-write, but only doing reads. -func (tx *RBFTx) Readonly() bool { - return !tx.tx.Writable() -} - -func (tx *RBFTx) Group() *TxGroup { - return tx.o.Group -} - -func (tx *RBFTx) Options() Txo { - return tx.o -} - -func (tx *RBFTx) Sn() int64 { - return tx.sn -} - -func (tx *RBFTx) UseRowCache() bool { - // since RFB returns memory mapped data, we can't use - // the rowCache without first making a copy. - // So we only use the rowCache if the copy is - // enabled. - return storage.EnableRowCache() -} - func (tx *RBFTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { return tx.tx.ApplyFilter(rbfName(index, field, view, shard), ckey, filter) } @@ -591,20 +521,16 @@ func (w *RbfDBWrapper) OpenDB() error { return nil } -var globalNextTxSnRBFTx int64 - func (w *RbfDBWrapper) NewTx(write bool, initialIndex string, o Txo) (_ Tx, err error) { tx, err := w.db.Begin(write) if err != nil { return nil, err } - sn := atomic.AddInt64(&globalNextTxSnRBFTx, 1) rtx := &RBFTx{ tx: tx, initialIndex: initialIndex, o: o, - sn: sn, Db: w, } @@ -629,20 +555,6 @@ func (w *RbfDBWrapper) DeleteFragment(index, field, view string, shard uint64, f return tx.Commit() } -func (w *RbfDBWrapper) DeleteDBPath(dbs *DBShard) error { - path := dbs.pathForType(rbfTxn) - return os.RemoveAll(path) -} - func (w *RbfDBWrapper) OpenListString() (r string) { return "rbf OpenListString not implemented yet" } - -func (w *RbfDBWrapper) OpenSnList() (slc []int64) { - w.muDb.Lock() - for v := range w.openTx { - slc = append(slc, v.sn) - } - w.muDb.Unlock() - return -} diff --git a/rbf/cursor_internal_test.go b/rbf/cursor_internal_test.go index 3d8dca800..e426dda5b 100644 --- a/rbf/cursor_internal_test.go +++ b/rbf/cursor_internal_test.go @@ -59,20 +59,12 @@ func TestCursor_RoaringImport(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. _ = rowSet if changed != 1 { t.Fatalf("expected 1 changed, got %v", changed) } - if false { - cur, err := tx.cursor(name) - PanicOn(err) - - cur.dump() - - _ = cur.tx.dumpAllPages(true) - } } func TestCursor_RoaringImport_clear_bits(t *testing.T) { @@ -93,7 +85,7 @@ func TestCursor_RoaringImport_clear_bits(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) _ = rowSet if changed != 1 { @@ -104,21 +96,12 @@ func TestCursor_RoaringImport_clear_bits(t *testing.T) { clear = true itr2 := getRoaringIter([]uint64{1}...) - changed, rowSet, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil) + changed, rowSet, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize) PanicOn(err) _ = rowSet if changed != 1 { t.Fatalf("expected 1 changed on clear true, got %v", changed) } - - if false { - cur, err := tx.cursor(name) - PanicOn(err) - - cur.dump() - - _ = cur.tx.dumpAllPages(true) - } } func TestCursor_RoaringImport_two_leaves(t *testing.T) { @@ -152,20 +135,12 @@ func TestCursor_RoaringImport_two_leaves(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. _ = rowSet if changed != 6000 { t.Fatalf("expected 6000 bits changed, got %v", changed) } - if false { - cur, err := tx.cursor(name) - PanicOn(err) - - cur.dump() - - _ = cur.tx.dumpAllPages(true) - } } func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) { @@ -211,7 +186,7 @@ func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) { defer tx.Rollback() //vv("DONE WITH Add()") - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != expectedBitsChanged-3000 { t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged-3000, changed) @@ -219,37 +194,23 @@ func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) { //vv("changed on set is %v", changed) //vv("about to do itr2, that starts with key %v", itr2.ContainerKeys()[0]) - changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil) + changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize) PanicOn(err) if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) } - //vv("done with itr2") - - dump := func() { - cur, err := tx.cursor(name) - PanicOn(err) - //cur.dump() - _ = cur.tx.dumpAllPages(true) - } - _ = dump - //dump() - //vv("now clear") - // now clear clear = true //itr3 := getRoaringIter(want[len(want)-3000:]...) itr3 := getRoaringIter(want...) - changed, _, err = tx.ImportRoaringBits(name, itr3, clear, false, rowSize, nil) + changed, _, err = tx.ImportRoaringBits(name, itr3, clear, false, rowSize) PanicOn(err) if changed != expectedBitsChanged { // cursor_internal_test.go:235: expected 2,724,000 bits changed, got 2,721,000 t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed) } - - //dump() } func TestCursor_RoaringImport_auto_many_leaves(t *testing.T) { @@ -292,34 +253,21 @@ func TestCursor_RoaringImport_auto_many_leaves(t *testing.T) { defer tx.Rollback() //vv("DONE WITH Add()") - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != expectedBitsChanged { t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed) } - //vv("changed on set is %v", changed) - - dump := func() { - cur, err := tx.cursor(name) - PanicOn(err) - //cur.dump() - _ = cur.tx.dumpAllPages(true) - } - _ = dump - //dump() - //vv("now clear") // now clear clear = true itr2 := getRoaringIter(want...) - changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil) + changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize) PanicOn(err) if changed != expectedBitsChanged { t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed) } - - //dump() } func TestCursor_putBranchCellsHandlesLotsOfNewBranchesAtTheRoot(t *testing.T) { @@ -430,8 +378,6 @@ func TestCursor_putBranchCellsHandlesLotsOfNewBranchesAtTheRoot(t *testing.T) { err = c.putBranchCells(0, branches) PanicOn(err) - - //c.tx.dumpAllPages(true) } func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) { @@ -475,29 +421,18 @@ func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - dump := func() { - cur, err := tx.cursor(name) - PanicOn(err) - //cur.dump() - _ = cur.tx.dumpAllPages(true) - } - _ = dump for i := 0; i < NbranchCells; i++ { itr := getRoaringIter(want[i*3000 : (i+1)*3000]...) - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) } - //vv("changed on set is %v", changed) - //dump() } - //vv("now clear") - // now clear clear = true @@ -505,13 +440,11 @@ func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) { itr := getRoaringIter(want[i*3000 : (i+1)*3000]...) - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) } - //vv("changed on set is %v", changed) - //dump() } } @@ -558,35 +491,22 @@ func TestCursor_from_B_to_C(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - dump := func() { - cur, err := tx.cursor(name) - PanicOn(err) - //cur.dump() - _ = cur.tx.dumpAllPages(true) - } - _ = dump itr := getRoaringIter(want[:6000]...) - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 6000 { t.Fatalf("expected %v bits changed, got %v", 6000, changed) } - //vv("changed on set is %v", changed) - //dump() //vv("STARTING TO ADD C") itr = getRoaringIter(want[6000:9000]...) - changed, _, err = tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err = tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) } - //vv("changed on set is %v", changed) - //dump() - - //vv("now clear") // now clear clear = true @@ -595,12 +515,10 @@ func TestCursor_from_B_to_C(t *testing.T) { itr := getRoaringIter(want[i*3000 : (i+1)*3000]...) - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) // failing here got 0 } - //vv("changed on clear is %v", changed) - //dump() } } diff --git a/rbf/cursorx.go b/rbf/cursorx.go index 0916d2c36..9b2a61111 100644 --- a/rbf/cursorx.go +++ b/rbf/cursorx.go @@ -174,7 +174,7 @@ func intoContainer(l leafCell, tx *Tx, replacing *roaring.Container, target []by orig := l.Data var cpMaybe []byte var mapped bool - if storage.EnableRowCache() || tx.db.cfg.DoAllocZero { + if storage.RowCacheEnabled() || tx.db.cfg.DoAllocZero { // make a copy, otherwise the rowCache will see corrupted data // or mmapped data that may disappear. cpMaybe = target[:len(orig)] @@ -191,7 +191,7 @@ func intoContainer(l leafCell, tx *Tx, replacing *roaring.Container, target []by case ContainerTypeBitmapPtr: _, bm, _ := tx.leafCellBitmap(toPgno(cpMaybe)) cloneMaybe := bm - if storage.EnableRowCache() { + if storage.RowCacheEnabled() { cloneMaybe = (*[1024]uint64)(unsafe.Pointer(&target[0]))[:1024] copy(cloneMaybe, bm) } @@ -217,7 +217,7 @@ func toContainer(l leafCell, tx *Tx) (c *roaring.Container) { orig := l.Data var cpMaybe []byte var mapped bool - if storage.EnableRowCache() || tx.db.cfg.DoAllocZero { + if storage.RowCacheEnabled() || tx.db.cfg.DoAllocZero { // make a copy, otherwise the rowCache will see corrupted data // or mmapped data that may disappear. cpMaybe = make([]byte, len(orig)) @@ -234,7 +234,7 @@ func toContainer(l leafCell, tx *Tx) (c *roaring.Container) { case ContainerTypeBitmapPtr: _, bm, _ := tx.leafCellBitmap(toPgno(cpMaybe)) cloneMaybe := bm - if storage.EnableRowCache() { + if storage.RowCacheEnabled() { cloneMaybe = make([]uint64, len(bm)) copy(cloneMaybe, bm) } diff --git a/rbf/db.go b/rbf/db.go index f1623ded8..4f88de259 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -319,7 +319,7 @@ func (db *DB) Close() (err error) { // least a single hot bit inside the db // in order to return hasAnyRecords true. // -// HasData is used by backend migration and blue/green checks. +// HasData is used by backend migration. // // If there is a disk error we return (false, error), so always // check the error before deciding if hasAnyRecords is valid. diff --git a/rbf/tx.go b/rbf/tx.go index f126cfbf9..9ecbe9a02 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -23,7 +23,6 @@ import ( "sync" "github.com/benbjohnson/immutable" - "github.com/molecula/featurebase/v2/hash" "github.com/molecula/featurebase/v2/roaring" txkey "github.com/molecula/featurebase/v2/short_txkey" . "github.com/molecula/featurebase/v2/vprint" @@ -1299,27 +1298,6 @@ func (tx *Tx) Min(name string) (uint64, bool, error) { return uint64((cell.Key << 16) | uint64(cell.firstValue(tx))), true, nil } -func (tx *Tx) UnionInPlace(name string, others ...*roaring.Bitmap) error { - rbm, err := tx.RoaringBitmap(name) - PanicOn(err) - - rbm.UnionInPlace(others...) - // iterate over the containers that changed within rbm, and write them back to disk. - - it, found := rbm.Containers.Iterator(0) - _ = found // don't care about the value of found, because first containerKey might be > 0 - - for it.Next() { - containerKey, rc := it.Value() - - // TODO: only write the changed ones back, as optimization? - // Compare to ImportRoaringBits. - err := tx.PutContainer(name, containerKey, rc) - PanicOn(err) - } - return nil -} - // roaring.countRange counts the number of bits set between [start, end). func (tx *Tx) CountRange(name string, start, end uint64) (uint64, error) { tx.mu.RLock() @@ -1538,130 +1516,7 @@ func (si *emptyContainerIterator) Value() (uint64, *roaring.Container) { return 0, nil } -func (tx *Tx) Dump(short bool, shard uint64) { - fmt.Println(tx.DumpString(short, shard)) -} -func (tx *Tx) DumpString(short bool, shard uint64) (r string) { - - r = "allkeys:[\n" - - // grab root records, for a list of bitmaps. - records, err := tx.RootRecords() - PanicOn(err) - n := 0 - - for itr := records.Iterator(); !itr.Done(); { - name, _ := itr.Next() - - c, err := tx.cursor(name.(string)) - PanicOn(err) - defer c.Close() - - err = c.First() // First will rewind to beginning. - if err == io.EOF { - r += "" - n++ - continue - } - PanicOn(err) - for { - err := c.Next() - if err == io.EOF { - break - } - PanicOn(err) - - elem := &c.stack.elems[c.stack.top] - leafPage, _, err := c.tx.readPage(elem.pgno) - PanicOn(err) - cell := readLeafCell(leafPage, elem.index) - - ckey := cell.Key - ct := toContainer(cell, tx) - - s := stringOfCkeyCt(ckey, ct, name.(string), short, true) - r += s - n++ - } - } - if n == 0 { - return "" - } - // note that we can have a bitmap present, but it can be empty - r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + "\n" - - return "rbf-" + r -} - -func containerToBytes(ct *roaring.Container) []byte { - - ty := roaring.ContainerType(ct) - switch ty { - case roaring.ContainerNil: - PanicOn("nil container") - case roaring.ContainerArray: - return fromArray16(roaring.AsArray(ct)) - case roaring.ContainerBitmap: - return fromArray64(roaring.AsBitmap(ct)) - case roaring.ContainerRun: - return fromInterval16(roaring.AsRuns(ct)) - } - PanicOn(fmt.Sprintf("unknown container type '%v'", int(ty))) - return nil -} - -func bitmapAsString(rbm *roaring.Bitmap) (r string) { - r = "c(" - slc := rbm.Slice() - width := 0 - s := "" - for _, v := range slc { - if width == 0 { - s = fmt.Sprintf("%v", v) - } else { - s = fmt.Sprintf(", %v", v) - } - width += len(s) - r += s - if width > 70 { - r += ",\n" - width = 0 - } - } - if width == 0 && len(r) > 2 { - r = r[:len(r)-2] - } - return r + ")" -} - -func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string, short, showHash bool) (s string) { - - hsh := "" - if showHash { - by := containerToBytes(ct) - hsh = hash.Blake3sum16(by) - } - - cts := roaring.NewSliceContainers() - cts.Put(ckey, ct) - rbm := &roaring.Bitmap{Containers: cts} - srbm := bitmapAsString(rbm) - - var pre string - if len(rrName) > 0 { - pre = txkey.PrefixToString([]byte(rrName)) - } - bkey := pre + fmt.Sprintf("ckey@%020d", ckey) - - s = fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hsh, ct.N()) - - if !short { - s += " ......." + srbm + "\n" - } - return -} - -func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { +func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { // begin write boilerplate if tx.db == nil { diff --git a/rbf/tx_test.go b/rbf/tx_test.go index 771e17722..f7cd78d6a 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -16,14 +16,12 @@ package rbf_test import ( "fmt" - "math" "math/rand" "sync" "testing" "time" "github.com/molecula/featurebase/v2/rbf" - txkey "github.com/molecula/featurebase/v2/short_txkey" ) func TestTx_CommitRollback(t *testing.T) { @@ -496,29 +494,6 @@ func BenchmarkTx_Contains(b *testing.B) { } } -func TestTx_Dump(t *testing.T) { - db := MustOpenDB(t) - defer MustCloseDB(t, db) - tx := MustBegin(t, db, true) - defer tx.Rollback() - - index, field, view, shard := "i", "f", "v", uint64(15) - - nm := rbfName(index, field, view, shard) - - if err := tx.CreateBitmap(nm); err != nil { - t.Fatal(err) - } else if _, err := tx.Add(nm, 0x00000001, 0x00000002, 0x00010003, 0x00030004); err != nil { - t.Fatal(err) - } - - // test that we don't crash, and get *something* back - s := tx.DumpString(true, math.MaxUint64) - if s == "" { - panic("should have had 3 containers!") - } -} - func TestTx_CreateBitmap(t *testing.T) { t.Run("Bulk", func(t *testing.T) { db := MustOpenDB(t) @@ -542,7 +517,3 @@ func TestTx_CreateBitmap(t *testing.T) { } }) } - -func rbfName(index, field, view string, shard uint64) string { - return string(txkey.Prefix(index, field, view, shard)) -} diff --git a/rbf/util.go b/rbf/util.go index fede5c190..7a44120a7 100644 --- a/rbf/util.go +++ b/rbf/util.go @@ -15,13 +15,16 @@ package rbf import ( "fmt" - "io" "strings" txkey "github.com/molecula/featurebase/v2/short_txkey" . "github.com/molecula/featurebase/v2/vprint" ) +// we don't currently use dumpAllPages but it's tricky enough to get right +// that it's probably worth keeping as a debugging tool. +var _ = (*Tx).dumpAllPages + func (tx *Tx) dumpAllPages(showLeaves bool) error { infos, err := tx.PageInfos() @@ -213,56 +216,6 @@ func prefixToString(s string) (ret string) { return txkey.PrefixToString([]byte(s)) } -func (c *Cursor) dump() { - fmt.Printf("\n Cursor %p has bitmaps:\n%v\n", c, c.debugStringBitmaps()) -} - -var _ = (&Cursor{}).dump -var _ = (&Cursor{}).debugStringBitmaps - -func (c_orig *Cursor) debugStringBitmaps() (r string) { - - // work with a totally new Cursor, so we don't impact our current cursor - // so any test using the cursor isn't disturbed. - c2 := Cursor{tx: c_orig.tx} - c2.stack.elems[0] = c_orig.stack.elems[0] - err := c2.First() - if err != nil { - if err == io.EOF { - // ok, can be empty - return "" - } else { - panic(err) - } - } - n := 0 - for { - err := c2.Next() - if err == io.EOF { - break - } - PanicOn(err) - - //instead of cell := c2.cell() - elem := &c2.stack.elems[c2.stack.top] - leafPage, _, err := c2.tx.readPage(elem.pgno) - PanicOn(err) - cell := readLeafCell(leafPage, elem.index) - - ckey := cell.Key - ct := toContainer(cell, c2.tx) - const short = true - s := stringOfCkeyCt(ckey, ct, "", short, true) - r += s - n++ - } - - if n == 0 { - return "" - } - return -} - ///////////////// happy linter var _ = printMetaPage diff --git a/roaring/roaring.go b/roaring/roaring.go index e35487a3e..4fd8a50c0 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2434,13 +2434,13 @@ func (b *Bitmap) writeOp(op *op) error { // Iterator returns a new iterator for the bitmap. func (b *Bitmap) Iterator() *Iterator { - itr := NewIterator(&BitmapIteratorFinder{b}) + itr := &Iterator{bitmap: b} itr.Seek(0) return itr } func (b *Bitmap) IteratorAt(start uint64) *Iterator { - itr := NewIterator(&BitmapIteratorFinder{b}) + itr := &Iterator{bitmap: b} itr.Seek(start) return itr } @@ -2701,37 +2701,18 @@ type BitmapInfo struct { From, To uintptr // if set, indicates the address range used when unpacking } -type IteratorFinder interface { - FindIterator(uint64) (ContainerIterator, bool) - Close() -} -type BitmapIteratorFinder struct { - bitmap *Bitmap -} - -func (bif *BitmapIteratorFinder) FindIterator(seek uint64) (ContainerIterator, bool) { - return bif.bitmap.Containers.Iterator(seek) -} -func (bif *BitmapIteratorFinder) Close() {} - // Iterator represents an iterator over a Bitmap. type Iterator struct { - finder IteratorFinder + bitmap *Bitmap citer ContainerIterator key uint64 c *Container j, k int32 // i: container; j: array index, bit index, or run index; k: offset within the run } -// NewIterator requires f as an IteratorFinder, it will -// crash if f is nil. -func NewIterator(f IteratorFinder) *Iterator { - return &Iterator{finder: f} -} - -func (itr *Iterator) Close() { - itr.finder.Close() -} +// This exists because we used to support a backend which needed it, and I +// don't want to re-experience the joy of figuring out where close calls are needed. +func (itr *Iterator) Close() {} // Seek moves to the first value equal to or greater than `seek`. func (itr *Iterator) Seek(seek uint64) { @@ -2740,7 +2721,7 @@ func (itr *Iterator) Seek(seek uint64) { itr.k = -1 // Move to the correct container. - itr.citer, _ = itr.finder.FindIterator(highbits(seek)) + itr.citer, _ = itr.bitmap.Containers.Iterator(highbits(seek)) if !itr.citer.Next() { itr.c = nil return // eof @@ -7537,10 +7518,6 @@ func (c *Container) Difference(other *Container) *Container { return difference(c, other) } -func NewSliceContainers() *sliceContainers { - return newSliceContainers() -} - // Slice returns an array of the values in the container as uint16. // Do NOT modify the result; it could be the container's actual storage. func (c *Container) Slice() (r []uint16) { diff --git a/rrtx.go b/rrtx.go index d5504cecb..247ae77f1 100644 --- a/rrtx.go +++ b/rrtx.go @@ -15,9 +15,7 @@ package pilosa import ( - "bytes" "fmt" - "io" "os" "path/filepath" "sort" @@ -49,27 +47,10 @@ type RoaringTx struct { w *RoaringWrapper } -func (tx *RoaringTx) IsDone() (done bool) { - tx.mu.Lock() - done = tx.done - tx.mu.Unlock() - return -} - func (tx *RoaringTx) Type() string { return RoaringTxn } -func (tx *RoaringTx) Dump(short bool, shard uint64) { - o := tx.o - o.Shard = shard - fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys(short, false, o)) -} - -func (tx *RoaringTx) UseRowCache() bool { - return storage.EnableRowCache() -} - // based on view.openFragments() func roaringMapOfShards(optionalViewPath string) (shardMap map[uint64]bool, err error) { @@ -112,10 +93,6 @@ func roaringMapOfShards(optionalViewPath string) (shardMap map[uint64]bool, err return } -func (tx *RoaringTx) Pointer() string { - return fmt.Sprintf("%p", tx) -} - // NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE // the transaction Commits or Rollsback. func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { @@ -127,33 +104,16 @@ func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roa // ImportRoaringBits return values changed and rowSet will be inaccurate if // the data []byte is supplied. This mimics the traditional roaring-per-file // and should be faster. -func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { +func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { f, err := tx.getFragment(index, field, view, shard) if err != nil { return 0, nil, err } - if len(data) > 0 { - // changed and rowSet are ignored anyway when len(data) > 0; - // when we are called from fragment.fillFragmentFromArchive() - // which is the only place the data []byte is supplied. - // blueGreenTx also turns off the checks in this case. - return 0, nil, f.readStorageFromArchive(bytes.NewBuffer(data)) - } changed, rowSet, err = f.storage.ImportRoaringRawIterator(rit, clear, true, rowSize) return } -func (tx *RoaringTx) Readonly() bool { - return !tx.write -} - -func (tx *RoaringTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - frag, err := tx.getFragment(index, field, view, shard) - PanicOn(err) - frag.incrementOpN(changedN) -} - func (c *RoaringTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { return GenericApplyFilter(c, index, field, view, shard, ckey, filter) } @@ -279,15 +239,6 @@ func (tx *RoaringTx) Min(index, field, view string, shard uint64) (uint64, bool, return v, ok, nil } -func (tx *RoaringTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return err - } - b.UnionInPlace(others...) - return nil -} - func (tx *RoaringTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) { b, err := tx.bitmap(index, field, view, shard) if err != nil { @@ -381,33 +332,6 @@ func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.B return frag.storage, nil } -func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - file, err := os.Open(fragmentPathForRoaring) // open the fragment file - if err != nil { - return nil, -1, err - } - fi, err := file.Stat() - if err != nil { - return nil, -1, errors.Wrap(err, "statting") - } - sz = fi.Size() - r = file - return -} - -func (tx *RoaringTx) Group() *TxGroup { - return tx.o.Group -} - -func (tx *RoaringTx) Options() Txo { - return tx.o -} - -// Sn retreives the serial number of the Tx. -func (tx *RoaringTx) Sn() int64 { - return tx.sn -} - func roaringGetFieldView2Shards(idx *Index) (vs *FieldView2Shards, err error) { vs = NewFieldView2Shards() @@ -651,18 +575,12 @@ func (w *RoaringWrapper) CleanupTx(tx Tx) { return } r.done = true - - r.o.dbs.Cleanup(tx) // release the read/write lock. } func (w *RoaringWrapper) OpenListString() (r string) { return "RoaringWrapper.OpenListString() not yet implemented" } -func (w *RoaringWrapper) OpenSnList() (slc []int64) { - return nil -} - func (w *RoaringWrapper) CloseDB() error { return errors.New("CloseDB not supported in roaring") } @@ -721,21 +639,12 @@ func (w *RoaringWrapper) IsClosed() (closed bool) { return } -func (w *RoaringWrapper) DeleteDBPath(dbs *DBShard) (err error) { - //vv("RoaringWrapper.DeleteDBPath called on dbs = '%#v'", dbs) - path := dbs.pathForType(roaringTxn) - return os.RemoveAll(path) -} - func (w *RoaringWrapper) DeleteField(index, field, fieldPath string) error { //vv("RoaringWrapper.DeleteField(index = '%v', field = '%v', fieldPath = '%v'", index, field, fieldPath) // match txn sn count vs lmdb/etc. atomic.AddInt64(&globalNextTxSnRoaring, 1) - // under blue-green bolt_roaring, the directory will not be found, b/c bolt will have - // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: - // "If the path does not exist, RemoveAll returns nil (no error)" err := os.RemoveAll(fieldPath) if err != nil { return errors.Wrap(err, "removing directory") diff --git a/server.go b/server.go index 8912d38bb..63077302b 100644 --- a/server.go +++ b/server.go @@ -334,8 +334,8 @@ func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption { } // OptServerStorageConfig is a functional option on Server used to specify the -// transactional-storage backend to use, resulting in RoaringTx, RbfTx, -// BadgerTx, or a blueGreen* Tx being used for all Tx interface calls. +// transactional-storage backend to use, resulting in RoaringTx or RbfTx +// being used for all Tx interface calls. func OptServerStorageConfig(cfg *storage.Config) ServerOption { return func(s *Server) error { s.holderConfig.StorageConfig = cfg diff --git a/server/cluster_test.go b/server/cluster_test.go index 1f7d59b14..51b8c8ebf 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -142,19 +142,6 @@ func TestClusterResize_EmptyNodes(t *testing.T) { // Ensure that adding a node correctly resizes the cluster. func TestClusterResize_AddNode(t *testing.T) { - // Why are we skipping this test under blue-green with Roaring? - // - // We see red test: during resize during importRoaringBits - // PILOSA_STORAGE_BACKEND=rbf_roaring go test -v -tags=' shardwidth20' "-gcflags=all=-d=checkptr=0" -run TestClusterResize_AddNode/"ContinuousShards" - // green: - // PILOSA_STORAGE_BACKEND=roaring_rbf go test -v -tags=' shardwidth20' "-gcflags=all=-d=checkptr=0" -run TestClusterResize_AddNode/"ContinuousShards" - // - // but rbf_badger and badger_rbf are both green (use the same data values for containers). - // - // Conclude: roaring reads a different size of data []byte in (due to ops log) bits vs others (RBF, badger), so - // we can't do blue-green with roaring on this test. - skipTestUnderBlueGreenWithRoaring(t) - t.Run("NoData", func(t *testing.T) { clus := test.MustRunCluster(t, 3) defer clus.Close() @@ -344,8 +331,6 @@ func TestClusterResize_AddNode(t *testing.T) { // Ensure that adding a node correctly resizes the cluster. func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { - skipTestUnderBlueGreenWithRoaring(t) - t.Run("WithIndex", func(t *testing.T) { c := test.MustRunCluster(t, 3) defer c.Close() @@ -630,12 +615,3 @@ func TestClusterMutualTLS(t *testing.T) { t.Fatal(err) } } - -func skipTestUnderBlueGreenWithRoaring(t *testing.T) { - src := pilosa.CurrentBackend() - if strings.Contains(src, "_") { - if strings.Contains(src, "roaring") { - t.Skip("skip for roaring blue-green") - } - } -} diff --git a/server/config.go b/server/config.go index dc6ae08ae..344440ded 100644 --- a/server/config.go +++ b/server/config.go @@ -207,15 +207,8 @@ type Config struct { // Storage.Backend determines which Tx implementation the holder/Index will // use; one of the available transactional-storage engines. Choices are - // listed in the string constants below. Should be one of "roaring","bolt", - // "rbf", "bolt_roaring", "roaring_bolt", "rbf_roaring", "roaring_rbf", - // "bolt_rbf", "rbf_bolt", or any later addition. The engines with _ - // underscore indicate use of a blueGreenTx with a comparison of values back - // from each Tx method, and a panic if they differ. This is an effective - // test for consistency. If "rbf_roaring" is specified, then the roaring - // values are the ones actually returned from the blueGreenTx. If - // "roaring_rbf" is chosen, then the RBF values are the ones actually - // returned from the blueGreenTx. + // listed in the string constants below. Should be one of "roaring" or + // "rbf". Storage *storage.Config `toml:"storage"` // RowcacheOn, if true, turns on the row cache for all storage backends. diff --git a/stattx.go b/stattx.go index 9dc5ab5f9..65a87fdf3 100644 --- a/stattx.go +++ b/stattx.go @@ -16,7 +16,6 @@ package pilosa import ( "fmt" - "io" "math" "runtime" "sort" @@ -151,8 +150,7 @@ type kall int // constants for kall argument to callStats.add() const ( - kIncrementOpN kall = iota - kNewTxIterator + kNewTxIterator kall = iota kImportRoaringBits kRollback kCommit @@ -169,23 +167,14 @@ const ( kCount kMax kMin - kUnionInPlace kCountRange kOffsetRange - kRoaringBitmapReader - kSliceOfShards kLast // mark the end, always keep this last. The following aren't tracked atm: kType - kDump - kReadonly - kPointer - kUseRowCache ) func (k kall) String() string { switch k { - case kIncrementOpN: - return "kIncrementOpN" case kNewTxIterator: return "kNewTxIterator" case kImportRoaringBits: @@ -220,62 +209,21 @@ func (k kall) String() string { return "kMax" case kMin: return "kMin" - case kUnionInPlace: - return "kUnionInPlace" case kCountRange: return "kCountRange" case kOffsetRange: return "kOffsetRange" - case kRoaringBitmapReader: - return "kRoaringBitmapReader" - case kSliceOfShards: - return "kSliceOfShards" case kLast: return "kLast" case kType: return "kType" - case kDump: - return "kDump" - case kReadonly: - return "kReadonly" - case kPointer: - return "kPointer" - case kUseRowCache: - return "kUseRowCache" } PanicOn(fmt.Sprintf("unknown kall '%v'", int(k))) return "" } -var _ = newStatTx // happy linter -var _ = kPointer -var _ = kUseRowCache -var _ = kType -var _ = kDump -var _ = kReadonly - var _ Tx = (*statTx)(nil) -func (c *statTx) Group() *TxGroup { - return c.b.Group() -} - -func (c *statTx) Options() Txo { - return c.b.Options() -} - -//IncrementOpN -func (c *statTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - me := kIncrementOpN - - t0 := time.Now() - defer func() { - c.stats.add(me, time.Since(t0)) - }() - - c.b.IncrementOpN(index, field, view, shard, changedN) -} - func (c *statTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { me := kNewTxIterator @@ -286,7 +234,7 @@ func (c *statTx) NewTxIterator(index, field, view string, shard uint64) *roaring return c.b.NewTxIterator(index, field, view, shard) } -func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { +func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { me := kImportRoaringBits t0 := time.Now() @@ -299,25 +247,7 @@ func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit PanicOn(r) } }() - return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) -} - -func (c *statTx) Dump(short bool, shard uint64) { - c.b.Dump(short, shard) -} - -func (c *statTx) Readonly() bool { - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Readonly() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.Readonly() -} - -func (tx *statTx) Pointer() string { - return fmt.Sprintf("%p", tx) + return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize) } func (c *statTx) Rollback() { @@ -421,14 +351,6 @@ func (c *statTx) RemoveContainer(index, field, view string, shard uint64, key ui return c.b.RemoveContainer(index, field, view, shard, key) } -func (c *statTx) UseRowCache() bool { - return c.b.UseRowCache() -} - -func (c *statTx) IsDone() (done bool) { - return c.b.IsDone() -} - func (c *statTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { me := kAdd @@ -586,23 +508,6 @@ func (c *statTx) Min(index, field, view string, shard uint64) (uint64, bool, err return c.b.Min(index, field, view, shard) } -func (c *statTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - me := kUnionInPlace - - t0 := time.Now() - defer func() { - c.stats.add(me, time.Since(t0)) - }() - - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.UnionInPlace(index, field, view, shard, others...) -} - func (c *statTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { me := kCountRange @@ -636,32 +541,10 @@ func (c *statTx) OffsetRange(index, field, view string, shard, offset, start, en return c.b.OffsetRange(index, field, view, shard, offset, start, end) } -func (c *statTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - me := kRoaringBitmapReader - - t0 := time.Now() - defer func() { - c.stats.add(me, time.Since(t0)) - }() - - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) -} - func (c *statTx) Type() string { return c.b.Type() } -// Sn retreives the serial number of the Tx. -func (c *statTx) Sn() int64 { - return c.b.Sn() -} - func (c *statTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) { return c.b.GetSortedFieldViewList(idx, shard) } diff --git a/storage/cache.go b/storage/cache.go index 7fda5af83..fe65a90f3 100644 --- a/storage/cache.go +++ b/storage/cache.go @@ -31,6 +31,6 @@ func SetRowCacheOn(on bool) { } } -func EnableRowCache() bool { +func RowCacheEnabled() bool { return atomic.LoadInt64(&enableRowcache) == 1 } diff --git a/tx.go b/tx.go index 13fd9fded..767969671 100644 --- a/tx.go +++ b/tx.go @@ -15,8 +15,6 @@ package pilosa import ( - "io" - "github.com/molecula/featurebase/v2/roaring" txkey "github.com/molecula/featurebase/v2/short_txkey" //txkey "github.com/molecula/featurebase/v2/txkey" @@ -45,8 +43,8 @@ const writable = true // that have not been committed. type Tx interface { - // Type returns "roaring", "rbf", "bolt", "badger_roaring", or one of the other - // blue-green Tx types at the top of txfactory.go + // Type returns "roaring", "rbf", or one of the other + // Tx types at the top of txfactory.go Type() string // Rollback must be called the end of read-only transactions. Either @@ -65,32 +63,6 @@ type Tx interface { // Commit makes the updates in the Tx visible to subsequent transactions. Commit() error - // IsDone must return true if Rollback() or Commit() has already - // been called. Otherwise it must return false. This allows - // DBWrapper.CleanupTx(tx Tx) to be idempotent. - IsDone() bool - - // Readonly returns the flag this transaction was created with - // during NewTx. If the transaction is writable, it will return false. - Readonly() bool - - // UseRowCache is used by fragment.go unprotectedRow() to determine - // dynamically at runtime if RoaringTx - // are in use, which for continuity wants to continue to use the - // rowCache, or if other storage engines (RBF, Badger) are in - // use, which will mean that the bitmap data stored by the - // rowCache can disappear as it is un-mmap-ed, causing crashes. - UseRowCache() bool - - // IncrementOpN updates internal statistics with the changedN provided. - IncrementOpN(index, field, view string, shard uint64, changedN int) - - // Pointer gives us a memory address for the underlying - // transaction for debugging. - // It is public because we use it in roaring to report invalid - // container memory access outside of a transaction. - Pointer() string - // NewTxIterator returns it, a *roaring.Iterator whose it.Next() will // successively return each uint64 stored in the conceptual roaring.Bitmap // for the specified fragment. @@ -103,8 +75,7 @@ type Tx interface { // Return value 'found' is true when the ckey container was present. // ckey of 0 gives all containers (in the fragment). // - // ContainerIterator must not have side-effects. blueGreenTx will - // call it at the very beginning of commit to verify db contents. + // ContainerIterator must not have side-effects. // // citer.Close() must be called when the client is done using it. ContainerIterator(index, field, view string, shard uint64, ckey uint64) (citer roaring.ContainerIterator, found bool, err error) @@ -154,9 +125,6 @@ type Tx interface { // Min Min(index, field, view string, shard uint64) (uint64, bool, error) - // UnionInPlace - UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error - // CountRange CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) @@ -170,31 +138,9 @@ type Tx interface { // If clear is true, the bits from rit are cleared, otherwise they are set in the // specifed fragment. // - // The data argument can be nil, its ignored for RBF/BadgerTx. It is supplied to - // RoaringTx.ImportRoaringBits() in fragment.go fragment.fillFragmentFromArchive() - // to do the traditional fragment.readStorageFromArchive() which - // does some in memory field/view/fragment metadata updates. - // It makes blueGreenTx testing viable too. - // // ImportRoaringBits return values changed and rowSet may be inaccurate if // the data []byte is supplied (the RoaringTx implementation neglects this for speed). - ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) - - RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) - - // Group returns nil or the TxGroup that this Tx is a part of. - Group() *TxGroup - - // Dump is for debugging, what does this Tx see as its database? - Dump(short bool, shard uint64) - - // Options returns the options used to create this Tx. This - // can be implementd by embedding Txo, and Txo provides the - // Options() method. - Options() Txo - - // Sn retreives the serial number of the Tx. - Sn() int64 + ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) // GetSortedFieldViewList gets the set of FieldView(s) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) @@ -202,51 +148,6 @@ type Tx interface { GetFieldSizeBytes(index, field string) (uint64, error) } -// Closer is used by Finders -type Closer interface { - Close() -} - -type Dumper interface { - // Dump is for debugging, what does this Tx see as its database? - AllDump() -} - -// TxStore has operations that will create and commit multiple -// Tx on a backing store. -type TxStore interface { - - // DeleteFragment deletes all the containers in a fragment. - // - // This is not in a Tx because it will often do too many deletes for a single - // transaction, and clients would be suprised to find their Tx had already - // been commited and they are getting an error on double-Commit. - // Instead each TxStore implementation creates and commits as many - // transactions as needed. - // - // Argument frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil. - // If not nil, it must be of type *fragment. If frag is supplied, then - // index must be equal to frag.index, field equal to frag.field, view equal - // to frag.view, and shard equal to frag.shard. - // - DeleteFragment(index, field, view string, shard uint64, frag interface{}) error - - DeleteField(index, field string) error - - // Close shuts down the database. - Close() error -} - -// RawRoaringData used by ImportRoaringBits. -// must be consumable by roaring.newRoaringIterator() -type RawRoaringData struct { - data []byte -} - -func (rr *RawRoaringData) Iterator() (roaring.RoaringIterator, error) { - return roaring.NewRoaringIterator(rr.data) -} - // GenericApplyFilter implements ApplyFilter in terms of tx.ContainerIterator, // as a convenience if a Tx backend hasn't implemented this new function yet. func GenericApplyFilter(tx Tx, index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { diff --git a/txfactory.go b/txfactory.go index 7c0d06add..e28a6fb03 100644 --- a/txfactory.go +++ b/txfactory.go @@ -16,32 +16,22 @@ package pilosa import ( "fmt" - "io" "os" "path" "path/filepath" - "runtime" "strconv" "strings" "sync" - "syscall" - "text/tabwriter" - "github.com/molecula/featurebase/v2/hash" - "github.com/molecula/featurebase/v2/roaring" - txkey "github.com/molecula/featurebase/v2/short_txkey" - "github.com/molecula/featurebase/v2/storage" "github.com/molecula/featurebase/v2/testhook" . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck "github.com/pkg/errors" - "github.com/zeebo/blake3" ) // public strings that pilosa/server/config.go can reference const ( RoaringTxn string = "roaring" RBFTxn string = "rbf" - BoltTxn string = "bolt" ) // DetectMemAccessPastTx true helps us catch places in api and executor @@ -267,7 +257,7 @@ func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) { // qcx.write reflects the top executor determination // if a write will be done at the end, so we upgrade // the "local" read Tx to be writes, so that they - // don't deadlock against themselves under blue-green. + // don't deadlock against themselves. o.Write = o.Write || qcx.write // In general, we make ALL write transactions local, and never reuse them @@ -305,9 +295,8 @@ func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) { if already { return } - o.Group = qcx.Grp tx = qcx.Txf.NewTx(o) - qcx.Grp.AddTx(tx) + qcx.Grp.AddTx(tx, o) return } @@ -351,7 +340,6 @@ func (qcx *Qcx) StartAtomicWriteTx(o Txo) { // new Tx needed tx := qcx.Txf.NewTx(o) qcx.RequiredForAtomicWriteTx = &tx - o := tx.Options() qcx.RequiredTxo = &o return } @@ -374,55 +362,23 @@ func (qcx *Qcx) StartAtomicWriteTx(o Txo) { } } -func (qcx *Qcx) SetRequiredForAtomicWriteTx(tx Tx) { - if tx == nil || NilInside(tx) { - PanicOn("cannot set nil tx in SetRequiredForAtomicWriteTx") - } - qcx.mu.Lock() - qcx.RequiredForAtomicWriteTx = &tx - o := tx.Options() - qcx.RequiredTxo = &o - qcx.mu.Unlock() -} - -func (qcx *Qcx) ClearRequiredForAtomicWriteTx() { - qcx.mu.Lock() - qcx.RequiredForAtomicWriteTx = nil - qcx.RequiredTxo = nil - qcx.mu.Unlock() -} - func (qcx *Qcx) ListOpenTx() string { return qcx.Grp.String() } // TxFactory abstracts the creation of Tx interface-level -// transactions so that RBF, BoltDB, or Roaring-fragment-files, or several +// transactions so that RBF, or Roaring-fragment-files, or several // of these at once in parallel, is used as the storage and transction layer. type TxFactory struct { typeOfTx string - mu sync.Mutex - - types []txtype // blue-green split individually here + typ txtype dbsClosed bool // idemopotent CloseDB() dbPerShard *DBPerShard holder *Holder - - blueGreenReg *blueGreenRegistry - - // allow holder to activate blue-green checking only - // once we have synced both sides at start up time. - blueGreenOff bool - - isBlueGreen bool -} - -func (f *TxFactory) Types() []txtype { - return f.types } // integer types for fast switch{} @@ -432,7 +388,6 @@ const ( noneTxn txtype = 0 roaringTxn txtype = 1 // these don't really have any transactions rbfTxn txtype = 2 - boltTxn txtype = 4 ) // DirectoryName just returns a string version of the transaction type. We @@ -445,73 +400,41 @@ func (ty txtype) DirectoryName() string { return "roaring" case rbfTxn: return "rbf" - case boltTxn: - return "boltdb" } PanicOn(fmt.Sprintf("unkown txtype %v", int(ty))) return "" } func (txf *TxFactory) NeedsSnapshot() (b bool) { - for _, ty := range txf.types { - switch ty { - case roaringTxn: - b = true - return - } - } - return + return txf.typ == roaringTxn } -func MustBackendToTxtype(backend string) (types []txtype) { - var srcs []string +func MustBackendToTxtype(backend string) (typ txtype) { if strings.Contains(backend, "_") { - srcs = strings.Split(backend, "_") - if len(srcs) != 2 { - PanicOn("only two blue-green comparisons permitted") - } - } else { - srcs = append(srcs, backend) + panic("blue-green comparisons removed") } - for i, s := range srcs { - switch s { - case RoaringTxn: // "roaring" - types = append(types, roaringTxn) - case RBFTxn: // "rbf" - types = append(types, rbfTxn) - case BoltTxn: // "bolt" - types = append(types, boltTxn) - default: - PanicOn(fmt.Sprintf("unknown backend '%v'", s)) - } - if i == 1 { - if types[1] == types[0] { - PanicOn(fmt.Sprintf("cannot blue-green the same backend on both arms: '%v'", s)) - } - } + switch backend { + case RoaringTxn: // "roaring" + return roaringTxn + case RBFTxn: // "rbf" + return rbfTxn } - return + panic(fmt.Sprintf("unknown backend '%v'", backend)) } // NewTxFactory always opens an existing database. If you // want to a fresh database, os.RemoveAll on dir/name ahead of time. // We always store files in a subdir of holderDir. func NewTxFactory(backend string, holderDir string, holder *Holder) (f *TxFactory, err error) { - types := MustBackendToTxtype(backend) + typ := MustBackendToTxtype(backend) f = &TxFactory{ - types: types, + typ: typ, typeOfTx: backend, holder: holder, } - if len(types) == 2 { - f.blueGreenReg = newBlueGreenReg(types) - f.isBlueGreen = true - // blue-green can never use the rowCache. - storage.SetRowCacheOn(false) - } - f.dbPerShard = f.NewDBPerShard(types, holderDir, holder) + f.dbPerShard = f.NewDBPerShard(typ, holderDir, holder) if f.hasRBF() { holder.Logger.Infof("rbf config = %#v", holder.cfg.RBFConfig) @@ -526,15 +449,6 @@ func (f *TxFactory) Open() error { return f.dbPerShard.LoadExistingDBs() } -// UseRowCache can be more "global" than Tx at the moment, because -// we are sharing the same bool flag in rbf at the moment. If -// this changes then fragment.openStorage() will need a new way -// to determine if it should use the rowCache. Currently it -// doesn't have a tx Tx parameter, so we use the Txf instead. -func (f *TxFactory) UseRowCache() bool { - return storage.EnableRowCache() -} - // Txo holds the transaction options type Txo struct { Write bool @@ -544,23 +458,14 @@ type Txo struct { Shard uint64 dbs *DBShard - per *DBPerShard - - Group *TxGroup - - blueGreenOff bool -} - -func (o Txo) String() string { - return fmt.Sprintf("Txo{Write:%v, Index:%v Shard:%v Group:%p}", o.Write, o.Index.name, o.Shard, o.Group) } func (f *TxFactory) TxType() string { return f.typeOfTx } -func (f *TxFactory) TxTypes() []txtype { - return f.types +func (f *TxFactory) TxTyp() txtype { + return f.typ } func (f *TxFactory) DeleteIndex(name string) (err error) { @@ -577,10 +482,6 @@ func (f *TxFactory) DeleteFragmentFromStore( return f.dbPerShard.DeleteFragment(index, field, view, shard, frag) } -func (f *TxFactory) DumpAll() { - f.dbPerShard.DumpAll() -} - // IndexUsageDetails computes the sum of filesizes used by the node, broken down // by index, field, fragments and keys. func (f *TxFactory) IndexUsageDetails(isClosing func() bool) (map[string]IndexUsage, uint64, error) { @@ -769,7 +670,6 @@ func directoryUsage(fname string, recursive bool) (uint64, error) { // CloseIndex is a no-op. This seems to be in place for debugging purposes. func (f *TxFactory) CloseIndex(idx *Index) error { - //idx.Dump("CloseIndex") return nil } @@ -790,24 +690,24 @@ func init() { } } -// TxGroup holds a set of read and a set of write transactions -// that will en-mass have Rollback() (for the read set) and -// Commit() (for the write set) called on +// TxGroup holds a set of read transactions +// that will en-mass have Rollback() (for the read set) called on // them when TxGroup.Finish() is invoked. // Alternatively, TxGroup.Abort() will call Rollback() // on all Tx group memebers. +// +// It used to have writes but we never actually used that because +// of the Qcx needing to make every commit get its own transaction. type TxGroup struct { mu sync.Mutex fac *TxFactory reads []Tx - writes []Tx finished bool all map[grpkey]Tx } type grpkey struct { - write bool index string shard uint64 } @@ -822,7 +722,7 @@ func (g *TxGroup) AlreadyHaveTx(o Txo) (tx Tx, already bool) { mustHaveIndexShard(&o) g.mu.Lock() defer g.mu.Unlock() - key := grpkey{write: o.Write, index: o.Index.name, shard: o.Shard} + key := grpkey{index: o.Index.name, shard: o.Shard} tx, already = g.all[key] return } @@ -830,21 +730,14 @@ func (g *TxGroup) AlreadyHaveTx(o Txo) (tx Tx, already bool) { func (g *TxGroup) String() (r string) { g.mu.Lock() defer g.mu.Unlock() - if len(g.reads) == 0 && len(g.writes) == 0 { + if len(g.reads) == 0 { return "" } - - i := 0 r += "\n" - for _, tx := range g.reads { - r += fmt.Sprintf("[%v]read: _sn_ %v %v, \n", i, tx.Sn(), tx.Options()) - i++ + for i, tx := range g.reads { + r += fmt.Sprintf("[%v]read: %#v,\n", i, tx) } - for _, tx := range g.writes { - r += fmt.Sprintf("[%v]write: _sn_ %v %v, \n", i, tx.Sn(), tx.Options()) - i++ - } - return + return r } // NewTxGroup @@ -857,7 +750,7 @@ func (f *TxFactory) NewTxGroup() (g *TxGroup) { } // AddTx adds tx to the group. -func (g *TxGroup) AddTx(tx Tx) { +func (g *TxGroup) AddTx(tx Tx, o Txo) { g.mu.Lock() defer g.mu.Unlock() if g.finished { @@ -867,15 +760,9 @@ func (g *TxGroup) AddTx(tx Tx) { PanicOn("Cannot add nil Tx to TxGroup") } - if tx.Readonly() { - g.reads = append(g.reads, tx) - } else { - g.writes = append(g.writes, tx) - } - o := tx.Options() - mustHaveIndexShard(&o) + g.reads = append(g.reads, tx) - key := grpkey{write: o.Write, index: o.Index.name, shard: o.Shard} + key := grpkey{index: o.Index.name, shard: o.Shard} prior, ok := g.all[key] if ok { PanicOn(fmt.Sprintf("already have Tx in group for this, we should have re-used it! prior is '%v'; tx='%v'", prior, tx)) @@ -893,15 +780,6 @@ func (g *TxGroup) FinishGroup() (err error) { PanicOn("in TxGroup.Finish(): TxGroup already finished") } g.finished = true - for i, tx := range g.writes { - _ = i - err0 := tx.Commit() - if err0 != nil { - if err == nil { - err = err0 // keep the first error, but Commit them all. - } - } - } for _, r := range g.reads { r.Rollback() } @@ -923,9 +801,6 @@ func (g *TxGroup) AbortGroup() { for _, r := range g.reads { r.Rollback() } - for _, tx := range g.writes { - tx.Rollback() - } } func (f *TxFactory) NewTx(o Txo) (txn Tx) { @@ -935,12 +810,6 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) { } }() - if f.isBlueGreen { - f.mu.Lock() - o.blueGreenOff = f.blueGreenOff - f.mu.Unlock() - } - indexName := "" if o.Index != nil { indexName = o.Index.name @@ -963,10 +832,8 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) { if dbs.Shard != o.Shard { PanicOn(fmt.Sprintf("asked for o.Shard=%v but got dbs.Shard=%v", int(o.Shard), int(dbs.Shard))) } - //vv("got dbs='%p' for o.Index='%v'; shard='%v'; dbs.types='%#v'; dbs.W='%#v'", dbs, o.Index.name, o.Shard, dbs.types, dbs.W) - - o.dbs = dbs // our specific database per shard. - o.per = f.dbPerShard // for top level debug Dumps + //vv("got dbs='%p' for o.Index='%v'; shard='%v'; dbs.typ='%#v'; dbs.W='%#v'", dbs, o.Index.name, o.Shard, dbs.typ, dbs.W) + o.dbs = dbs tx, err := dbs.NewTx(o.Write, indexName, o) if err != nil { @@ -984,8 +851,6 @@ func (ty txtype) String() string { return "roaring" case rbfTxn: return "rbf" - case boltTxn: - return "bolt" } PanicOn(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty))) return "" @@ -1022,147 +887,6 @@ func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64, return } -// hashOnly means only show the value hash, not the content bits. -// showOps means display the ops log. -func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool, o Txo) (r string) { - paths, err := listFilesUnderDir(idx.path, false, "", true) - PanicOn(err) - index := idx.name - - r = "allkeys:[\n" - n := 0 - for _, relpath := range paths { - field, view, shard, err := fragmentSpecFromRoaringPath(relpath) - if err != nil { - continue // ignore .meta paths - } - if shard != o.Shard { - continue // only print the shard the Txo is on. - } - abspath := idx.path + sep + relpath - - s, _, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard, showOps, hashOnly, os.Stdout) - PanicOn(err) - //r += fmt.Sprintf("path:'%v' fragment contains:\n") + s - //if s == "" { - //s = "" - //} - r += s - n++ - } - if n == 0 { - return "" - } - // note that we can have a bitmap present, but it can be empty - r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + "\n" - - return "roaring-" + r -} - -func RoaringFragmentChecksum(path string, index, field, view string, shard uint64) (r string, hotbits int) { - defer func() { - r := recover() - if r != nil { - PanicOn(fmt.Sprintf("caught PanicOn on path='%v', index='%v', field='%v', view='%v', shard='%v': %v", - path, index, field, view, shard, r)) - } - }() - hasher := blake3.New() - showOps := false - hashOnly := true - hash, hotbits, err := stringifiedRawRoaringFragment(path, index, field, view, shard, showOps, hashOnly, hasher) - PanicOn(err) - fmt.Fprintf(hasher, "%v/%v/%v/%v/%v", index, field, view, shard, hash) - var buf [16]byte - _, _ = hasher.Digest().Read(buf[0:]) - return fmt.Sprintf("%x", buf), hotbits - -} - -func stringifiedRawRoaringFragment(path string, index, field, view string, shard uint64, showOps, hashOnly bool, w io.Writer) (r string, hotbits int, err error) { - - var info roaring.BitmapInfo - _ = info - var f *os.File - f, err = os.Open(path) - PanicOn(err) - if err != nil { - return - } - - var fi os.FileInfo - fi, err = f.Stat() - PanicOn(err) - if err != nil { - return - } - // Memory map the file. - data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) - if err != nil { - err = errors.Wrap(err, "mmapping") - return - } - defer func() { - err := syscall.Munmap(data) - if err != nil { - PanicOn(fmt.Errorf("loadRawRoaringContainer: munmap failed: %v", err)) - } - PanicOn(f.Close()) - }() - - // Attach the mmap file to the bitmap. - var rbm *roaring.Bitmap - rbm, _, err = roaring.InspectBinary(data, true, &info) - if err != nil { - err = errors.Wrap(err, "inspecting") - return - } - - //cmd.DisplayInfo(info) - // inlined - if showOps { - pC := pointerContext{ - from: info.From, - to: info.To, - } - if info.ContainerCount > 0 { - printContainers(w, info, pC) - } - if info.Ops > 0 { - printOps(w, info) - } - } - - citer, found := rbm.Containers.Iterator(0) - _ = found // probably gonna use just the Ops log instead, so don't PanicOn if !found. - - for citer.Next() { - ckey, ct := citer.Value() - by := containerToBytes(ct) - hash := hash.Blake3sum16(by) - - cts := roaring.NewSliceContainers() - cts.Put(ckey, ct) - rbm := &roaring.Bitmap{Containers: cts} - - var srbm string - if !hashOnly { - srbm = BitmapAsString(rbm) - } - - bkey := txkey.ToString(txkey.Key(index, field, view, shard, ckey)) - - n := ct.N() - hotbits += int(n) - r += fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, n) - if !hashOnly { - r += " ......." + srbm + "\n" - } - } - - return -} - // listFilesUnderDir returns the paths of files found under directory root. // If includeRoot is true, it returns the full path, otherwise paths are relative to root. // If requriedSuffix is supplied, the returned file paths will end in that, @@ -1218,130 +942,6 @@ func fileSize(name string) (int64, error) { return fi.Size(), nil } -var _ = fileSize // happy linter - -func containerToBytes(ct *roaring.Container) []byte { - ty := roaring.ContainerType(ct) - switch ty { - case roaring.ContainerNil: - PanicOn("nil roaring.Container") - case roaring.ContainerArray: - return fromArray16(roaring.AsArray(ct)) - case roaring.ContainerBitmap: - return fromArray64(roaring.AsBitmap(ct)) - case roaring.ContainerRun: - return fromInterval16(roaring.AsRuns(ct)) - } - PanicOn(fmt.Sprintf("unknown roaring.Container type '%v'", int(ty))) - return nil -} - -type pointerContext struct { - from, to uintptr -} - -func printOps(w io.Writer, info roaring.BitmapInfo) { - fmt.Fprintln(w, " Ops:") - tw := tabwriter.NewWriter(w, 0, 8, 0, '\t', 0) - fmt.Fprintf(tw, " \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE") - printed := 0 - for _, op := range info.OpDetails { - fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size) - printed++ - } - tw.Flush() -} - -func (p *pointerContext) pretty(c roaring.ContainerInfo) string { - var pointer string - if c.Mapped { - if c.Pointer >= p.from && c.Pointer < p.to { - pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from) - } else { - pointer = fmt.Sprintf("!0x%x!", c.Pointer) - } - } else { - pointer = fmt.Sprintf("0x%x", c.Pointer) - } - return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer) -} - -// stolen from ctl/inspect.go -func printContainers(w io.Writer, info roaring.BitmapInfo, pC pointerContext) { - fmt.Fprintln(w, " Containers:") - tw := tabwriter.NewWriter(w, 0, 8, 0, '\t', 0) - fmt.Fprintf(tw, " \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n") - fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS") - c1s := info.Containers - c2s := info.OpContainers - l1 := len(c1s) - l2 := len(c2s) - i1 := 0 - i2 := 0 - var c1, c2 roaring.ContainerInfo - c1.Key = ^uint64(0) - c2.Key = ^uint64(0) - c1e := false - c2e := false - if i1 < l1 { - c1 = c1s[i1] - i1++ - c1e = true - } - if i2 < l2 { - c2 = c2s[i2] - i2++ - c2e = true - } - printed := 0 - for c1e || c2e { - c1used := false - c2used := false - var key uint64 - c1fmt := "-\t\t\t" - c2fmt := "-\t\t\t" - // If c2 exists, we'll always prefer its flags, - // if it doesn't, this gets overwritten. - flags := c2.Flags - if !c2e || (c1e && c1.Key < c2.Key) { - c1fmt = pC.pretty(c1) - key = c1.Key - c1used = true - flags = c1.Flags - } else if !c1e || (c2e && c2.Key < c1.Key) { - c2fmt = pC.pretty(c2) - key = c2.Key - c2used = true - } else { - // c1e and c2e both set, and neither key is < the other. - c1fmt = pC.pretty(c1) - c2fmt = pC.pretty(c2) - key = c1.Key - c1used = true - c2used = true - } - if c1used { - if i1 < l1 { - c1 = c1s[i1] - i1++ - } else { - c1e = false - } - } - if c2used { - if i2 < l2 { - c2 = c2s[i2] - i2++ - } else { - c2e = false - } - } - fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags) - printed++ - } - tw.Flush() -} - var _ = anyGlobalDBWrappersStillOpen // happy linter func anyGlobalDBWrappersStillOpen() bool { @@ -1351,225 +951,19 @@ func anyGlobalDBWrappersStillOpen() bool { if globalRbfDBReg.Size() != 0 { return true } - if globalBoltReg.Size() != 0 { - return true - } return false } -func (f *TxFactory) blueGreenOnIfRunningBlueGreen() { - if len(f.types) == 2 { - f.blueGreenOff = false - } -} - -func (f *TxFactory) blueGreenOffIfRunningBlueGreen() { - if len(f.types) == 2 { - f.blueGreenOff = true - } -} - func (f *TxFactory) hasRoaring() bool { - return f.types[0] == roaringTxn || (len(f.types) > 1 && f.types[1] == roaringTxn) + return f.typ == roaringTxn } func (f *TxFactory) hasRBF() bool { - return f.types[0] == rbfTxn || (len(f.types) > 1 && f.types[1] == rbfTxn) + return f.typ == rbfTxn } var _ = (&TxFactory{}).hasRoaring // happy linter -func (f *TxFactory) blueHasData() (hasData bool, err error) { - if len(f.types) != 2 { - return false, nil - } - return f.dbPerShard.HasData(0) -} - -func (f *TxFactory) greenHasData() (hasData bool, err error) { - n := len(f.types) - switch n { - case 1: - return f.dbPerShard.HasData(0) - case 2: - return f.dbPerShard.HasData(1) - } - err = fmt.Errorf("unsupported len(f.types): %v; must be 1 or 2", n) - PanicOn(err) - return -} - -// green2blue is called at the very end of Holder.Open(), so -// we know that the holder is ready to go, knowing its holder.Indexes(), fields, -// view, shards, and other metadata if any. -// -// Called by test Test_TxFactory_UpdateBlueFromGreen_OnStartup() in -// txfactory_internal_test.go as well. -// -// This is a noop if we aren't running under a blue_green PILOSA_STORAGE_BACKEND. -func (f *TxFactory) green2blue(holder *Holder) (err0 error) { - - // Holder.Open will always call us, even without blue_green. Which is fine. - // We are just a no-op in that case. - if len(f.types) != 2 { - return nil - } - - holder.Logger.Infof("green2blue analysis begins.") - - blueDest := f.types[0] - greenSrc := f.types[1] - - if blueDest == roaringTxn { - return fmt.Errorf("error: cannot migrate to 'roaring': not implemented") - } - - idxs := holder.Indexes() - - verifyInsteadOfCopy := false - - blueHasData, err := f.blueHasData() - if err != nil { - return errors.Wrap(err, "TxFactory.green2blue f.blueHasData()") - } - - greenHasData, err := f.greenHasData() - if err != nil { - return errors.Wrap(err, "TxFactory.green2blue f.greenHasData()") - } - if !blueHasData && !greenHasData { - holder.Logger.Infof("no data in blue or green. No migration or verification to do") - return nil - } - // INVAR: blue has data. - if !greenHasData { - holder.Logger.Errorf("cannot migrate from green '%v' because it has no data in it", greenSrc) - return fmt.Errorf("error: cannot migrate from green '%v' because it has no data in it", greenSrc) - } - - nGoro := runtime.NumCPU() - if nGoro < 5 { - // try to get some overlapped IO - nGoro = 5 - } - pj := newParallelJobs(nGoro) - - action := "verify" - if blueHasData { - verifyInsteadOfCopy = true - defer holder.Logger.Infof("bitmap-backend verification done : %v compared to %v", blueDest, greenSrc) - } else { - action = "migrate" - holder.Logger.Infof("bitmap-backend migration starting: populating %v from %v with %v threads", blueDest, greenSrc, nGoro) - defer holder.Logger.Infof("bitmap-backend migration done : populated %v from %v", blueDest, greenSrc) - } - firstPjobStarted := false - -indexloop: - for k, idx := range idxs { - - // scan directories - blueShards, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(blueDest, idx, "", false) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v') error fetching blueShards", idx.name)) - } - - // scan directories - greenShards, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(greenSrc, idx, "", true) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v') error fetching greenShards", idx.name)) - } - - if verifyInsteadOfCopy { - diff := f.shardSetDiff(blueShards, greenShards) - if diff != "" { - return fmt.Errorf("verifyInsteadOfCopy true, blue[%v]=%#v and green[%v]=%#v have different shards for index '%v': '%v'; stack=\n%v", blueDest, blueShards, greenSrc, greenShards, idx.name, diff, Stack()) - } - - // can also check against meta data - shards := idx.AvailableShards(localOnly).Slice() - meta := make(map[uint64]bool) - for _, shard := range shards { - meta[shard] = true - } - diff2 := f.shardSetDiff(greenShards, meta) - if diff2 != "" { - return fmt.Errorf("green[%v] = '%#v' and meta data '%#v' have different shards for index '%v': %v", greenSrc, greenShards, shards, idx.name, diff2) - } - } - - shardNum := 0 - for shard := range greenShards { - shardNum++ - shnum := shardNum - idx := idx - shard := shard - k := k - fun := func(worker int) error { - - dbs, err := f.dbPerShard.GetDBShard(idx.name, shard, idx) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v', shard='%v')", idx.name, int(shard))) - } - - holder.Logger.Infof("%v progress on index '%v' (%v of %v): on shard '%v' (%v of %v) [worker %v]", - action, idx.name, k+1, len(idxs), shard, shnum, len(greenShards), worker) - - if verifyInsteadOfCopy { - // verify all containers - err = dbs.verifyBlueEqualsGreen() - if err != nil { - return errors.Wrap(err, - fmt.Sprintf("dbs.verifyBlueEqualsGreen(blue='%v', "+ - "green='%v') for index='%v', shard='%v'", - blueDest, greenSrc, idx.name, int(shard))) - } - } else { - // the main copy work - err = dbs.populateBlueFromGreen() - if err != nil { - return errors.Wrap(err, - fmt.Sprintf("dbs.copyGreenToBlue(blue='%v', "+ - "green='%v') for index='%v', shard='%v'", - blueDest, greenSrc, idx.name, int(shard))) - } - } - return nil - } // end of fun definition - - if !pj.run(fun) { - break indexloop - } - if !firstPjobStarted { - firstPjobStarted = true - defer func() { - err1 := pj.waitForFinish() - if err0 == nil { - err0 = err1 - } - }() - } - } - } - return nil -} - -func (f *TxFactory) shardSetDiff(blueShards, greenShards map[uint64]bool) (diff string) { - nb := len(blueShards) - ng := len(greenShards) - if nb != ng { - diff = fmt.Sprintf("blueShard[%v] count = %v; greenShard[%v] count = %v; ", f.types[0], nb, f.types[1], ng) - } - bmg := mapDiff(blueShards, greenShards) // get blue - green - gmb := mapDiff(greenShards, blueShards) // get green - blue - - if len(bmg) == 0 && len(gmb) == 0 { - return "" - } - diff += fmt.Sprintf("shard diff: blueMinusGreen shards: '%#v'; greenMinusBlue shards: '%#v'", bmg, gmb) - return -} - func (f *TxFactory) GetDBShardPath(index string, shard uint64, idx *Index, ty txtype, write bool) (shardPath string, err error) { dbs, err := f.dbPerShard.GetDBShard(index, shard, idx) if err != nil { diff --git a/txfactory_internal_test.go b/txfactory_internal_test.go index b310dbdfe..9cfa9badb 100644 --- a/txfactory_internal_test.go +++ b/txfactory_internal_test.go @@ -15,393 +15,14 @@ package pilosa import ( - "context" - "fmt" - "os" "testing" - "time" - - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck ) -func Test_TxFactory_Qcx_query_context(t *testing.T) { - src := CurrentBackend() - if src == "rbf" || src == "bolt" { - // ok - } else { - t.Skip("this test only for rbf and bolt") - } - - shard := uint64(0) - f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, shard, "") - defer f.Clean(t) - tx.Rollback() - - barrier := NewBarrier() - defer barrier.Close() - - done := make(chan bool) - - setter := func(k int) { - for i := 0; ; i++ { - barrier.WaitAtGate(0) - select { - case <-done: - return - default: - } - // add to the group txn on the txf. - qcx := idx.holder.txf.NewQcx() - - tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: idx, Shard: f.shard}) - PanicOn(err) - - // Set bits on the fragment. - if _, err := f.setBit(tx, 120, 1); err != nil { - panic(err) - } else if _, err := f.setBit(tx, 120, 6); err != nil { - panic(err) - } else if _, err := f.setBit(tx, 121, 0); err != nil { - panic(err) - } - // should have two containers set in the fragment. - - // Verify counts on rows. - if n := f.mustRow(tx, 120).Count(); n != 2 { - panic(fmt.Sprintf("unexpected count: %d", n)) - } else if n := f.mustRow(tx, 121).Count(); n != 1 { - panic(fmt.Sprintf("unexpected count: %d", n)) - } - finisher(nil) // hit the write tx.Commit path - // commit the change, and verify it is still there - PanicOn(qcx.Finish()) - qcx.Reset() - - tx, finread, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - PanicOn(err) - if n := f.mustRow(tx, 120).Count(); n != 2 { - panic(fmt.Sprintf("unexpected count (reopen): %d", n)) - } else if n := f.mustRow(tx, 121).Count(); n != 1 { - panic(fmt.Sprintf("unexpected count (reopen): %d", n)) - } - finread(nil) // no-op on reads that are in a group, so must qcx.Abort() to stop them. - qcx.Abort() - qcx.Reset() - } - } - N := 1000 - for i := 0; i < N; i++ { - go setter(i) - } - time.Sleep(time.Second * 1) - close(done) - - // allow all goro to finish before Closing the lmdb.env, otherwise - // we will crash as the goroutines making Tx will try to use the env - // after it is closed. It can take quite a while. - // one writer might be blocking the other... so ask for only N-2 at first - // to avoid deadlock. - barrier.BlockUntil(N - 2) - barrier.UnblockReaders() - time.Sleep(1 * time.Second) -} - -// test TxFactory.green2blue -// -// blue_green starting with an empty or full blue database -// should copy all of green (if blue is empty); or if blue is ull, -// verify that blue has all the same bits as green. -// -// Benefits: a) we start with known identical state so our testing/comparisons can be valid; -// and b) we have an easy migration mechanism, to go from one storage format to another. -// -func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) { - checked := []string{"roaring", "rbf"} - - expectError := false - for _, blue := range checked { - for _, green := range checked { - if blue == green { - continue - } - if blue == "roaring" { - // not supported - expectError = true - } else { - expectError = false - } - blue_green := blue + "_" + green - //vv("setting blue_green to '%v'", blue_green) - - // ============================= - // Begin setup. - // - // Setup happens with green only. - - h, path, err := makeHolder(t, green) - if err != nil { - t.Fatalf("creating holder: %v", err) - } - defer os.RemoveAll(path) - //vv("path = %v", path) - - // we will manually h.Close() below - - // Write bits to separate indexes. - testSetBit(t, h, "i0", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 12345678) - - testOp := testHolderOperator{} - ctx := context.Background() - err = h.Process(ctx, &testOp) - if err != nil { - t.Fatalf("processing holder: %v", err) - } - expected := testHolderOperator{ - indexSeen: 2, indexProcessed: 2, - fieldSeen: 2, fieldProcessed: 2, - viewSeen: 2, viewProcessed: 2, - fragmentSeen: 3, fragmentProcessed: 3, - } - if testOp != expected { - t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp) - } - - // verify data is there - rowID := uint64(100) - colID := uint64(200) - _, _ = rowID, colID - testMustHaveBit(t, h, "i0", "f", rowID, colID) - testMustHaveBit(t, h, "i1", "f", 100, 200) - testMustHaveBit(t, h, "i1", "f", 100, 12345678) - - //vv("about to reopen; blue_green = '%v' but PILOSA_STORAGE_BACKEND='%v'", blue_green, os.Getenv("PILOSA_STORAGE_BACKEND")) - //h.DumpAllShards() - - //vv("after dump, about to close") - h.Close() - - //vv("after close, about to re-open") - - // can we re.Open the same holder h? hopefully without a problem. - PanicOn(h.Open()) - - //vv("h.Open() re-open worked; blue_green = '%v'; dump; with PILOSA_STORAGE_BACKEND='%v'", blue_green, os.Getenv("PILOSA_STORAGE_BACKEND")) - //h.DumpAllShards() - - testMustHaveBit(t, h, "i0", "f", rowID, colID) // panic here, colID 200 bit was cold. - testMustHaveBit(t, h, "i1", "f", 100, 200) - testMustHaveBit(t, h, "i1", "f", 100, 12345678) - h.Close() - - //vv("successful re-open and then Close again of h.") - - // check that we can open a NewHolder on green, on same path, and still see our bits. - // Because the NewHolder is the code that creates and configures TxFactory as blue_green. - cfg := mustHolderConfig() - cfg.StorageConfig.Backend = green - h2 := NewHolder(path, cfg) - PanicOn(h2.Open()) - - testMustHaveBit(t, h2, "i0", "f", rowID, colID) - testMustHaveBit(t, h2, "i1", "f", 100, 200) - testMustHaveBit(t, h2, "i1", "f", 100, 12345678) - h2.Close() - - // verify that blue does not have it. - // open a new holder on path, just looking at blue. - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue - h3 := NewHolder(path, cfg) - PanicOn(h3.Open()) - - testMustNotHaveBit(t, h3, "i0", "f", rowID, colID) - testMustNotHaveBit(t, h3, "i1", "f", 100, 200) - testMustNotHaveBit(t, h3, "i1", "f", 100, 12345678) - - h3.Close() - - // ============================= - // Setup done. On to actual test. - - // Opening in blue_green mode means that once Holder.Open() - // returns without error, the blue and green databases are - // identical. - // Since blue is empty, the blue database will get synched up - // with the green during Holder.Open(). - - // open a holder with path again, now looking at both blue and green. - // The Holder.Open should do the migration from green, populating blue. - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue_green - h4 := NewHolder(path, cfg) - - //vv("about to h4.Open we should populate blue from green") - err = h4.Open() - if expectError { - if err == nil { - panic("expected error since migration to roaring not supported") - } - } else { - PanicOn(err) - } - - testMustHaveBit(t, h4, "i0", "f", rowID, colID) - testMustHaveBit(t, h4, "i1", "f", 100, 200) - testMustHaveBit(t, h4, "i1", "f", 100, 12345678) - - //vv("successfully verified populatingBlueFromGreen with blue_green = '%v'", blue_green) - h4.Close() - os.RemoveAll(path) - } - } -} - -// test the situation where we startup blue_green with existing data and -// go to verify it but blue has more data than green. -// That will also cause query divergence. -func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) { - checked := []string{"roaring", "bolt", "rbf"} - - for _, blue := range checked { - for _, green := range checked { - if blue == green { - continue - } - if blue == "roaring" { - // not supported - continue - } - blue_green := blue + "_" + green - - // ============================= - // Begin setup. - // - // Setup happens with green only. - - h, path, err := makeHolder(t, green) - if err != nil { - t.Fatalf("creating holder: %v", err) - } - defer os.RemoveAll(path) - - //vv("on green, which is '%v'", green) - // we will manually h.Close() below - - // Write bits to separate indexes. - testSetBit(t, h, "i0", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 12345678) - - testOp := testHolderOperator{} - ctx := context.Background() - err = h.Process(ctx, &testOp) - if err != nil { - t.Fatalf("processing holder: %v", err) - } - expected := testHolderOperator{ - indexSeen: 2, indexProcessed: 2, - fieldSeen: 2, fieldProcessed: 2, - viewSeen: 2, viewProcessed: 2, - fragmentSeen: 3, fragmentProcessed: 3, - } - if testOp != expected { - t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp) - } - - // verify data is there - rowID := uint64(100) - colID := uint64(200) - _, _ = rowID, colID - testMustHaveBit(t, h, "i0", "f", rowID, colID) - testMustHaveBit(t, h, "i1", "f", 100, 200) - testMustHaveBit(t, h, "i1", "f", 100, 12345678) - - h.Close() - - // verify that blue does not have it. - // open a new holder on path, just looking at blue. - - //vv("on blue, which is '%v'", blue) - - cfg := mustHolderConfig() - cfg.StorageConfig.Backend = blue - h3 := NewHolder(path, cfg) - PanicOn(h3.Open()) - - testMustNotHaveBit(t, h3, "i0", "f", rowID, colID) - testMustNotHaveBit(t, h3, "i1", "f", 100, 200) - testMustNotHaveBit(t, h3, "i1", "f", 100, 12345678) - - h3.Close() - - // ============================= - // Setup done. On to actual test. - - // Opening in blue_green mode means that once Holder.Open() - // returns without error, the blue and green databases are - // identical. - // Since blue is empty, the blue database will get synched up - // with the green during Holder.Open(). - - //vv("on blue_green, which is '%v'", blue_green) - - // open a holder with path again, now looking at both blue and green. - // The Holder.Open should do the migration from green, populating blue. - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue_green - h4 := NewHolder(path, cfg) - PanicOn(h4.Open()) - - testMustHaveBit(t, h4, "i0", "f", rowID, colID) - testMustHaveBit(t, h4, "i1", "f", 100, 200) - testMustHaveBit(t, h4, "i1", "f", 100, 12345678) - h4.Close() - - // now open just blue, and add a bit to a new index, i2. - //vv("on blue, which is '%v'", blue) - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue - h5 := NewHolder(path, cfg) - PanicOn(h5.Open()) - testSetBit(t, h5, "i2", "f", 500, 777) - - //vv("after adding a bit to blue, we have:") - //h5.DumpAllShards() - - h5.Close() - - // now open blue_green. should get a verification failure - // due to the extra bit in blue. - - // BEGIN verficiation that should ERROR out b/c blue has more data. - - // open a holder with path again, now looking at both blue and green. - // The Holder.Open should verify blue against green and notice the extra bit. - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue_green - h6 := NewHolder(path, cfg) - err = h6.Open() - //h6.DumpAllShards() - - if err == nil { - h6.Close() - t.Fatalf("should have had blue-green verification fail on Holder.Open") - } - - h6.Close() - } - } -} - func Test_TxFactory_verifyStringConstantsMatch(t *testing.T) { // txtype.String() method MUST return strings that match - // our const definitions at the top of txfactory.go, or - // else blue-green transactions cannot determine when - // the second transaction is being released in dbshard.go. - check := []txtype{roaringTxn, rbfTxn, boltTxn} - expect := []string{RoaringTxn, RBFTxn, BoltTxn} + // our const definitions at the top of txfactory.go. + check := []txtype{roaringTxn, rbfTxn} + expect := []string{RoaringTxn, RBFTxn} for i, chk := range check { obs := chk.String() if obs != expect[i] { diff --git a/util.go b/util.go index ebbff6e75..6d4b8e09e 100644 --- a/util.go +++ b/util.go @@ -17,19 +17,12 @@ package pilosa // util.go: a place for generic, reusable utilities. import ( - "fmt" - "io/ioutil" "os" - "path/filepath" "reflect" - "sort" - "strings" "syscall" "time" - "unsafe" "github.com/molecula/featurebase/v2/roaring" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck "github.com/pkg/errors" ) @@ -63,229 +56,6 @@ func NilInside(iface interface{}) bool { func highbits(v uint64) uint64 { return v >> 16 } func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) } -func toArray16(a []byte) []uint16 { - return (*[4096]uint16)(unsafe.Pointer(&a[0]))[: len(a)/2 : len(a)/2] -} -func toArray64(a []byte) []uint64 { - return (*[1024]uint64)(unsafe.Pointer(&a[0]))[:1024:1024] -} -func toInterval16(a []byte) []roaring.Interval16 { - return (*[2048]roaring.Interval16)(unsafe.Pointer(&a[0]))[: len(a)/4 : len(a)/4] -} - -func sliceToMap(slc []uint64) (m map[uint64]bool) { - m = make(map[uint64]bool) - for _, v := range slc { - m[v] = true - } - return -} - -// return A - B -func mapDiff(mapA, mapB map[uint64]bool) (r []int) { - for a := range mapA { - _, ok := mapB[a] - if !ok { - r = append(r, int(a)) - } - } - sort.Ints(r) - return -} - -func asInts(a []uint64) (r []int) { - r = make([]int, len(a)) - for i, v := range a { - r[i] = int(v) - } - return -} - -func containerAsString(ckey uint64, rc *roaring.Container) (r string) { - rbm := roaring.NewBitmap() - rbm.Containers.Put(ckey, rc) - return BitmapAsString(rbm) -} - -var _ = containerAsString // happy linter - -func roaringBitmapDiff(a, b *roaring.Bitmap) error { - nA := a.Count() - nB := b.Count() - - slcA := a.Slice() - slcB := b.Slice() - - mapA := sliceToMap(slcA) - mapB := sliceToMap(slcB) - - AminusB := mapDiff(mapA, mapB) - BminusA := mapDiff(mapB, mapA) - - sort.Ints(AminusB) - sort.Ints(BminusA) - - res := fmt.Sprintf("nA = %v; nB = %v;\n", nA, nB) - ndiff := 0 - if nA != nB { - ndiff++ - } - - if len(AminusB) > 0 { - res += fmt.Sprintf("==> AminusB = (len %v) '%#v'; ", len(AminusB), AminusB) - ndiff++ - } - if len(BminusA) > 0 { - res += fmt.Sprintf("\n==> BminusA = (len %v) '%#v'; ", len(BminusA), BminusA) - ndiff++ - } - if ndiff == 0 { - return nil - } - res += fmt.Sprintf("\n ==> A = '%#v'\n ==> B = '%#v'", asInts(slcA), asInts(slcB)) - return errors.New(res) -} - -func dirAsString(path string) (r string) { - r = fmt.Sprintf("dump of directory '%v':\n", path) - files, err := ioutil.ReadDir(path) - PanicOn(err) - for _, f := range files { - r += f.Name() + "\n" - } - return r -} - -var _ = dirAsString // happy linter - -var _ = zeroKeyContainerAsString // happy linter - -// for debugging -func zeroKeyContainerAsString(ct *roaring.Container) (r string) { - cts := roaring.NewSliceContainers() - cts.Put(0, ct) - rbm := &roaring.Bitmap{Containers: cts} - r = fmt.Sprintf("[%v]:", containerTypeNames[roaring.ContainerType(ct)]) + BitmapAsString(rbm) - return -} - -var containerTypeNames = map[byte]string{ - roaring.ContainerArray: "array", - roaring.ContainerBitmap: "bitmap", - roaring.ContainerRun: "run", -} - -func BitmapAsString(rbm *roaring.Bitmap) (r string) { - r = "c(" - slc := rbm.Slice() - width := 0 - s := "" - for _, v := range slc { - if width == 0 { - s = fmt.Sprintf("%v", v) - } else { - s = fmt.Sprintf(", %v", v) - } - width += len(s) - r += s - if width > 70 { - r += ",\n" - width = 0 - } - } - if width == 0 && len(r) > 2 { - r = r[:len(r)-2] - } - return r + ")" -} - -// fromArray16 converts to an 8KB page -func fromArray16(a []uint16) []byte { - if len(a) == 0 { - return []byte{} - } - if len(a) > 4096 { - PanicOn(fmt.Sprintf("cannot put more than 4096 integers into an array container: %v too big", len(a))) - } - return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*2 : len(a)*2] -} - -// fromArray64 converts to an 8KB page -func fromArray64(a []uint64) []byte { - if len(a) == 0 { - return []byte{} - } - return (*[8192]byte)(unsafe.Pointer(&a[0]))[:8192:8192] -} - -// fromInterval16 converts to 8KB page -func fromInterval16(a []roaring.Interval16) []byte { - if len(a) == 0 { - return []byte{} - } - if len(a) > 2048 { - PanicOn(fmt.Sprintf("cannot put more than 2048 roaring.Interval16 into a container: %v too big", len(a))) - } - return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4] -} - -// DiskUse reports the total bytes uses by all files under root -// that match requiredSuffix. requiredSuffix can be empty string. -// Space used by directories is not counted. -func DiskUse(root string, requiredSuffix string) (tot int, err error) { - if !DirExists(root) { - return -1, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root) - } - - err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if info == nil { - PanicOn(fmt.Sprintf("info was nil for path = '%v'", path)) - } - if info.IsDir() { - // skip the size of directories themselves, only summing files. - } else { - sz := info.Size() - if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) { - tot += int(sz) - } - } - return nil - }) - return -} - -// rootDir must exist. Return the size in bytes of the largest sub-directory -// that has the required suffix. The largestSize is from DiskUse() called -// on the sub-dir. DiskUse only counts file size, nothing for directory inodes. -func SubdirLargestDirWithSuffix(rootDir, requiredDirSuffix string) (exists bool, largestSize int, err error) { - if !DirExists(rootDir) { - return false, -1, fmt.Errorf("SubdirExistsWithSuffix error: root directory '%v' not found", rootDir) - } - - err = filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error { - if info == nil { - PanicOn(fmt.Sprintf("info was nil for path = '%v'", path)) - } - - if info.IsDir() && strings.HasSuffix(path, requiredDirSuffix) { - exists = true - size, err := DiskUse(path, "") - if err != nil { - // disk error? report it - return err - } - if size > largestSize { - largestSize = size - } - } - return nil - }) - if err != nil { - return exists, -1, err - } - return -} - // called by Holder.hasRoaringData() func roaringFragmentHasData(path string, index, field, view string, shard uint64) (hasData bool, err error) { diff --git a/utils_internal_test.go b/utils_internal_test.go index 76c3aa66f..fa9b666a8 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -15,47 +15,17 @@ package pilosa import ( - "bytes" "fmt" "testing" "time" pnet "github.com/molecula/featurebase/v2/net" - "github.com/molecula/featurebase/v2/roaring" "github.com/molecula/featurebase/v2/testhook" "github.com/molecula/featurebase/v2/topology" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck ) // utilities used by tests -// mustAddR is a helper for calling roaring.Container.Add() in tests to -// keep the linter happy that we are checking the error. -func mustAddR(changed bool, err error) { - PanicOn(err) -} - -// mustRemove is a helper for calling Tx.Remove() in tests to -// keep the linter happy that we are checking the error. -func mustRemove(changeCount int, err error) { - PanicOn(err) -} - -func getTestBitmapAsRawRoaring(bitsToSet ...uint64) []byte { - b := roaring.NewBitmap() - changed := b.DirectAddN(bitsToSet...) - n := len(bitsToSet) - if changed != n { - panic(fmt.Sprintf("changed=%v but bitsToSet len = %v", changed, n)) - } - buf := bytes.NewBuffer(make([]byte, 0, 100000)) - _, err := b.WriteTo(buf) - if err != nil { - panic(err) - } - return buf.Bytes() -} - // NewTestCluster returns a cluster with n nodes and uses a mod-based hasher. func NewTestCluster(tb testing.TB, n int) *cluster { path, err := testhook.TempDir(tb, "pilosa-cluster-") diff --git a/view.go b/view.go index ac94f64d8..911a2d19c 100644 --- a/view.go +++ b/view.go @@ -158,26 +158,28 @@ func (v *view) openWithShardSet(ss *shardSet) error { if nGoro < 4 { nGoro = 4 } - pj := newParallelJobs(nGoro) + var eg errgroup.Group + throttle := make(chan struct{}, nGoro) + for i := range frags { // create a new variable frag on each time through // the loop (instead of i, frag := range frags) // so that the closure run on the // goroutine has its own variable. frag := frags[i] - accepted := pj.run(func(worker int) error { + throttle <- struct{}{} + eg.Go(func() error { + defer func() { + <-throttle + }() if err := frag.Open(); err != nil { return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err) } return nil }) - if !accepted { - // have error/shutting down the pj, so stop - break - } } - err := pj.waitForFinish() + err := eg.Wait() if err != nil { return err } From 7c885c813052138807fe430eca89c62d649797d9 Mon Sep 17 00:00:00 2001 From: reesporte Date: Tue, 26 Oct 2021 15:39:06 -0500 Subject: [PATCH 14/40] export ShardSlice --- executor.go | 60 ++++++++++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/executor.go b/executor.go index ac9f4455e..0d8954085 100644 --- a/executor.go +++ b/executor.go @@ -605,12 +605,12 @@ func (e *executor) preprocessQuery(ctx context.Context, qcx *Qcx, index string, } } -type shardSlice []uint64 +type ShardSlice []uint64 // String creates a run-length encoded representation of a slice of shard IDs (integers). // For example, []uint64{0, 1, 3, 4, 5, 7, 8, 9, 11, 13} is represented as // [0-1,3-5,7-9,11,13]. -func (s shardSlice) String() string { +func (s ShardSlice) String() string { if len(s) == 0 { // surely this is impossible return "[]" @@ -689,105 +689,105 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p case "Sum": statFn() res, err := e.executeSum(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeSum %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeSum %v", ShardSlice(shards)) case "Min": statFn() res, err := e.executeMin(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMin %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeMin %v", ShardSlice(shards)) case "Max": statFn() res, err := e.executeMax(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMax %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeMax %v", ShardSlice(shards)) case "MinRow": statFn() res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMinRow %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeMinRow %v", ShardSlice(shards)) case "MaxRow": statFn() res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMaxRow %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeMaxRow %v", ShardSlice(shards)) case "Clear": statFn() res, err := e.executeClearBit(ctx, qcx, index, c, opt) - return res, errors.Wrapf(err, "executeClearBit %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeClearBit %v", ShardSlice(shards)) case "ClearRow": statFn() res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeClearRow %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeClearRow %v", ShardSlice(shards)) case "Distinct": statFn() res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeDistinct %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeDistinct %v", ShardSlice(shards)) case "Store": statFn() res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeSetRow %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeSetRow %v", ShardSlice(shards)) case "Count": statFn() res, err := e.executeCount(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeCount %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeCount %v", ShardSlice(shards)) case "Set": statFn() res, err := e.executeSet(ctx, qcx, index, c, opt) - return res, errors.Wrapf(err, "executeSet %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeSet %v", ShardSlice(shards)) case "TopK": statFn() res, err := e.executeTopK(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeTopK %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeTopK %v", ShardSlice(shards)) case "TopN": statFn() res, err := e.executeTopN(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeTopN %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeTopN %v", ShardSlice(shards)) case "Rows": statFn() res, err := e.executeRows(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeRows %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeRows %v", ShardSlice(shards)) case "ExternalLookup": statFn() res, err := e.executeExternalLookup(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeExternalLookup %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeExternalLookup %v", ShardSlice(shards)) case "Extract": statFn() res, err := e.executeExtract(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeExtract %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeExtract %v", ShardSlice(shards)) case "GroupBy": statFn() res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeGroupBy %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeGroupBy %v", ShardSlice(shards)) case "Options": statFn() res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeOptionsCall %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeOptionsCall %v", ShardSlice(shards)) case "IncludesColumn": res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeIncludesColumnCall %v", ShardSlice(shards)) case "FieldValue": statFn() res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeFieldValueCall %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeFieldValueCall %v", ShardSlice(shards)) case "Precomputed": res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executePrecomputedCall %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executePrecomputedCall %v", ShardSlice(shards)) case "UnionRows": res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeUnionRows %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeUnionRows %v", ShardSlice(shards)) case "ConstRow": res, err := e.executeConstRow(ctx, index, c) - return res, errors.Wrapf(err, "executeConstRow %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeConstRow %v", ShardSlice(shards)) case "Limit": res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeLimitCall %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeLimitCall %v", ShardSlice(shards)) case "Percentile": res, err := e.executePercentile(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executePercentile %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executePercentile %v", ShardSlice(shards)) case "Delete": statFn() //TODO(twg) need this? res, err := e.executeDeleteRecords(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeDelete %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeDelete %v", ShardSlice(shards)) default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap. statFn() res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeBitmapCall %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeBitmapCall %v", ShardSlice(shards)) } } @@ -2939,7 +2939,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, errors.Wrapf(err, "mapReduce shards: %v", shardSlice(shards)) + return nil, errors.Wrapf(err, "mapReduce shards: %v", ShardSlice(shards)) } results, _ := other.([]GroupCount) From eb8460c291dc8148cbabc40009a495752aa0e5eb Mon Sep 17 00:00:00 2001 From: reesporte Date: Tue, 26 Oct 2021 15:39:52 -0500 Subject: [PATCH 15/40] wrap shards in error message as ShardSlice for pretty output --- http/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/client.go b/http/client.go index 1d41a114f..587cca17d 100644 --- a/http/client.go +++ b/http/client.go @@ -522,7 +522,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index str // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { - return nil, errors.Wrapf(err, "'%s', shards %v", queryRequest.Query, queryRequest.Shards) + return nil, errors.Wrapf(err, "'%s', shards %v", queryRequest.Query, pilosa.ShardSlice(queryRequest.Shards)) } defer resp.Body.Close() From 10f235dac13a714cc0eb6a50e246614f53b8d223 Mon Sep 17 00:00:00 2001 From: Stephanie Yang Date: Wed, 27 Oct 2021 13:00:40 -0500 Subject: [PATCH 16/40] attempt to upgrade node (to v14) and node-sass (to v4.14) --- lattice/package.json | 6 +- lattice/src/App/QueryBuilder/utils.ts | 2 +- lattice/yarn.lock | 461 ++++++++++---------------- 3 files changed, 182 insertions(+), 287 deletions(-) diff --git a/lattice/package.json b/lattice/package.json index 15211e27f..e50cc0bdb 100644 --- a/lattice/package.json +++ b/lattice/package.json @@ -39,15 +39,13 @@ "devDependencies": { "@types/d3-array": "^2.0.0", "@types/jest": "^26.0.4", - "@types/node": "^14.0.20", - "@types/node-sass": "^4.11.0", "@types/react": "^16.9.41", "@types/react-beautiful-dnd": "^13.0.0", "@types/react-dom": "^16.9.8", "@types/react-router": "^5.1.8", "@types/react-router-dom": "^5.1.5", - "node-sass": "^4.12.0", - "react-scripts": "^4.0.0", + "node-sass": "^4.14.1", + "react-scripts": "^4.0.3", "tslint": "^6.1.2", "typescript": "^4.2.2" }, diff --git a/lattice/src/App/QueryBuilder/utils.ts b/lattice/src/App/QueryBuilder/utils.ts index c90cafaed..8706e8140 100644 --- a/lattice/src/App/QueryBuilder/utils.ts +++ b/lattice/src/App/QueryBuilder/utils.ts @@ -58,7 +58,7 @@ export const stringifyRowData = (rowCalls: RowGrouping[], operator?: Operator) = .map((ip) => `Row(${field}="${ip}")`) .join(', '); rowString = `Union(${ipRows})`; - } catch (error) { + } catch (error: any) { return { error: true, queryString: error.message }; } } else if (rowOperator === 'like') { diff --git a/lattice/yarn.lock b/lattice/yarn.lock index 335aa9264..31499a6ea 100644 --- a/lattice/yarn.lock +++ b/lattice/yarn.lock @@ -2443,23 +2443,16 @@ resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA== -"@types/node-sass@^4.11.0": - version "4.11.1" - resolved "https://registry.yarnpkg.com/@types/node-sass/-/node-sass-4.11.1.tgz#bda27c5181cbf7c090c3058e119633dfb2b6504c" - integrity sha512-wPOmOEEtbwQiPTIgzUuRSQZ3H5YHinsxRGeZzPSDefAm4ylXWnZG9C0adses8ymyplKK0gwv3JkDNO8GGxnWfg== - dependencies: - "@types/node" "*" +"@types/minimist@^1.2.0": + version "1.2.2" + resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c" + integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ== "@types/node@*": version "13.9.2" resolved "https://registry.yarnpkg.com/@types/node/-/node-13.9.2.tgz#ace1880c03594cc3e80206d96847157d8e7fa349" integrity sha512-bnoqK579sAYrQbp73wwglccjJ4sfRdKU7WNEZ5FW4K2U6Kc0/eZ5kvXG0JKsEKFB50zrFmfFt52/cvBbZa7eXg== -"@types/node@^14.0.20": - version "14.0.20" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.0.20.tgz#0da05cddbc761e1fa98af88a17244c8c1ff37231" - integrity sha512-MRn/NP3dee8yL5QhbSA6riuwkS+UOcsPUMOIOG3KMUQpuor/2TopdRBu8QaaB4fGU+gz/bzyDWt0FtUbeJ8H1A== - "@types/normalize-package-data@^2.4.0": version "2.4.0" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e" @@ -3120,11 +3113,6 @@ arr-union@^3.1.0: resolved "https://registry.yarnpkg.com/arr-union/-/arr-union-3.1.0.tgz#e39b09aea9def866a8f206e288af63919bae39c4" integrity sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ= -array-find-index@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/array-find-index/-/array-find-index-1.0.2.tgz#df010aa1287e164bbda6f9723b0a96a1ec4187a1" - integrity sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E= - array-flatten@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" @@ -3196,6 +3184,11 @@ array.prototype.flatmap@^1.2.3: es-abstract "^1.18.0-next.1" function-bind "^1.1.1" +arrify@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d" + integrity sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0= + arrify@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" @@ -3582,13 +3575,6 @@ bindings@^1.5.0: dependencies: file-uri-to-path "1.0.0" -block-stream@*: - version "0.0.9" - resolved "https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a" - integrity sha1-E+v+d4oDIFz+A3UUgeu0szAMEmo= - dependencies: - inherits "~2.0.0" - bluebird@^3.5.5: version "3.7.2" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" @@ -3928,24 +3914,20 @@ camel-case@^4.1.1: pascal-case "^3.1.1" tslib "^1.10.0" -camelcase-keys@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-2.1.0.tgz#308beeaffdf28119051efa1d932213c91b8f92e7" - integrity sha1-MIvur/3ygRkFHvodkyITyRuPkuc= +camelcase-keys@^6.2.2: + version "6.2.2" + resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-6.2.2.tgz#5e755d6ba51aa223ec7d3d52f25778210f9dc3c0" + integrity sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg== dependencies: - camelcase "^2.0.0" - map-obj "^1.0.0" + camelcase "^5.3.1" + map-obj "^4.0.0" + quick-lru "^4.0.1" camelcase@5.3.1, camelcase@^5.0.0, camelcase@^5.3.1: version "5.3.1" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== -camelcase@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-2.1.1.tgz#7c1d16d679a1bbe59ca02cacecfb011e201f5a1f" - integrity sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8= - camelcase@^6.0.0, camelcase@^6.1.0, camelcase@^6.2.0: version "6.2.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.2.0.tgz#924af881c9d525ac9d87f40d964e5cea982a1809" @@ -4491,7 +4473,7 @@ create-hmac@^1.1.0, create-hmac@^1.1.2, create-hmac@^1.1.4: safe-buffer "^5.0.1" sha.js "^2.4.8" -cross-spawn@7.0.3, cross-spawn@^7.0.0, cross-spawn@^7.0.2: +cross-spawn@7.0.3, cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== @@ -4500,14 +4482,6 @@ cross-spawn@7.0.3, cross-spawn@^7.0.0, cross-spawn@^7.0.2: shebang-command "^2.0.0" which "^2.0.1" -cross-spawn@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-3.0.1.tgz#1256037ecb9f0c5f79e3d6ef135e30770184b982" - integrity sha1-ElYDfsufDF9549bvE14wdwGEuYI= - dependencies: - lru-cache "^4.0.1" - which "^1.2.9" - cross-spawn@^6.0.0: version "6.0.5" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" @@ -4795,13 +4769,6 @@ csstype@^2.5.5: resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.6.13.tgz#a6893015b90e84dd6e85d0e3b442a1e84f2dbe0f" integrity sha512-ul26pfSQTZW8dcOnD2iiJssfXw0gdNVX9IJDH/X3K5DGPfj+fUYe3kB+swUY6BF3oZDxaID3AJt+9/ojSAE05A== -currently-unhandled@^0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea" - integrity sha1-mI3zP+qxke95mmE2nddsF635V+o= - dependencies: - array-find-index "^1.0.1" - cyclist@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9" @@ -4989,7 +4956,15 @@ debug@^4.0.1, debug@^4.1.0, debug@^4.1.1: dependencies: ms "^2.1.1" -decamelize@^1.1.2, decamelize@^1.2.0: +decamelize-keys@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/decamelize-keys/-/decamelize-keys-1.1.0.tgz#d171a87933252807eb3cb61dc1c1445d078df2d9" + integrity sha1-0XGoeTMlKAfrPLYdwcFEXQeN8tk= + dependencies: + decamelize "^1.1.0" + map-obj "^1.0.0" + +decamelize@^1.1.0, decamelize@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA= @@ -5422,6 +5397,11 @@ entities@^2.0.0: resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.0.tgz#68d6084cab1b079767540d80e56a39b423e4abf4" integrity sha512-D9f7V0JSRwIxlRI2mjMqufDrRDnx8p+eEOz7aUM9SuvF8gsBzra0/6tbjl1m8eQHrZlYj6PxqE00hZ1SAIKPLw== +env-paths@^2.2.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.1.tgz#420399d416ce1fbe9bc0a07c62fa68d67fd0f8f2" + integrity sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A== + errno@^0.1.3, errno@~0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.7.tgz#4684d71779ad39af177e3f007996f7c67c852618" @@ -6139,14 +6119,6 @@ find-up@4.1.0, find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" -find-up@^1.0.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f" - integrity sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8= - dependencies: - path-exists "^2.0.0" - pinkie-promise "^2.0.0" - find-up@^2.0.0, find-up@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-2.1.0.tgz#45d1b7e506c717ddd482775a2b77920a3c0c57a7" @@ -6344,16 +6316,6 @@ fsevents@^2.1.2, fsevents@^2.1.3, fsevents@~2.3.1: resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== -fstream@^1.0.0, fstream@^1.0.12: - version "1.0.12" - resolved "https://registry.yarnpkg.com/fstream/-/fstream-1.0.12.tgz#4e8ba8ee2d48be4f7d0de505455548eae5932045" - integrity sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg== - dependencies: - graceful-fs "^4.1.2" - inherits "~2.0.0" - mkdirp ">=0.5 0" - rimraf "2" - function-bind@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" @@ -6579,6 +6541,11 @@ graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6 resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.3.tgz#4a12ff1b60376ef09862c2093edd908328be8423" integrity sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ== +graceful-fs@^4.2.3: + version "4.2.8" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.8.tgz#e412b8d33f5e006593cbd3cee6df9f2cebbe802a" + integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg== + graceful-fs@^4.2.4: version "4.2.6" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee" @@ -6615,6 +6582,11 @@ har-validator@~5.1.3: ajv "^6.5.5" har-schema "^2.0.0" +hard-rejection@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/hard-rejection/-/hard-rejection-2.1.0.tgz#1c6eda5c1685c63942766d79bb40ae773cecd883" + integrity sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA== + harmony-reflect@^1.4.6: version "1.6.1" resolved "https://registry.yarnpkg.com/harmony-reflect/-/harmony-reflect-1.6.1.tgz#c108d4f2bb451efef7a37861fdbdae72c9bdefa9" @@ -6759,6 +6731,13 @@ hosted-git-info@^2.1.4: resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488" integrity sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg== +hosted-git-info@^4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-4.0.2.tgz#5e425507eede4fea846b7262f0838456c4209961" + integrity sha512-c9OGXbZ3guC/xOlCg1Ci/VgWlwsqDv1yMQL1CWqXDL0hDjXuNcq0zuR4xqPSuasI3kqFDhqSyTjREz5gzq0fXg== + dependencies: + lru-cache "^6.0.0" + hpack.js@^2.1.6: version "2.1.6" resolved "https://registry.yarnpkg.com/hpack.js/-/hpack.js-2.1.6.tgz#87774c0949e513f42e84575b3c45681fade2a0b2" @@ -7046,18 +7025,6 @@ imurmurhash@^0.1.4: resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= -in-publish@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/in-publish/-/in-publish-2.0.1.tgz#948b1a535c8030561cea522f73f78f4be357e00c" - integrity sha512-oDM0kUSNFC31ShNxHKUyfZKy8ZeXZBWMjMdZHKLOk13uvT27VTL/QzRGfRUcevJhpkZAvlhPYuXkF7eNWrtyxQ== - -indent-string@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-2.1.0.tgz#8e2d48348742121b4a8218b7a137e9a52049dc80" - integrity sha1-ji1INIdCEhtKghi3oTfppSBJ3IA= - dependencies: - repeating "^2.0.0" - indent-string@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251" @@ -7081,7 +7048,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: +inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -7253,6 +7220,13 @@ is-core-module@^2.0.0, is-core-module@^2.2.0: dependencies: has "^1.0.3" +is-core-module@^2.5.0: + version "2.8.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.8.0.tgz#0321336c3d0925e497fd97f5d95cb114a5ccd548" + integrity sha512-vd15qHsaqrRL7dtH6QNuy0ndJmRDrS9HAM1CAiSifNUFv4x1a0CCVsj18hJ1mShxIG6T2i1sO78MkP56r0nYRw== + dependencies: + has "^1.0.3" + is-data-descriptor@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56" @@ -7317,11 +7291,6 @@ is-extglob@^2.1.0, is-extglob@^2.1.1: resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI= -is-finite@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/is-finite/-/is-finite-1.1.0.tgz#904135c77fb42c0641d6aa1bcdbc4daa8da082f3" - integrity sha512-cdyMtqX/BOqqNBBiKlIVkytNHm49MtMlYyn1zxzvJKWmFMlGzm+ry5BBfYyeY9YmNKbRSo/o7OX9w9ale0wg3w== - is-fullwidth-code-point@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" @@ -7414,7 +7383,7 @@ is-path-inside@^2.1.0: dependencies: path-is-inside "^1.0.2" -is-plain-obj@^1.0.0: +is-plain-obj@^1.0.0, is-plain-obj@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz#71a50c8429dfca773c92a390a4a03b39fcd51d3e" integrity sha1-caUMhCnfync8kqOQpKA7OfzVHT4= @@ -7495,11 +7464,6 @@ is-typedarray@^1.0.0, is-typedarray@~1.0.0: resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= -is-utf8@^0.2.0: - version "0.2.1" - resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" - integrity sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI= - is-windows@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" @@ -8297,7 +8261,7 @@ kind-of@^5.0.0: resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-5.1.0.tgz#729c91e2d857b7a419a1f9aa65685c4c33f5845d" integrity sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw== -kind-of@^6.0.0, kind-of@^6.0.2: +kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== @@ -8358,17 +8322,6 @@ lines-and-columns@^1.1.6: resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00" integrity sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA= -load-json-file@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-1.1.0.tgz#956905708d58b4bab4c2261b04f59f31c99374c0" - integrity sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA= - dependencies: - graceful-fs "^4.1.2" - parse-json "^2.2.0" - pify "^2.0.0" - pinkie-promise "^2.0.0" - strip-bom "^2.0.0" - load-json-file@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-2.0.0.tgz#7947e42149af80d696cbf797bcaabcfe1fe29ca8" @@ -8531,14 +8484,6 @@ loose-envify@^1.1.0, loose-envify@^1.2.0, loose-envify@^1.3.1, loose-envify@^1.4 dependencies: js-tokens "^3.0.0 || ^4.0.0" -loud-rejection@^1.0.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/loud-rejection/-/loud-rejection-1.6.0.tgz#5b46f80147edee578870f086d04821cf998e551f" - integrity sha1-W0b4AUft7leIcPCG0Eghz5mOVR8= - dependencies: - currently-unhandled "^0.4.1" - signal-exit "^3.0.0" - lower-case@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-2.0.1.tgz#39eeb36e396115cc05e29422eaea9e692c9408c7" @@ -8546,14 +8491,6 @@ lower-case@^2.0.1: dependencies: tslib "^1.10.0" -lru-cache@^4.0.1: - version "4.1.5" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd" - integrity sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g== - dependencies: - pseudomap "^1.0.2" - yallist "^2.1.2" - lru-cache@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" @@ -8609,11 +8546,16 @@ map-cache@^0.2.2: resolved "https://registry.yarnpkg.com/map-cache/-/map-cache-0.2.2.tgz#c32abd0bd6525d9b051645bb4f26ac5dc98a0dbf" integrity sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8= -map-obj@^1.0.0, map-obj@^1.0.1: +map-obj@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d" integrity sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0= +map-obj@^4.0.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-4.3.0.tgz#9304f906e93faae70880da102a9f1df0ea8bb05a" + integrity sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ== + map-visit@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/map-visit/-/map-visit-1.0.0.tgz#ecdca8f13144e660f1b5bd41f12f3479d98dfb8f" @@ -8671,21 +8613,23 @@ memory-fs@^0.5.0: errno "^0.1.3" readable-stream "^2.0.1" -meow@^3.7.0: - version "3.7.0" - resolved "https://registry.yarnpkg.com/meow/-/meow-3.7.0.tgz#72cb668b425228290abbfa856892587308a801fb" - integrity sha1-cstmi0JSKCkKu/qFaJJYcwioAfs= +meow@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/meow/-/meow-9.0.0.tgz#cd9510bc5cac9dee7d03c73ee1f9ad959f4ea364" + integrity sha512-+obSblOQmRhcyBt62furQqRAQpNyWXo8BuQ5bN7dG8wmwQ+vwHKp/rCFD4CrTP8CsDQD1sjoZ94K417XEUk8IQ== dependencies: - camelcase-keys "^2.0.0" - decamelize "^1.1.2" - loud-rejection "^1.0.0" - map-obj "^1.0.1" - minimist "^1.1.3" - normalize-package-data "^2.3.4" - object-assign "^4.0.1" - read-pkg-up "^1.0.1" - redent "^1.0.0" - trim-newlines "^1.0.0" + "@types/minimist" "^1.2.0" + camelcase-keys "^6.2.2" + decamelize "^1.2.0" + decamelize-keys "^1.1.0" + hard-rejection "^2.1.0" + minimist-options "4.1.0" + normalize-package-data "^3.0.0" + read-pkg-up "^7.0.1" + redent "^3.0.0" + trim-newlines "^3.0.0" + type-fest "^0.18.0" + yargs-parser "^20.2.3" merge-descriptors@1.0.1: version "1.0.1" @@ -8786,6 +8730,11 @@ mimic-fn@^2.1.0: resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== +min-indent@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/min-indent/-/min-indent-1.0.1.tgz#a63f681673b30571fbe8bc25686ae746eefa9869" + integrity sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg== + mini-create-react-context@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/mini-create-react-context/-/mini-create-react-context-0.4.0.tgz#df60501c83151db69e28eac0ef08b4002efab040" @@ -8821,7 +8770,16 @@ minimatch@3.0.4, minimatch@^3.0.4, minimatch@~3.0.2: dependencies: brace-expansion "^1.1.7" -minimist@^1.1.1, minimist@^1.1.3, minimist@^1.2.0, minimist@^1.2.5: +minimist-options@4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619" + integrity sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A== + dependencies: + arrify "^1.0.1" + is-plain-obj "^1.1.0" + kind-of "^6.0.3" + +minimist@^1.1.1, minimist@^1.2.0, minimist@^1.2.5: version "1.2.5" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== @@ -8886,13 +8844,6 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" -"mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.5: - version "0.5.5" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" - integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== - dependencies: - minimist "^1.2.5" - mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@~0.5.1: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.3.tgz#5a514b7179259287952881e94410ec5465659f8c" @@ -8900,6 +8851,13 @@ mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@~0.5.1: dependencies: minimist "^1.2.5" +mkdirp@^0.5.5: + version "0.5.5" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" + integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== + dependencies: + minimist "^1.2.5" + mkdirp@^1.0.3, mkdirp@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" @@ -9058,23 +9016,21 @@ node-forge@^0.10.0: resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA== -node-gyp@^3.8.0: - version "3.8.0" - resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-3.8.0.tgz#540304261c330e80d0d5edce253a68cb3964218c" - integrity sha512-3g8lYefrRRzvGeSowdJKAKyks8oUpLEd/DyPV4eMhVlhJ0aNaZqIrNUIPuEWWTAoPqyFkfGrM67MC69baqn6vA== +node-gyp@^7.1.0: + version "7.1.2" + resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-7.1.2.tgz#21a810aebb187120251c3bcec979af1587b188ae" + integrity sha512-CbpcIo7C3eMu3dL1c3d0xw449fHIGALIJsRP4DDPHpyiW8vcriNY7ubh9TE4zEKfSxscY7PjeFnshE7h75ynjQ== dependencies: - fstream "^1.0.0" - glob "^7.0.3" - graceful-fs "^4.1.2" - mkdirp "^0.5.0" - nopt "2 || 3" - npmlog "0 || 1 || 2 || 3 || 4" - osenv "0" - request "^2.87.0" - rimraf "2" - semver "~5.3.0" - tar "^2.0.0" - which "1" + env-paths "^2.2.0" + glob "^7.1.4" + graceful-fs "^4.2.3" + nopt "^5.0.0" + npmlog "^4.1.2" + request "^2.88.2" + rimraf "^3.0.2" + semver "^7.3.2" + tar "^6.0.2" + which "^2.0.2" node-int64@^0.4.0: version "0.4.0" @@ -9139,37 +9095,35 @@ node-releases@^1.1.61, node-releases@^1.1.70: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.71.tgz#cb1334b179896b1c89ecfdd4b725fb7bbdfc7dbb" integrity sha512-zR6HoT6LrLCRBwukmrVbHv0EpEQjksO6GmFcZQQuCAy139BEsoVKPYnf3jongYW83fAa1torLGYwxxky/p28sg== -node-sass@^4.12.0: - version "4.14.1" - resolved "https://registry.yarnpkg.com/node-sass/-/node-sass-4.14.1.tgz#99c87ec2efb7047ed638fb4c9db7f3a42e2217b5" - integrity sha512-sjCuOlvGyCJS40R8BscF5vhVlQjNN069NtQ1gSxyK1u9iqvn6tf7O1R4GNowVZfiZUCRt5MmMs1xd+4V/7Yr0g== +node-sass@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/node-sass/-/node-sass-6.0.1.tgz#cad1ccd0ce63e35c7181f545d8b986f3a9a887fe" + integrity sha512-f+Rbqt92Ful9gX0cGtdYwjTrWAaGURgaK5rZCWOgCNyGWusFYHhbqCCBoFBeat+HKETOU02AyTxNhJV0YZf2jQ== dependencies: async-foreach "^0.1.3" chalk "^1.1.1" - cross-spawn "^3.0.0" + cross-spawn "^7.0.3" gaze "^1.0.0" get-stdin "^4.0.1" glob "^7.0.3" - in-publish "^2.0.0" lodash "^4.17.15" - meow "^3.7.0" - mkdirp "^0.5.1" + meow "^9.0.0" nan "^2.13.2" - node-gyp "^3.8.0" + node-gyp "^7.1.0" npmlog "^4.0.0" request "^2.88.0" sass-graph "2.2.5" stdout-stream "^1.4.0" "true-case-path" "^1.0.2" -"nopt@2 || 3": - version "3.0.6" - resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9" - integrity sha1-xkZdvwirzU2zWTF/eaxopkayj/k= +nopt@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-5.0.0.tgz#530942bb58a512fccafe53fe210f13a25355dc88" + integrity sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ== dependencies: abbrev "1" -normalize-package-data@^2.3.2, normalize-package-data@^2.3.4, normalize-package-data@^2.5.0: +normalize-package-data@^2.3.2, normalize-package-data@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8" integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA== @@ -9179,6 +9133,16 @@ normalize-package-data@^2.3.2, normalize-package-data@^2.3.4, normalize-package- semver "2 || 3 || 4 || 5" validate-npm-package-license "^3.0.1" +normalize-package-data@^3.0.0: + version "3.0.3" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz#dbcc3e2da59509a0983422884cd172eefdfa525e" + integrity sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA== + dependencies: + hosted-git-info "^4.0.1" + is-core-module "^2.5.0" + semver "^7.3.4" + validate-npm-package-license "^3.0.1" + normalize-path@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-2.1.1.tgz#1ab28b556e198363a8c1a6f7e6fa20137fe6aed9" @@ -9225,7 +9189,7 @@ npm-run-path@^4.0.0: dependencies: path-key "^3.0.0" -"npmlog@0 || 1 || 2 || 3 || 4", npmlog@^4.0.0: +npmlog@^4.0.0, npmlog@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" integrity sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg== @@ -9468,24 +9432,6 @@ os-browserify@^0.3.0: resolved "https://registry.yarnpkg.com/os-browserify/-/os-browserify-0.3.0.tgz#854373c7f5c2315914fc9bfc6bd8238fdda1ec27" integrity sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc= -os-homedir@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" - integrity sha1-/7xJiDNuDoM94MFox+8VISGqf7M= - -os-tmpdir@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" - integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ= - -osenv@0: - version "0.1.5" - resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410" - integrity sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g== - dependencies: - os-homedir "^1.0.0" - os-tmpdir "^1.0.0" - p-each-series@^2.1.0: version "2.2.0" resolved "https://registry.yarnpkg.com/p-each-series/-/p-each-series-2.2.0.tgz#105ab0357ce72b202a8a8b94933672657b5e2a9a" @@ -9666,13 +9612,6 @@ path-dirname@^1.0.0: resolved "https://registry.yarnpkg.com/path-dirname/-/path-dirname-1.0.2.tgz#cc33d24d525e099a5388c0336c6e32b9160609e0" integrity sha1-zDPSTVJeCZpTiMAzbG4yuRYGCeA= -path-exists@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-2.1.0.tgz#0feb6c64f0fc518d9a754dd5efb62c7022761f4b" - integrity sha1-D+tsZPD8UY2adU3V77YscCJ2H0s= - dependencies: - pinkie-promise "^2.0.0" - path-exists@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" @@ -9720,15 +9659,6 @@ path-to-regexp@^1.7.0: dependencies: isarray "0.0.1" -path-type@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" - integrity sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE= - dependencies: - graceful-fs "^4.1.2" - pify "^2.0.0" - pinkie-promise "^2.0.0" - path-type@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/path-type/-/path-type-2.0.0.tgz#f012ccb8415b7096fc2daa1054c3d72389594c73" @@ -10669,11 +10599,6 @@ prr@~1.0.1: resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" integrity sha1-0/wRS6BplaRexok/SEzrHXj19HY= -pseudomap@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" - integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM= - psl@^1.1.28: version "1.7.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.7.0.tgz#f1c4c47a8ef97167dea5d6bbf4816d736e884a3c" @@ -10779,6 +10704,11 @@ queue-microtask@^1.2.2: resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.2.tgz#abf64491e6ecf0f38a6502403d4cda04f372dfd3" integrity sha512-dB15eXv3p2jDlbOiNLyMabYg1/sXvppd8DP2J3EOCQ0AkuSXCW2tP7mnVouVLJKgUMY6yP0kcQDVpLCN13h4Xg== +quick-lru@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f" + integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g== + raf-schd@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/raf-schd/-/raf-schd-4.0.2.tgz#bd44c708188f2e84c810bf55fcea9231bcaed8a0" @@ -10983,7 +10913,7 @@ react-router@5.2.0, react-router@^5.2.0: tiny-invariant "^1.0.2" tiny-warning "^1.0.0" -react-scripts@^4.0.0: +react-scripts@^4.0.3: version "4.0.3" resolved "https://registry.yarnpkg.com/react-scripts/-/react-scripts-4.0.3.tgz#b1cafed7c3fa603e7628ba0f187787964cb5d345" integrity sha512-S5eO4vjUzUisvkIPB7jVsKtuH2HhWcASREYWHAQ1FP5HyCv3xgn+wpILAEWkmy+A+tTNbSZClhxjT3qz6g4L1A== @@ -11112,14 +11042,6 @@ react@^17.0.1: loose-envify "^1.1.0" object-assign "^4.1.1" -read-pkg-up@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-1.0.1.tgz#9d63c13276c065918d57f002a57f40a1b643fb02" - integrity sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI= - dependencies: - find-up "^1.0.0" - read-pkg "^1.0.0" - read-pkg-up@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-2.0.0.tgz#6b72a8048984e0c41e79510fd5e9fa99b3b549be" @@ -11137,15 +11059,6 @@ read-pkg-up@^7.0.1: read-pkg "^5.2.0" type-fest "^0.8.1" -read-pkg@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-1.1.0.tgz#f5ffaa5ecd29cb31c0474bca7d756b6bb29e3f28" - integrity sha1-9f+qXs0pyzHAR0vKfXVra7KePyg= - dependencies: - load-json-file "^1.0.0" - normalize-package-data "^2.3.2" - path-type "^1.0.0" - read-pkg@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-2.0.0.tgz#8ef1c0623c6a6db0dc6713c4bfac46332b2368f8" @@ -11210,13 +11123,13 @@ recursive-readdir@2.2.2: dependencies: minimatch "3.0.4" -redent@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/redent/-/redent-1.0.0.tgz#cf916ab1fd5f1f16dfb20822dd6ec7f730c2afde" - integrity sha1-z5Fqsf1fHxbfsggi3W7H9zDCr94= +redent@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/redent/-/redent-3.0.0.tgz#e557b7998316bb53c9f1f56fa626352c6963059f" + integrity sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg== dependencies: - indent-string "^2.1.0" - strip-indent "^1.0.1" + indent-string "^4.0.0" + strip-indent "^3.0.0" redux@^4.0.4: version "4.0.5" @@ -11359,13 +11272,6 @@ repeat-string@^1.6.1: resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= -repeating@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/repeating/-/repeating-2.0.1.tgz#5214c53a926d3552707527fbab415dbc08d06dda" - integrity sha1-UhTFOpJtNVJwdSf7q0FdvAjQbdo= - dependencies: - is-finite "^1.0.0" - request-promise-core@1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.4.tgz#3eedd4223208d419867b78ce815167d10593a22f" @@ -11382,7 +11288,7 @@ request-promise-native@^1.0.8: stealthy-require "^1.1.1" tough-cookie "^2.3.3" -request@^2.87.0, request@^2.88.0, request@^2.88.2: +request@^2.88.0, request@^2.88.2: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== @@ -11556,7 +11462,7 @@ rifm@^0.7.0: dependencies: "@babel/runtime" "^7.3.1" -rimraf@2, rimraf@^2.5.4, rimraf@^2.6.3: +rimraf@^2.5.4, rimraf@^2.6.3: version "2.7.1" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== @@ -11809,10 +11715,12 @@ semver@^7.2.1, semver@^7.3.2: dependencies: lru-cache "^6.0.0" -semver@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" - integrity sha1-myzl094C0XxgEq0yaqa00M9U+U8= +semver@^7.3.4: + version "7.3.5" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" + integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ== + dependencies: + lru-cache "^6.0.0" send@0.17.1: version "0.17.1" @@ -12473,13 +12381,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-bom@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e" - integrity sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4= - dependencies: - is-utf8 "^0.2.0" - strip-bom@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" @@ -12508,12 +12409,12 @@ strip-final-newline@^2.0.0: resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad" integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== -strip-indent@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/strip-indent/-/strip-indent-1.0.1.tgz#0c7962a6adefa7bbd4ac366460a638552ae1a0a2" - integrity sha1-DHlipq3vp7vUrDZkYKY4VSrhoKI= +strip-indent@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/strip-indent/-/strip-indent-3.0.0.tgz#c32e1cee940b6b3432c771bc2c54bcce73cd3001" + integrity sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ== dependencies: - get-stdin "^4.0.1" + min-indent "^1.0.0" strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: version "3.1.1" @@ -12633,15 +12534,6 @@ tapable@^1.0.0, tapable@^1.1.3: resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2" integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA== -tar@^2.0.0: - version "2.2.2" - resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.2.tgz#0ca8848562c7299b8b446ff6a4d60cdbb23edc40" - integrity sha512-FCEhQ/4rE1zYv9rYXJw/msRqsnmlje5jHP6huWeBZ704jUTy02c5AZyWujpMR1ax6mVw9NyJMfuK2CMDWVIfgA== - dependencies: - block-stream "*" - fstream "^1.0.12" - inherits "2" - tar@^6.0.2: version "6.1.0" resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.0.tgz#d1724e9bcc04b977b18d5c573b333a2207229a83" @@ -12873,10 +12765,10 @@ tr46@^2.0.2: dependencies: punycode "^2.1.1" -trim-newlines@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-1.0.0.tgz#5887966bb582a4503a41eb524f7d35011815a613" - integrity sha1-WIeWa7WCpFA6QetST301ARgVphM= +trim-newlines@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144" + integrity sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw== "true-case-path@^1.0.2": version "1.0.3" @@ -12999,6 +12891,11 @@ type-fest@^0.11.0: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.11.0.tgz#97abf0872310fed88a5c466b25681576145e33f1" integrity sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ== +type-fest@^0.18.0: + version "0.18.1" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.18.1.tgz#db4bc151a4a2cf4eebf9add5db75508db6cc841f" + integrity sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw== + type-fest@^0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.3.1.tgz#63d00d204e059474fe5e1b7c011112bbd1dc29e1" @@ -13524,7 +13421,7 @@ which-module@^2.0.0: resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" integrity sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho= -which@1, which@^1.2.9, which@^1.3.1: +which@^1.2.9, which@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== @@ -13782,11 +13679,6 @@ y18n@^4.0.0: resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b" integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w== -yallist@^2.1.2: - version "2.1.2" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" - integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI= - yallist@^3.0.2: version "3.1.1" resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" @@ -13825,6 +13717,11 @@ yargs-parser@^18.1.2: camelcase "^5.0.0" decamelize "^1.2.0" +yargs-parser@^20.2.3: + version "20.2.9" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee" + integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== + yargs@^13.3.2: version "13.3.2" resolved "https://registry.yarnpkg.com/yargs/-/yargs-13.3.2.tgz#ad7ffefec1aa59565ac915f82dccb38a9c31a2dd" From bc44f9b8d1b040ec386fb0e9581d00a6e4e5f526 Mon Sep 17 00:00:00 2001 From: reesporte Date: Wed, 27 Oct 2021 16:45:17 -0500 Subject: [PATCH 17/40] remove shards list from error message entirely --- http/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/client.go b/http/client.go index 587cca17d..ca3c52c11 100644 --- a/http/client.go +++ b/http/client.go @@ -522,7 +522,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index str // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { - return nil, errors.Wrapf(err, "'%s', shards %v", queryRequest.Query, pilosa.ShardSlice(queryRequest.Shards)) + return nil, errors.Wrapf(err, "'%s'", queryRequest.Query) } defer resp.Body.Close() From 968ce78c7325e899b65a76e1ebe734b2bebd5991 Mon Sep 17 00:00:00 2001 From: reesporte Date: Wed, 27 Oct 2021 17:05:56 -0500 Subject: [PATCH 18/40] remove ShardSlice entirely --- executor.go | 91 +++++++++++++++++------------------------------------ 1 file changed, 28 insertions(+), 63 deletions(-) diff --git a/executor.go b/executor.go index 0d8954085..82f215fef 100644 --- a/executor.go +++ b/executor.go @@ -605,41 +605,6 @@ func (e *executor) preprocessQuery(ctx context.Context, qcx *Qcx, index string, } } -type ShardSlice []uint64 - -// String creates a run-length encoded representation of a slice of shard IDs (integers). -// For example, []uint64{0, 1, 3, 4, 5, 7, 8, 9, 11, 13} is represented as -// [0-1,3-5,7-9,11,13]. -func (s ShardSlice) String() string { - if len(s) == 0 { - // surely this is impossible - return "[]" - } - runs := make([]string, 0, len(s)/2) - start := s[0] - end := start - for n := 1; n < len(s); n++ { - if s[n] == end+1 { - end = s[n] - } else { - repr := fmt.Sprintf("%d", start) - if end > start { - repr += fmt.Sprintf("-%d", end) - } - runs = append(runs, repr) - start = s[n] - end = start - } - } - repr := fmt.Sprintf("%d", start) - if end > start { - repr += fmt.Sprintf("-%d", end) - } - runs = append(runs, repr) - - return "[" + strings.Join(runs, ",") + "]" -} - // executeCall executes a call. func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCall") @@ -689,105 +654,105 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p case "Sum": statFn() res, err := e.executeSum(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeSum %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeSum") case "Min": statFn() res, err := e.executeMin(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMin %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeMin") case "Max": statFn() res, err := e.executeMax(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMax %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeMax") case "MinRow": statFn() res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMinRow %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeMinRow") case "MaxRow": statFn() res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMaxRow %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeMaxRow") case "Clear": statFn() res, err := e.executeClearBit(ctx, qcx, index, c, opt) - return res, errors.Wrapf(err, "executeClearBit %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeClearBit") case "ClearRow": statFn() res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeClearRow %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeClearRow") case "Distinct": statFn() res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeDistinct %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeDistinct") case "Store": statFn() res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeSetRow %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeSetRow") case "Count": statFn() res, err := e.executeCount(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeCount %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeCount") case "Set": statFn() res, err := e.executeSet(ctx, qcx, index, c, opt) - return res, errors.Wrapf(err, "executeSet %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeSet") case "TopK": statFn() res, err := e.executeTopK(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeTopK %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeTopK") case "TopN": statFn() res, err := e.executeTopN(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeTopN %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeTopN") case "Rows": statFn() res, err := e.executeRows(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeRows %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeRows") case "ExternalLookup": statFn() res, err := e.executeExternalLookup(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeExternalLookup %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeExternalLookup") case "Extract": statFn() res, err := e.executeExtract(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeExtract %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeExtract") case "GroupBy": statFn() res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeGroupBy %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeGroupBy") case "Options": statFn() res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeOptionsCall %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeOptionsCall") case "IncludesColumn": res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeIncludesColumnCall %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeIncludesColumnCall") case "FieldValue": statFn() res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeFieldValueCall %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeFieldValueCall") case "Precomputed": res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executePrecomputedCall %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executePrecomputedCall") case "UnionRows": res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeUnionRows %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeUnionRows") case "ConstRow": res, err := e.executeConstRow(ctx, index, c) - return res, errors.Wrapf(err, "executeConstRow %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeConstRow") case "Limit": res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeLimitCall %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeLimitCall") case "Percentile": res, err := e.executePercentile(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executePercentile %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executePercentile") case "Delete": statFn() //TODO(twg) need this? res, err := e.executeDeleteRecords(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeDelete %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeDelete") default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap. statFn() res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeBitmapCall %v", ShardSlice(shards)) + return res, errors.Wrap(err, "executeBitmapCall") } } @@ -2939,7 +2904,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, errors.Wrapf(err, "mapReduce shards: %v", ShardSlice(shards)) + return nil, errors.Wrap(err, "mapReduce") } results, _ := other.([]GroupCount) From c3e14cb9ae2ac98e307f17d60b1a248aebb7e513 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 27 Oct 2021 15:08:58 -0500 Subject: [PATCH 19/40] don't fsync on RBF Open if WAL is empty This is targeted at reducing startup times, especially on OSX where the fsync calls seem to be taking an egregiously long time. I got one index to go from ~1min to open to ~1sec. This looks safe to me, but will get opinions from RBF experts. --- rbf/db.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rbf/db.go b/rbf/db.go index 4f88de259..68a41ac08 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -206,6 +206,13 @@ func (db *DB) checkpoint() error { return nil // skip if transactions open } + // Check if there are any WAL pages, if not do nothing as + // checkpointing and calling fsync can be very expensive even if + // there are no writes. + if db.walPageN == 0 { + return nil + } + for i := 0; i < db.walPageN; i++ { page, err := db.readWALPageAt(i) if err != nil { @@ -245,7 +252,7 @@ func (db *DB) checkpoint() error { db.walPageN = 0 db.pageMap = NewPageMap() - // Notify halted tranactions that the WAL has been checkpointed. + // Notify halted transactions that the WAL has been checkpointed. db.haltCond.Broadcast() return nil From b787fccf3a356ad9cc6c54799008f1eca44282ca Mon Sep 17 00:00:00 2001 From: Samir Patel <48686912+54mir@users.noreply.github.com> Date: Thu, 28 Oct 2021 10:25:26 -0500 Subject: [PATCH 20/40] Add cmd option to disable cardinality calc --- api.go | 17 +++++++++++++++-- ctl/server.go | 3 +++ server/config.go | 6 ++++++ server/server.go | 1 + 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index 7bcc0dda7..f7d9001f4 100644 --- a/api.go +++ b/api.go @@ -62,7 +62,8 @@ type API struct { importWorkerPoolSize int importWork chan importJob - usageCache *usageCache + usageCache *usageCache + schemaDetailsOn bool Serializer Serializer } @@ -84,6 +85,14 @@ func OptAPIServer(s *Server) apiOption { } } +// Used to configure API option: schemaDetailsOn +func OptAPISchemaDetailsOn(isOn bool) apiOption { + return func(a *API) error { + a.schemaDetailsOn = isOn + return nil + } +} + func OptAPIImportWorkerPoolSize(size int) apiOption { return func(a *API) error { a.importWorkerPoolSize = size @@ -1237,7 +1246,8 @@ func (api *API) Schema(ctx context.Context, withViews bool) ([]*IndexInfo, error } // SchemaDetails returns information about each index in Pilosa including which -// fields they contain, and additional field information such as cardinality +// fields they contain. Additional field information such as cardinality unless +// turned off via the schemaDetailsOn cli option. func (api *API) SchemaDetails(ctx context.Context) ([]*IndexInfo, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.Schema") defer span.Finish() @@ -1245,6 +1255,9 @@ func (api *API) SchemaDetails(ctx context.Context) ([]*IndexInfo, error) { if err != nil { return nil, errors.Wrap(err, "getting schema") } + if !api.schemaDetailsOn { + return schema, nil + } for _, index := range schema { for _, field := range index.Fields { q := fmt.Sprintf("Count(Distinct(field=%s))", field.Name) diff --git a/ctl/server.go b/ctl/server.go index 71c26bb3c..d53bf81af 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -118,4 +118,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // Future flags. flags.BoolVar(&srv.Config.Future.Rename, "future.rename", false, "Present application name as FeatureBase. Defaults to false, will default to true in an upcoming release.") + + // Toggle /schema/details endpoint. + flags.BoolVar(&srv.Config.SchemaDetailsOn, "schema-details-on", true, "Disable /schema/details endpoint") } diff --git a/server/config.go b/server/config.go index 344440ded..a42eb7917 100644 --- a/server/config.go +++ b/server/config.go @@ -237,6 +237,9 @@ type Config struct { // as FeatureBase instead of Pilosa. Rename bool `toml:"rename"` } `toml:"future"` + + // Toggles /schema/details endpoint. If off, it returns empty. + SchemaDetailsOn bool `toml:"schema-details-on"` } // Namespace returns the namespace to use based on the Future flag. @@ -386,6 +389,9 @@ func NewConfig() *Config { // Future flags. c.Future.Rename = false + // Schema Details Toggle + c.SchemaDetailsOn = true + return c } diff --git a/server/server.go b/server/server.go index 8d227ade7..929404852 100644 --- a/server/server.go +++ b/server/server.go @@ -511,6 +511,7 @@ func (m *Command) SetupServer() error { m.API, err = pilosa.NewAPI( pilosa.OptAPIServer(m.Server), pilosa.OptAPIImportWorkerPoolSize(m.Config.ImportWorkerPoolSize), + pilosa.OptAPISchemaDetailsOn(m.Config.SchemaDetailsOn), ) if err != nil { return errors.Wrap(err, "new api") From aadfc63bf0563e4538b6232f9219105305d83178 Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 28 Oct 2021 13:45:07 -0500 Subject: [PATCH 21/40] add quotes around strings --- lattice/src/shared/DataTable/DataTable.tsx | 42 ++++++++++++++-------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/lattice/src/shared/DataTable/DataTable.tsx b/lattice/src/shared/DataTable/DataTable.tsx index 7d7b53f1a..e080c1cf4 100644 --- a/lattice/src/shared/DataTable/DataTable.tsx +++ b/lattice/src/shared/DataTable/DataTable.tsx @@ -70,6 +70,32 @@ export const DataTable: FC = ({ } }; + const formatTableCell = (row: any, col: any) => { + if (typeof row[col.name] === 'object') { + return ( + 
+        {JSON.stringify(row[col.name], null, 2)}
+
) + } else if (row[col.name] !== undefined) { + if (col.datatype === '[]string') { + return ( + + {'"' + (row[col.name]) + '"'} + + ) + } + return ( + + {col.datatype === 'timestamp' && row[col.name] + ? moment + .utc(row[col.name]) + .format('MM/DD/YYYY hh:mm:ss a') + : row[col.name].toLocaleString()} + ) + } + return null + } + return (
@@ -117,20 +143,8 @@ export const DataTable: FC = ({ key={`table-cell-${rowIdx}-${colIdx}`} className={css.tableCell} > - {typeof row[col.name] === 'object' ? ( - 
-                            {JSON.stringify(row[col.name], null, 2)}
-
- ) : row[col.name] !== undefined ? ( - - {col.datatype === 'timestamp' && row[col.name] - ? moment - .utc(row[col.name]) - .format('MM/DD/YYYY hh:mm:ss a') - : row[col.name].toLocaleString()} - - ) : null} - + {formatTableCell(row, col)} + ))} {autoWidth ? : null} From 360f161303f0169e64af751468c7b417b59505bd Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 28 Oct 2021 14:03:52 -0500 Subject: [PATCH 22/40] uprev gopsutil The old revision emits a warning on MacOS X that looks concerning, and even though it's actually mostly-harmless, it is an annoyance. Also run `go mod tidy` which affected go.sum. --- go.mod | 3 +-- go.sum | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 67d9c30bd..25e8925e5 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/rs/cors v1.7.0 // indirect github.com/satori/go.uuid v1.2.0 - github.com/shirou/gopsutil/v3 v3.20.11 + github.com/shirou/gopsutil/v3 v3.21.9 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 @@ -51,7 +51,6 @@ require ( golang.org/x/mod v0.4.2 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect golang.org/x/text v0.3.5 // indirect google.golang.org/grpc v1.28.0 gopkg.in/yaml.v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 60db445b5..7f90a4524 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgR github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -92,8 +92,8 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= @@ -283,8 +283,8 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554 h1:88K0ffxhVphUHxlqW4ewOaXdnJByH4LcCuvYfv0QI/M= github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA= -github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4= +github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg= +github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= @@ -319,11 +319,14 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= +github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -435,17 +438,17 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71 h1:ikCpsnYR+Ew0vu99XlDp55lGgDJdIMx3f4a18jfse/s= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 5ef7975475f19f8acc0c86fe81876bec4aa0a43b Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 28 Oct 2021 15:15:43 -0500 Subject: [PATCH 23/40] refactor and add tests --- lattice/src/shared/DataTable/DataTable.tsx | 29 +-------- .../src/shared/utils/formatTableCell.test.tsx | 59 +++++++++++++++++++ lattice/src/shared/utils/formatTableCell.tsx | 26 ++++++++ 3 files changed, 88 insertions(+), 26 deletions(-) create mode 100644 lattice/src/shared/utils/formatTableCell.test.tsx create mode 100644 lattice/src/shared/utils/formatTableCell.tsx diff --git a/lattice/src/shared/DataTable/DataTable.tsx b/lattice/src/shared/DataTable/DataTable.tsx index e080c1cf4..5a84b8329 100644 --- a/lattice/src/shared/DataTable/DataTable.tsx +++ b/lattice/src/shared/DataTable/DataTable.tsx @@ -12,6 +12,7 @@ import TableRow from '@material-ui/core/TableRow'; import Typography from '@material-ui/core/Typography'; import { ColumnInfo } from 'proto/pilosa_pb'; import { Pager } from 'shared/Pager'; +import { formatTableCell } from 'shared/utils/formatTableCell'; import css from './DataTable.module.scss'; type TableProps = { @@ -70,31 +71,7 @@ export const DataTable: FC = ({ } }; - const formatTableCell = (row: any, col: any) => { - if (typeof row[col.name] === 'object') { - return ( - 
-        {JSON.stringify(row[col.name], null, 2)}
-
) - } else if (row[col.name] !== undefined) { - if (col.datatype === '[]string') { - return ( - - {'"' + (row[col.name]) + '"'} - - ) - } - return ( - - {col.datatype === 'timestamp' && row[col.name] - ? moment - .utc(row[col.name]) - .format('MM/DD/YYYY hh:mm:ss a') - : row[col.name].toLocaleString()} - ) - } - return null - } + return ( @@ -143,7 +120,7 @@ export const DataTable: FC = ({ key={`table-cell-${rowIdx}-${colIdx}`} className={css.tableCell} > - {formatTableCell(row, col)} + {formatTableCell(row, col, css)} ))} {autoWidth ? : null} diff --git a/lattice/src/shared/utils/formatTableCell.test.tsx b/lattice/src/shared/utils/formatTableCell.test.tsx new file mode 100644 index 000000000..9db2b0b84 --- /dev/null +++ b/lattice/src/shared/utils/formatTableCell.test.tsx @@ -0,0 +1,59 @@ +import { formatTableCell } from './formatTableCell'; +import React from "react"; +import { render, unmountComponentAtNode } from "react-dom"; +import { act } from "react-dom/test-utils"; + + +let container = null; +beforeEach(() => { + // setup a DOM element as a render target + container = document.createElement("div"); + document.body.appendChild(container); +}); + +afterEach(() => { + // cleanup on exiting + unmountComponentAtNode(container); + container.remove(); + container = null; +}); + +it("it renders strings in quotes", () => { + let row = {thing:"quoted string!"}; + let col = {name: "thing", datatype: "[]string"}; + // this is just so we can actually test it, the value doesn't matter that much + let css = {preFormat: "preFormat"}; + + act(() => { + render(formatTableCell(row, col, css), container); + }); + expect(container.textContent).toBe('"quoted string!"'); +}); + +it("renders objects as stringified", () => { + let row = {thing:{val:"quoted string!"}}; + let col = {name: "thing", datatype: "object"}; + // this is just so we can actually test it, the value doesn't matter that much + let css = {preFormat: "preFormat"}; + + act(() => { + render(formatTableCell(row, col, css), container); + }); + expect(container.textContent).toBe(`{ + "val": "quoted string!" +}`); +}); + +it("it puts timestamps in MM/DD/YYYY hh:mm:ss a format", () => { + let row = {thing:1635452050094}; + let col = {name: "thing", datatype: "timestamp"}; + // this is just so we can actually test it, the value doesn't matter that much + let css = {preFormat: "preFormat"}; + + act(() => { + render(formatTableCell(row, col, css), container); + }); + expect(container.textContent).toBe("10/28/2021 08:14:10 pm"); +}); + + diff --git a/lattice/src/shared/utils/formatTableCell.tsx b/lattice/src/shared/utils/formatTableCell.tsx new file mode 100644 index 000000000..a8ff9f8c4 --- /dev/null +++ b/lattice/src/shared/utils/formatTableCell.tsx @@ -0,0 +1,26 @@ +import moment from 'moment'; +export const formatTableCell = (row: any, col: any, css: any) => { + if (typeof row[col.name] === 'object') { + return ( + 
+        {JSON.stringify(row[col.name], null, 2)}
+
) + } else if (row[col.name] !== undefined) { + if (col.datatype === '[]string') { + return ( + + {'"' + (row[col.name]) + '"'} + + ) + } + return ( + + {col.datatype === 'timestamp' && row[col.name] + ? moment + .utc(row[col.name]) + .format('MM/DD/YYYY hh:mm:ss a') + : row[col.name].toLocaleString()} + ) + } + return null +} From cdbfcef0ebf6f8d1582e46d8521357a2eb9c4159 Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 28 Oct 2021 16:19:03 -0500 Subject: [PATCH 24/40] hopefully updates the ci pipeline to run jest i updated the gitlab ci pipeline so that hopefully it will run jest coverage stuff for sonar --- .gitlab/.gitlab-ci.yml | 13 +++++++++++-- lattice/package.json | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 0e41bfd1e..97eb7de80 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -64,16 +64,25 @@ generate coverage html: needs: - job: run go tests +run jest tests: + stage: test + image: sonarsource/sonar-scanner-cli:latest + script: + - npm install + - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter + - sonar-scanner + upload to sonarcloud: stage: test image: sonarsource/sonar-scanner-cli:4.6 variables: SONAR_TOKEN: $SONAR_TOKEN script: - - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out + - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.testExecutionReportPaths=test-report.xml needs: - job: run go tests - job: run go tests with output + - job: run jest tests build for linux amd64: stage: build @@ -121,4 +130,4 @@ build for darwin arm64: - go build -o featurebase_darwin_arm64 ./cmd/featurebase artifacts: paths: - - featurebase_darwin_arm64 \ No newline at end of file + - featurebase_darwin_arm64 diff --git a/lattice/package.json b/lattice/package.json index e50cc0bdb..48785d4c7 100644 --- a/lattice/package.json +++ b/lattice/package.json @@ -44,6 +44,7 @@ "@types/react-dom": "^16.9.8", "@types/react-router": "^5.1.8", "@types/react-router-dom": "^5.1.5", + "jest-sonar-reporter": "^2.0.0", "node-sass": "^4.14.1", "react-scripts": "^4.0.3", "tslint": "^6.1.2", From ae69ad0d4e2fec7017a6977247062bc59aa1b3d2 Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 28 Oct 2021 16:21:58 -0500 Subject: [PATCH 25/40] add artifact path(s) --- .gitlab/.gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 97eb7de80..8f3612108 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -71,6 +71,9 @@ run jest tests: - npm install - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter - sonar-scanner + artifacts: + paths: + - coverage/lcov.info upload to sonarcloud: stage: test From 04642dad0fbcf81dd4bae9c582a9d2032c9316c3 Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 29 Oct 2021 08:17:10 -0500 Subject: [PATCH 26/40] hopefully sets up jest test coverage in gitlab --- .gitlab/.gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8f3612108..ddf1c5a66 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -66,14 +66,15 @@ generate coverage html: run jest tests: stage: test - image: sonarsource/sonar-scanner-cli:latest + image: sonarsource/sonar-scanner-cli:4.6 script: + - cd lattice - npm install - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter - sonar-scanner artifacts: paths: - - coverage/lcov.info + - lattice/coverage/lcov.info upload to sonarcloud: stage: test From 8bf4d196e68cf75f3c63aeaf96222c65cdcc2846 Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 29 Oct 2021 08:31:08 -0500 Subject: [PATCH 27/40] set up jest test coverage in gitlab ci --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index ddf1c5a66..ff3210c09 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -66,7 +66,7 @@ generate coverage html: run jest tests: stage: test - image: sonarsource/sonar-scanner-cli:4.6 + image: sonarsource/sonar-scanner-cli:latest script: - cd lattice - npm install From f6de92ecdacaa8ca1298d1e116f1cb74134b89a6 Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 29 Oct 2021 09:17:12 -0500 Subject: [PATCH 28/40] set up jest test coverage in gitlab ci --- .gitlab/.gitlab-ci.yml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index ff3210c09..79a3c3329 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -32,6 +32,21 @@ build lattice: - job: install lattice allow_failure: true +run jest tests: + stage: test + image: node:14 + variables: + CI: "true" + script: + - echo "Testing lattice..." + - cd lattice + - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter + artifacts: + paths: + - lattice/coverage/lcov.info + needs: + - job: install lattice + run go tests: stage: test image: golang:1.16.9 @@ -64,18 +79,6 @@ generate coverage html: needs: - job: run go tests -run jest tests: - stage: test - image: sonarsource/sonar-scanner-cli:latest - script: - - cd lattice - - npm install - - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter - - sonar-scanner - artifacts: - paths: - - lattice/coverage/lcov.info - upload to sonarcloud: stage: test image: sonarsource/sonar-scanner-cli:4.6 From 029fad1fb77bf1edb5647c6a1fc824a027fa2514 Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 29 Oct 2021 09:34:20 -0500 Subject: [PATCH 29/40] hopefully this works to set up test coverage --- .gitlab/.gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 79a3c3329..7563891de 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -40,12 +40,11 @@ run jest tests: script: - echo "Testing lattice..." - cd lattice + - npm install --force - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter artifacts: paths: - lattice/coverage/lcov.info - needs: - - job: install lattice run go tests: stage: test From ff435708812a57b6cc74b807ed7fd55b87eabc19 Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 29 Oct 2021 10:32:32 -0500 Subject: [PATCH 30/40] hopefully this works --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 7563891de..0a54fdb9b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -84,7 +84,7 @@ upload to sonarcloud: variables: SONAR_TOKEN: $SONAR_TOKEN script: - - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.testExecutionReportPaths=test-report.xml + - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out needs: - job: run go tests - job: run go tests with output From e520277c71a26af77bc42884648c36b51e0113b6 Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 29 Oct 2021 11:33:40 -0500 Subject: [PATCH 31/40] please work holy moly --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 0a54fdb9b..bb734ca27 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -84,7 +84,7 @@ upload to sonarcloud: variables: SONAR_TOKEN: $SONAR_TOKEN script: - - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out + - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info needs: - job: run go tests - job: run go tests with output From f04cb01e5d40fcfc12eb0ebbabc6253c5c094814 Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 29 Oct 2021 11:56:09 -0500 Subject: [PATCH 32/40] meaningless commit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e221fbb1d..eba94455b 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,4 @@ See our [internal documentation](https://internal-docs.molecula.cloud), which includes all [external documentation](https://docs.molecula.cloud), plus many internal-only pages, listed under the "Internal" heading in the main navigation bar. Follow along with the [Sample Project](https://internal-docs.molecula.cloud/tutorials/getting-started) to get a better understanding of FeatureBase's capabilities. + From ad0b6db84172845afbc032a73696036f90aea3fe Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 29 Oct 2021 16:04:58 -0500 Subject: [PATCH 33/40] refactor css as import --- lattice/src/shared/DataTable/DataTable.tsx | 3 +- .../src/shared/utils/formatTableCell.test.tsx | 12 +- lattice/src/shared/utils/formatTableCell.tsx | 3 +- lattice/yarn.lock | 461 +++++++++++------- 4 files changed, 288 insertions(+), 191 deletions(-) diff --git a/lattice/src/shared/DataTable/DataTable.tsx b/lattice/src/shared/DataTable/DataTable.tsx index 5a84b8329..94510f86d 100644 --- a/lattice/src/shared/DataTable/DataTable.tsx +++ b/lattice/src/shared/DataTable/DataTable.tsx @@ -1,7 +1,6 @@ import React, { FC, Fragment, useEffect, useRef, useState } from 'react'; import ArrowDropDownIcon from '@material-ui/icons/ArrowDropDown'; import classNames from 'classnames'; -import moment from 'moment'; import OrderBy from 'lodash/orderBy'; import Table from '@material-ui/core/Table'; import TableBody from '@material-ui/core/TableBody'; @@ -120,7 +119,7 @@ export const DataTable: FC = ({ key={`table-cell-${rowIdx}-${colIdx}`} className={css.tableCell} > - {formatTableCell(row, col, css)} + {formatTableCell(row, col)} ))} {autoWidth ? : null} diff --git a/lattice/src/shared/utils/formatTableCell.test.tsx b/lattice/src/shared/utils/formatTableCell.test.tsx index 9db2b0b84..cbbc26ca3 100644 --- a/lattice/src/shared/utils/formatTableCell.test.tsx +++ b/lattice/src/shared/utils/formatTableCell.test.tsx @@ -21,11 +21,9 @@ afterEach(() => { it("it renders strings in quotes", () => { let row = {thing:"quoted string!"}; let col = {name: "thing", datatype: "[]string"}; - // this is just so we can actually test it, the value doesn't matter that much - let css = {preFormat: "preFormat"}; act(() => { - render(formatTableCell(row, col, css), container); + render(formatTableCell(row, col), container); }); expect(container.textContent).toBe('"quoted string!"'); }); @@ -33,11 +31,9 @@ it("it renders strings in quotes", () => { it("renders objects as stringified", () => { let row = {thing:{val:"quoted string!"}}; let col = {name: "thing", datatype: "object"}; - // this is just so we can actually test it, the value doesn't matter that much - let css = {preFormat: "preFormat"}; act(() => { - render(formatTableCell(row, col, css), container); + render(formatTableCell(row, col), container); }); expect(container.textContent).toBe(`{ "val": "quoted string!" @@ -47,11 +43,9 @@ it("renders objects as stringified", () => { it("it puts timestamps in MM/DD/YYYY hh:mm:ss a format", () => { let row = {thing:1635452050094}; let col = {name: "thing", datatype: "timestamp"}; - // this is just so we can actually test it, the value doesn't matter that much - let css = {preFormat: "preFormat"}; act(() => { - render(formatTableCell(row, col, css), container); + render(formatTableCell(row, col), container); }); expect(container.textContent).toBe("10/28/2021 08:14:10 pm"); }); diff --git a/lattice/src/shared/utils/formatTableCell.tsx b/lattice/src/shared/utils/formatTableCell.tsx index a8ff9f8c4..9dd428ef8 100644 --- a/lattice/src/shared/utils/formatTableCell.tsx +++ b/lattice/src/shared/utils/formatTableCell.tsx @@ -1,5 +1,6 @@ import moment from 'moment'; -export const formatTableCell = (row: any, col: any, css: any) => { +import css from '../DataTable/DataTable.module.scss'; +export const formatTableCell = (row: any, col: any) => { if (typeof row[col.name] === 'object') { return ( 
diff --git a/lattice/yarn.lock b/lattice/yarn.lock
index 31499a6ea..914b40f4b 100644
--- a/lattice/yarn.lock
+++ b/lattice/yarn.lock
@@ -2443,11 +2443,6 @@
   resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
   integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
 
-"@types/minimist@^1.2.0":
-  version "1.2.2"
-  resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c"
-  integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==
-
 "@types/node@*":
   version "13.9.2"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-13.9.2.tgz#ace1880c03594cc3e80206d96847157d8e7fa349"
@@ -3113,6 +3108,11 @@ arr-union@^3.1.0:
   resolved "https://registry.yarnpkg.com/arr-union/-/arr-union-3.1.0.tgz#e39b09aea9def866a8f206e288af63919bae39c4"
   integrity sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=
 
+array-find-index@^1.0.1:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/array-find-index/-/array-find-index-1.0.2.tgz#df010aa1287e164bbda6f9723b0a96a1ec4187a1"
+  integrity sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E=
+
 array-flatten@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2"
@@ -3184,11 +3184,6 @@ array.prototype.flatmap@^1.2.3:
     es-abstract "^1.18.0-next.1"
     function-bind "^1.1.1"
 
-arrify@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d"
-  integrity sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=
-
 arrify@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa"
@@ -3575,6 +3570,13 @@ bindings@^1.5.0:
   dependencies:
     file-uri-to-path "1.0.0"
 
+block-stream@*:
+  version "0.0.9"
+  resolved "https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a"
+  integrity sha1-E+v+d4oDIFz+A3UUgeu0szAMEmo=
+  dependencies:
+    inherits "~2.0.0"
+
 bluebird@^3.5.5:
   version "3.7.2"
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
@@ -3914,20 +3916,24 @@ camel-case@^4.1.1:
     pascal-case "^3.1.1"
     tslib "^1.10.0"
 
-camelcase-keys@^6.2.2:
-  version "6.2.2"
-  resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-6.2.2.tgz#5e755d6ba51aa223ec7d3d52f25778210f9dc3c0"
-  integrity sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg==
+camelcase-keys@^2.0.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-2.1.0.tgz#308beeaffdf28119051efa1d932213c91b8f92e7"
+  integrity sha1-MIvur/3ygRkFHvodkyITyRuPkuc=
   dependencies:
-    camelcase "^5.3.1"
-    map-obj "^4.0.0"
-    quick-lru "^4.0.1"
+    camelcase "^2.0.0"
+    map-obj "^1.0.0"
 
 camelcase@5.3.1, camelcase@^5.0.0, camelcase@^5.3.1:
   version "5.3.1"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320"
   integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==
 
+camelcase@^2.0.0:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-2.1.1.tgz#7c1d16d679a1bbe59ca02cacecfb011e201f5a1f"
+  integrity sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8=
+
 camelcase@^6.0.0, camelcase@^6.1.0, camelcase@^6.2.0:
   version "6.2.0"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.2.0.tgz#924af881c9d525ac9d87f40d964e5cea982a1809"
@@ -4473,7 +4479,7 @@ create-hmac@^1.1.0, create-hmac@^1.1.2, create-hmac@^1.1.4:
     safe-buffer "^5.0.1"
     sha.js "^2.4.8"
 
-cross-spawn@7.0.3, cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3:
+cross-spawn@7.0.3, cross-spawn@^7.0.0, cross-spawn@^7.0.2:
   version "7.0.3"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
   integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
@@ -4482,6 +4488,14 @@ cross-spawn@7.0.3, cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3:
     shebang-command "^2.0.0"
     which "^2.0.1"
 
+cross-spawn@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-3.0.1.tgz#1256037ecb9f0c5f79e3d6ef135e30770184b982"
+  integrity sha1-ElYDfsufDF9549bvE14wdwGEuYI=
+  dependencies:
+    lru-cache "^4.0.1"
+    which "^1.2.9"
+
 cross-spawn@^6.0.0:
   version "6.0.5"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
@@ -4769,6 +4783,13 @@ csstype@^2.5.5:
   resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.6.13.tgz#a6893015b90e84dd6e85d0e3b442a1e84f2dbe0f"
   integrity sha512-ul26pfSQTZW8dcOnD2iiJssfXw0gdNVX9IJDH/X3K5DGPfj+fUYe3kB+swUY6BF3oZDxaID3AJt+9/ojSAE05A==
 
+currently-unhandled@^0.4.1:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea"
+  integrity sha1-mI3zP+qxke95mmE2nddsF635V+o=
+  dependencies:
+    array-find-index "^1.0.1"
+
 cyclist@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9"
@@ -4956,15 +4977,7 @@ debug@^4.0.1, debug@^4.1.0, debug@^4.1.1:
   dependencies:
     ms "^2.1.1"
 
-decamelize-keys@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/decamelize-keys/-/decamelize-keys-1.1.0.tgz#d171a87933252807eb3cb61dc1c1445d078df2d9"
-  integrity sha1-0XGoeTMlKAfrPLYdwcFEXQeN8tk=
-  dependencies:
-    decamelize "^1.1.0"
-    map-obj "^1.0.0"
-
-decamelize@^1.1.0, decamelize@^1.2.0:
+decamelize@^1.1.2, decamelize@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
   integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=
@@ -5397,11 +5410,6 @@ entities@^2.0.0:
   resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.0.tgz#68d6084cab1b079767540d80e56a39b423e4abf4"
   integrity sha512-D9f7V0JSRwIxlRI2mjMqufDrRDnx8p+eEOz7aUM9SuvF8gsBzra0/6tbjl1m8eQHrZlYj6PxqE00hZ1SAIKPLw==
 
-env-paths@^2.2.0:
-  version "2.2.1"
-  resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.1.tgz#420399d416ce1fbe9bc0a07c62fa68d67fd0f8f2"
-  integrity sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==
-
 errno@^0.1.3, errno@~0.1.7:
   version "0.1.7"
   resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.7.tgz#4684d71779ad39af177e3f007996f7c67c852618"
@@ -6119,6 +6127,14 @@ find-up@4.1.0, find-up@^4.0.0, find-up@^4.1.0:
     locate-path "^5.0.0"
     path-exists "^4.0.0"
 
+find-up@^1.0.0:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f"
+  integrity sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=
+  dependencies:
+    path-exists "^2.0.0"
+    pinkie-promise "^2.0.0"
+
 find-up@^2.0.0, find-up@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/find-up/-/find-up-2.1.0.tgz#45d1b7e506c717ddd482775a2b77920a3c0c57a7"
@@ -6316,6 +6332,16 @@ fsevents@^2.1.2, fsevents@^2.1.3, fsevents@~2.3.1:
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
   integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
 
+fstream@^1.0.0, fstream@^1.0.12:
+  version "1.0.12"
+  resolved "https://registry.yarnpkg.com/fstream/-/fstream-1.0.12.tgz#4e8ba8ee2d48be4f7d0de505455548eae5932045"
+  integrity sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==
+  dependencies:
+    graceful-fs "^4.1.2"
+    inherits "~2.0.0"
+    mkdirp ">=0.5 0"
+    rimraf "2"
+
 function-bind@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"
@@ -6541,11 +6567,6 @@ graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.3.tgz#4a12ff1b60376ef09862c2093edd908328be8423"
   integrity sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==
 
-graceful-fs@^4.2.3:
-  version "4.2.8"
-  resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.8.tgz#e412b8d33f5e006593cbd3cee6df9f2cebbe802a"
-  integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==
-
 graceful-fs@^4.2.4:
   version "4.2.6"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee"
@@ -6582,11 +6603,6 @@ har-validator@~5.1.3:
     ajv "^6.5.5"
     har-schema "^2.0.0"
 
-hard-rejection@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/hard-rejection/-/hard-rejection-2.1.0.tgz#1c6eda5c1685c63942766d79bb40ae773cecd883"
-  integrity sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==
-
 harmony-reflect@^1.4.6:
   version "1.6.1"
   resolved "https://registry.yarnpkg.com/harmony-reflect/-/harmony-reflect-1.6.1.tgz#c108d4f2bb451efef7a37861fdbdae72c9bdefa9"
@@ -6731,13 +6747,6 @@ hosted-git-info@^2.1.4:
   resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488"
   integrity sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==
 
-hosted-git-info@^4.0.1:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-4.0.2.tgz#5e425507eede4fea846b7262f0838456c4209961"
-  integrity sha512-c9OGXbZ3guC/xOlCg1Ci/VgWlwsqDv1yMQL1CWqXDL0hDjXuNcq0zuR4xqPSuasI3kqFDhqSyTjREz5gzq0fXg==
-  dependencies:
-    lru-cache "^6.0.0"
-
 hpack.js@^2.1.6:
   version "2.1.6"
   resolved "https://registry.yarnpkg.com/hpack.js/-/hpack.js-2.1.6.tgz#87774c0949e513f42e84575b3c45681fade2a0b2"
@@ -7025,6 +7034,18 @@ imurmurhash@^0.1.4:
   resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea"
   integrity sha1-khi5srkoojixPcT7a21XbyMUU+o=
 
+in-publish@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/in-publish/-/in-publish-2.0.1.tgz#948b1a535c8030561cea522f73f78f4be357e00c"
+  integrity sha512-oDM0kUSNFC31ShNxHKUyfZKy8ZeXZBWMjMdZHKLOk13uvT27VTL/QzRGfRUcevJhpkZAvlhPYuXkF7eNWrtyxQ==
+
+indent-string@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-2.1.0.tgz#8e2d48348742121b4a8218b7a137e9a52049dc80"
+  integrity sha1-ji1INIdCEhtKghi3oTfppSBJ3IA=
+  dependencies:
+    repeating "^2.0.0"
+
 indent-string@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251"
@@ -7048,7 +7069,7 @@ inflight@^1.0.4:
     once "^1.3.0"
     wrappy "1"
 
-inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3:
+inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
   integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@@ -7220,13 +7241,6 @@ is-core-module@^2.0.0, is-core-module@^2.2.0:
   dependencies:
     has "^1.0.3"
 
-is-core-module@^2.5.0:
-  version "2.8.0"
-  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.8.0.tgz#0321336c3d0925e497fd97f5d95cb114a5ccd548"
-  integrity sha512-vd15qHsaqrRL7dtH6QNuy0ndJmRDrS9HAM1CAiSifNUFv4x1a0CCVsj18hJ1mShxIG6T2i1sO78MkP56r0nYRw==
-  dependencies:
-    has "^1.0.3"
-
 is-data-descriptor@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56"
@@ -7291,6 +7305,11 @@ is-extglob@^2.1.0, is-extglob@^2.1.1:
   resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2"
   integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=
 
+is-finite@^1.0.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/is-finite/-/is-finite-1.1.0.tgz#904135c77fb42c0641d6aa1bcdbc4daa8da082f3"
+  integrity sha512-cdyMtqX/BOqqNBBiKlIVkytNHm49MtMlYyn1zxzvJKWmFMlGzm+ry5BBfYyeY9YmNKbRSo/o7OX9w9ale0wg3w==
+
 is-fullwidth-code-point@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb"
@@ -7383,7 +7402,7 @@ is-path-inside@^2.1.0:
   dependencies:
     path-is-inside "^1.0.2"
 
-is-plain-obj@^1.0.0, is-plain-obj@^1.1.0:
+is-plain-obj@^1.0.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz#71a50c8429dfca773c92a390a4a03b39fcd51d3e"
   integrity sha1-caUMhCnfync8kqOQpKA7OfzVHT4=
@@ -7464,6 +7483,11 @@ is-typedarray@^1.0.0, is-typedarray@~1.0.0:
   resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
   integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=
 
+is-utf8@^0.2.0:
+  version "0.2.1"
+  resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72"
+  integrity sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=
+
 is-windows@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d"
@@ -7933,6 +7957,13 @@ jest-snapshot@^26.6.0, jest-snapshot@^26.6.2:
     pretty-format "^26.6.2"
     semver "^7.3.2"
 
+jest-sonar-reporter@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/jest-sonar-reporter/-/jest-sonar-reporter-2.0.0.tgz#faa54a7d2af7198767ee246a82b78c576789cf08"
+  integrity sha512-ZervDCgEX5gdUbdtWsjdipLN3bKJwpxbvhkYNXTAYvAckCihobSLr9OT/IuyNIRT1EZMDDwR6DroWtrq+IL64w==
+  dependencies:
+    xml "^1.0.1"
+
 jest-util@^26.6.0, jest-util@^26.6.2:
   version "26.6.2"
   resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-26.6.2.tgz#907535dbe4d5a6cb4c47ac9b926f6af29576cbc1"
@@ -8261,7 +8292,7 @@ kind-of@^5.0.0:
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-5.1.0.tgz#729c91e2d857b7a419a1f9aa65685c4c33f5845d"
   integrity sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==
 
-kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3:
+kind-of@^6.0.0, kind-of@^6.0.2:
   version "6.0.3"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd"
   integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==
@@ -8322,6 +8353,17 @@ lines-and-columns@^1.1.6:
   resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00"
   integrity sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=
 
+load-json-file@^1.0.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-1.1.0.tgz#956905708d58b4bab4c2261b04f59f31c99374c0"
+  integrity sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=
+  dependencies:
+    graceful-fs "^4.1.2"
+    parse-json "^2.2.0"
+    pify "^2.0.0"
+    pinkie-promise "^2.0.0"
+    strip-bom "^2.0.0"
+
 load-json-file@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-2.0.0.tgz#7947e42149af80d696cbf797bcaabcfe1fe29ca8"
@@ -8484,6 +8526,14 @@ loose-envify@^1.1.0, loose-envify@^1.2.0, loose-envify@^1.3.1, loose-envify@^1.4
   dependencies:
     js-tokens "^3.0.0 || ^4.0.0"
 
+loud-rejection@^1.0.0:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/loud-rejection/-/loud-rejection-1.6.0.tgz#5b46f80147edee578870f086d04821cf998e551f"
+  integrity sha1-W0b4AUft7leIcPCG0Eghz5mOVR8=
+  dependencies:
+    currently-unhandled "^0.4.1"
+    signal-exit "^3.0.0"
+
 lower-case@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-2.0.1.tgz#39eeb36e396115cc05e29422eaea9e692c9408c7"
@@ -8491,6 +8541,14 @@ lower-case@^2.0.1:
   dependencies:
     tslib "^1.10.0"
 
+lru-cache@^4.0.1:
+  version "4.1.5"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd"
+  integrity sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==
+  dependencies:
+    pseudomap "^1.0.2"
+    yallist "^2.1.2"
+
 lru-cache@^5.1.1:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920"
@@ -8546,16 +8604,11 @@ map-cache@^0.2.2:
   resolved "https://registry.yarnpkg.com/map-cache/-/map-cache-0.2.2.tgz#c32abd0bd6525d9b051645bb4f26ac5dc98a0dbf"
   integrity sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8=
 
-map-obj@^1.0.0:
+map-obj@^1.0.0, map-obj@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d"
   integrity sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=
 
-map-obj@^4.0.0:
-  version "4.3.0"
-  resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-4.3.0.tgz#9304f906e93faae70880da102a9f1df0ea8bb05a"
-  integrity sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==
-
 map-visit@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/map-visit/-/map-visit-1.0.0.tgz#ecdca8f13144e660f1b5bd41f12f3479d98dfb8f"
@@ -8613,23 +8666,21 @@ memory-fs@^0.5.0:
     errno "^0.1.3"
     readable-stream "^2.0.1"
 
-meow@^9.0.0:
-  version "9.0.0"
-  resolved "https://registry.yarnpkg.com/meow/-/meow-9.0.0.tgz#cd9510bc5cac9dee7d03c73ee1f9ad959f4ea364"
-  integrity sha512-+obSblOQmRhcyBt62furQqRAQpNyWXo8BuQ5bN7dG8wmwQ+vwHKp/rCFD4CrTP8CsDQD1sjoZ94K417XEUk8IQ==
+meow@^3.7.0:
+  version "3.7.0"
+  resolved "https://registry.yarnpkg.com/meow/-/meow-3.7.0.tgz#72cb668b425228290abbfa856892587308a801fb"
+  integrity sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=
   dependencies:
-    "@types/minimist" "^1.2.0"
-    camelcase-keys "^6.2.2"
-    decamelize "^1.2.0"
-    decamelize-keys "^1.1.0"
-    hard-rejection "^2.1.0"
-    minimist-options "4.1.0"
-    normalize-package-data "^3.0.0"
-    read-pkg-up "^7.0.1"
-    redent "^3.0.0"
-    trim-newlines "^3.0.0"
-    type-fest "^0.18.0"
-    yargs-parser "^20.2.3"
+    camelcase-keys "^2.0.0"
+    decamelize "^1.1.2"
+    loud-rejection "^1.0.0"
+    map-obj "^1.0.1"
+    minimist "^1.1.3"
+    normalize-package-data "^2.3.4"
+    object-assign "^4.0.1"
+    read-pkg-up "^1.0.1"
+    redent "^1.0.0"
+    trim-newlines "^1.0.0"
 
 merge-descriptors@1.0.1:
   version "1.0.1"
@@ -8730,11 +8781,6 @@ mimic-fn@^2.1.0:
   resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b"
   integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==
 
-min-indent@^1.0.0:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/min-indent/-/min-indent-1.0.1.tgz#a63f681673b30571fbe8bc25686ae746eefa9869"
-  integrity sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==
-
 mini-create-react-context@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/mini-create-react-context/-/mini-create-react-context-0.4.0.tgz#df60501c83151db69e28eac0ef08b4002efab040"
@@ -8770,16 +8816,7 @@ minimatch@3.0.4, minimatch@^3.0.4, minimatch@~3.0.2:
   dependencies:
     brace-expansion "^1.1.7"
 
-minimist-options@4.1.0:
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619"
-  integrity sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==
-  dependencies:
-    arrify "^1.0.1"
-    is-plain-obj "^1.1.0"
-    kind-of "^6.0.3"
-
-minimist@^1.1.1, minimist@^1.2.0, minimist@^1.2.5:
+minimist@^1.1.1, minimist@^1.1.3, minimist@^1.2.0, minimist@^1.2.5:
   version "1.2.5"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
   integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
@@ -8844,6 +8881,13 @@ mixin-deep@^1.2.0:
     for-in "^1.0.2"
     is-extendable "^1.0.1"
 
+"mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.5:
+  version "0.5.5"
+  resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def"
+  integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==
+  dependencies:
+    minimist "^1.2.5"
+
 mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@~0.5.1:
   version "0.5.3"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.3.tgz#5a514b7179259287952881e94410ec5465659f8c"
@@ -8851,13 +8895,6 @@ mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@~0.5.1:
   dependencies:
     minimist "^1.2.5"
 
-mkdirp@^0.5.5:
-  version "0.5.5"
-  resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def"
-  integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==
-  dependencies:
-    minimist "^1.2.5"
-
 mkdirp@^1.0.3, mkdirp@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
@@ -9016,21 +9053,23 @@ node-forge@^0.10.0:
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3"
   integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==
 
-node-gyp@^7.1.0:
-  version "7.1.2"
-  resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-7.1.2.tgz#21a810aebb187120251c3bcec979af1587b188ae"
-  integrity sha512-CbpcIo7C3eMu3dL1c3d0xw449fHIGALIJsRP4DDPHpyiW8vcriNY7ubh9TE4zEKfSxscY7PjeFnshE7h75ynjQ==
+node-gyp@^3.8.0:
+  version "3.8.0"
+  resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-3.8.0.tgz#540304261c330e80d0d5edce253a68cb3964218c"
+  integrity sha512-3g8lYefrRRzvGeSowdJKAKyks8oUpLEd/DyPV4eMhVlhJ0aNaZqIrNUIPuEWWTAoPqyFkfGrM67MC69baqn6vA==
   dependencies:
-    env-paths "^2.2.0"
-    glob "^7.1.4"
-    graceful-fs "^4.2.3"
-    nopt "^5.0.0"
-    npmlog "^4.1.2"
-    request "^2.88.2"
-    rimraf "^3.0.2"
-    semver "^7.3.2"
-    tar "^6.0.2"
-    which "^2.0.2"
+    fstream "^1.0.0"
+    glob "^7.0.3"
+    graceful-fs "^4.1.2"
+    mkdirp "^0.5.0"
+    nopt "2 || 3"
+    npmlog "0 || 1 || 2 || 3 || 4"
+    osenv "0"
+    request "^2.87.0"
+    rimraf "2"
+    semver "~5.3.0"
+    tar "^2.0.0"
+    which "1"
 
 node-int64@^0.4.0:
   version "0.4.0"
@@ -9095,35 +9134,37 @@ node-releases@^1.1.61, node-releases@^1.1.70:
   resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.71.tgz#cb1334b179896b1c89ecfdd4b725fb7bbdfc7dbb"
   integrity sha512-zR6HoT6LrLCRBwukmrVbHv0EpEQjksO6GmFcZQQuCAy139BEsoVKPYnf3jongYW83fAa1torLGYwxxky/p28sg==
 
-node-sass@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/node-sass/-/node-sass-6.0.1.tgz#cad1ccd0ce63e35c7181f545d8b986f3a9a887fe"
-  integrity sha512-f+Rbqt92Ful9gX0cGtdYwjTrWAaGURgaK5rZCWOgCNyGWusFYHhbqCCBoFBeat+HKETOU02AyTxNhJV0YZf2jQ==
+node-sass@^4.14.1:
+  version "4.14.1"
+  resolved "https://registry.yarnpkg.com/node-sass/-/node-sass-4.14.1.tgz#99c87ec2efb7047ed638fb4c9db7f3a42e2217b5"
+  integrity sha512-sjCuOlvGyCJS40R8BscF5vhVlQjNN069NtQ1gSxyK1u9iqvn6tf7O1R4GNowVZfiZUCRt5MmMs1xd+4V/7Yr0g==
   dependencies:
     async-foreach "^0.1.3"
     chalk "^1.1.1"
-    cross-spawn "^7.0.3"
+    cross-spawn "^3.0.0"
     gaze "^1.0.0"
     get-stdin "^4.0.1"
     glob "^7.0.3"
+    in-publish "^2.0.0"
     lodash "^4.17.15"
-    meow "^9.0.0"
+    meow "^3.7.0"
+    mkdirp "^0.5.1"
     nan "^2.13.2"
-    node-gyp "^7.1.0"
+    node-gyp "^3.8.0"
     npmlog "^4.0.0"
     request "^2.88.0"
     sass-graph "2.2.5"
     stdout-stream "^1.4.0"
     "true-case-path" "^1.0.2"
 
-nopt@^5.0.0:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/nopt/-/nopt-5.0.0.tgz#530942bb58a512fccafe53fe210f13a25355dc88"
-  integrity sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==
+"nopt@2 || 3":
+  version "3.0.6"
+  resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9"
+  integrity sha1-xkZdvwirzU2zWTF/eaxopkayj/k=
   dependencies:
     abbrev "1"
 
-normalize-package-data@^2.3.2, normalize-package-data@^2.5.0:
+normalize-package-data@^2.3.2, normalize-package-data@^2.3.4, normalize-package-data@^2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8"
   integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==
@@ -9133,16 +9174,6 @@ normalize-package-data@^2.3.2, normalize-package-data@^2.5.0:
     semver "2 || 3 || 4 || 5"
     validate-npm-package-license "^3.0.1"
 
-normalize-package-data@^3.0.0:
-  version "3.0.3"
-  resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz#dbcc3e2da59509a0983422884cd172eefdfa525e"
-  integrity sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==
-  dependencies:
-    hosted-git-info "^4.0.1"
-    is-core-module "^2.5.0"
-    semver "^7.3.4"
-    validate-npm-package-license "^3.0.1"
-
 normalize-path@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-2.1.1.tgz#1ab28b556e198363a8c1a6f7e6fa20137fe6aed9"
@@ -9189,7 +9220,7 @@ npm-run-path@^4.0.0:
   dependencies:
     path-key "^3.0.0"
 
-npmlog@^4.0.0, npmlog@^4.1.2:
+"npmlog@0 || 1 || 2 || 3 || 4", npmlog@^4.0.0:
   version "4.1.2"
   resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b"
   integrity sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==
@@ -9432,6 +9463,24 @@ os-browserify@^0.3.0:
   resolved "https://registry.yarnpkg.com/os-browserify/-/os-browserify-0.3.0.tgz#854373c7f5c2315914fc9bfc6bd8238fdda1ec27"
   integrity sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=
 
+os-homedir@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3"
+  integrity sha1-/7xJiDNuDoM94MFox+8VISGqf7M=
+
+os-tmpdir@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
+  integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=
+
+osenv@0:
+  version "0.1.5"
+  resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410"
+  integrity sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==
+  dependencies:
+    os-homedir "^1.0.0"
+    os-tmpdir "^1.0.0"
+
 p-each-series@^2.1.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/p-each-series/-/p-each-series-2.2.0.tgz#105ab0357ce72b202a8a8b94933672657b5e2a9a"
@@ -9612,6 +9661,13 @@ path-dirname@^1.0.0:
   resolved "https://registry.yarnpkg.com/path-dirname/-/path-dirname-1.0.2.tgz#cc33d24d525e099a5388c0336c6e32b9160609e0"
   integrity sha1-zDPSTVJeCZpTiMAzbG4yuRYGCeA=
 
+path-exists@^2.0.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-2.1.0.tgz#0feb6c64f0fc518d9a754dd5efb62c7022761f4b"
+  integrity sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=
+  dependencies:
+    pinkie-promise "^2.0.0"
+
 path-exists@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515"
@@ -9659,6 +9715,15 @@ path-to-regexp@^1.7.0:
   dependencies:
     isarray "0.0.1"
 
+path-type@^1.0.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441"
+  integrity sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE=
+  dependencies:
+    graceful-fs "^4.1.2"
+    pify "^2.0.0"
+    pinkie-promise "^2.0.0"
+
 path-type@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/path-type/-/path-type-2.0.0.tgz#f012ccb8415b7096fc2daa1054c3d72389594c73"
@@ -10599,6 +10664,11 @@ prr@~1.0.1:
   resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476"
   integrity sha1-0/wRS6BplaRexok/SEzrHXj19HY=
 
+pseudomap@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
+  integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM=
+
 psl@^1.1.28:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.7.0.tgz#f1c4c47a8ef97167dea5d6bbf4816d736e884a3c"
@@ -10704,11 +10774,6 @@ queue-microtask@^1.2.2:
   resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.2.tgz#abf64491e6ecf0f38a6502403d4cda04f372dfd3"
   integrity sha512-dB15eXv3p2jDlbOiNLyMabYg1/sXvppd8DP2J3EOCQ0AkuSXCW2tP7mnVouVLJKgUMY6yP0kcQDVpLCN13h4Xg==
 
-quick-lru@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f"
-  integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==
-
 raf-schd@^4.0.2:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/raf-schd/-/raf-schd-4.0.2.tgz#bd44c708188f2e84c810bf55fcea9231bcaed8a0"
@@ -11042,6 +11107,14 @@ react@^17.0.1:
     loose-envify "^1.1.0"
     object-assign "^4.1.1"
 
+read-pkg-up@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-1.0.1.tgz#9d63c13276c065918d57f002a57f40a1b643fb02"
+  integrity sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI=
+  dependencies:
+    find-up "^1.0.0"
+    read-pkg "^1.0.0"
+
 read-pkg-up@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-2.0.0.tgz#6b72a8048984e0c41e79510fd5e9fa99b3b549be"
@@ -11059,6 +11132,15 @@ read-pkg-up@^7.0.1:
     read-pkg "^5.2.0"
     type-fest "^0.8.1"
 
+read-pkg@^1.0.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-1.1.0.tgz#f5ffaa5ecd29cb31c0474bca7d756b6bb29e3f28"
+  integrity sha1-9f+qXs0pyzHAR0vKfXVra7KePyg=
+  dependencies:
+    load-json-file "^1.0.0"
+    normalize-package-data "^2.3.2"
+    path-type "^1.0.0"
+
 read-pkg@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-2.0.0.tgz#8ef1c0623c6a6db0dc6713c4bfac46332b2368f8"
@@ -11123,13 +11205,13 @@ recursive-readdir@2.2.2:
   dependencies:
     minimatch "3.0.4"
 
-redent@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/redent/-/redent-3.0.0.tgz#e557b7998316bb53c9f1f56fa626352c6963059f"
-  integrity sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==
+redent@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/redent/-/redent-1.0.0.tgz#cf916ab1fd5f1f16dfb20822dd6ec7f730c2afde"
+  integrity sha1-z5Fqsf1fHxbfsggi3W7H9zDCr94=
   dependencies:
-    indent-string "^4.0.0"
-    strip-indent "^3.0.0"
+    indent-string "^2.1.0"
+    strip-indent "^1.0.1"
 
 redux@^4.0.4:
   version "4.0.5"
@@ -11272,6 +11354,13 @@ repeat-string@^1.6.1:
   resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637"
   integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc=
 
+repeating@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/repeating/-/repeating-2.0.1.tgz#5214c53a926d3552707527fbab415dbc08d06dda"
+  integrity sha1-UhTFOpJtNVJwdSf7q0FdvAjQbdo=
+  dependencies:
+    is-finite "^1.0.0"
+
 request-promise-core@1.1.4:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.4.tgz#3eedd4223208d419867b78ce815167d10593a22f"
@@ -11288,7 +11377,7 @@ request-promise-native@^1.0.8:
     stealthy-require "^1.1.1"
     tough-cookie "^2.3.3"
 
-request@^2.88.0, request@^2.88.2:
+request@^2.87.0, request@^2.88.0, request@^2.88.2:
   version "2.88.2"
   resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3"
   integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==
@@ -11462,7 +11551,7 @@ rifm@^0.7.0:
   dependencies:
     "@babel/runtime" "^7.3.1"
 
-rimraf@^2.5.4, rimraf@^2.6.3:
+rimraf@2, rimraf@^2.5.4, rimraf@^2.6.3:
   version "2.7.1"
   resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec"
   integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==
@@ -11715,12 +11804,10 @@ semver@^7.2.1, semver@^7.3.2:
   dependencies:
     lru-cache "^6.0.0"
 
-semver@^7.3.4:
-  version "7.3.5"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7"
-  integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==
-  dependencies:
-    lru-cache "^6.0.0"
+semver@~5.3.0:
+  version "5.3.0"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
+  integrity sha1-myzl094C0XxgEq0yaqa00M9U+U8=
 
 send@0.17.1:
   version "0.17.1"
@@ -12381,6 +12468,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
   dependencies:
     ansi-regex "^4.1.0"
 
+strip-bom@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e"
+  integrity sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=
+  dependencies:
+    is-utf8 "^0.2.0"
+
 strip-bom@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3"
@@ -12409,12 +12503,12 @@ strip-final-newline@^2.0.0:
   resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad"
   integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==
 
-strip-indent@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/strip-indent/-/strip-indent-3.0.0.tgz#c32e1cee940b6b3432c771bc2c54bcce73cd3001"
-  integrity sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==
+strip-indent@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/strip-indent/-/strip-indent-1.0.1.tgz#0c7962a6adefa7bbd4ac366460a638552ae1a0a2"
+  integrity sha1-DHlipq3vp7vUrDZkYKY4VSrhoKI=
   dependencies:
-    min-indent "^1.0.0"
+    get-stdin "^4.0.1"
 
 strip-json-comments@^3.1.0, strip-json-comments@^3.1.1:
   version "3.1.1"
@@ -12534,6 +12628,15 @@ tapable@^1.0.0, tapable@^1.1.3:
   resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2"
   integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==
 
+tar@^2.0.0:
+  version "2.2.2"
+  resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.2.tgz#0ca8848562c7299b8b446ff6a4d60cdbb23edc40"
+  integrity sha512-FCEhQ/4rE1zYv9rYXJw/msRqsnmlje5jHP6huWeBZ704jUTy02c5AZyWujpMR1ax6mVw9NyJMfuK2CMDWVIfgA==
+  dependencies:
+    block-stream "*"
+    fstream "^1.0.12"
+    inherits "2"
+
 tar@^6.0.2:
   version "6.1.0"
   resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.0.tgz#d1724e9bcc04b977b18d5c573b333a2207229a83"
@@ -12765,10 +12868,10 @@ tr46@^2.0.2:
   dependencies:
     punycode "^2.1.1"
 
-trim-newlines@^3.0.0:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144"
-  integrity sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw==
+trim-newlines@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-1.0.0.tgz#5887966bb582a4503a41eb524f7d35011815a613"
+  integrity sha1-WIeWa7WCpFA6QetST301ARgVphM=
 
 "true-case-path@^1.0.2":
   version "1.0.3"
@@ -12891,11 +12994,6 @@ type-fest@^0.11.0:
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.11.0.tgz#97abf0872310fed88a5c466b25681576145e33f1"
   integrity sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ==
 
-type-fest@^0.18.0:
-  version "0.18.1"
-  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.18.1.tgz#db4bc151a4a2cf4eebf9add5db75508db6cc841f"
-  integrity sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw==
-
 type-fest@^0.3.1:
   version "0.3.1"
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.3.1.tgz#63d00d204e059474fe5e1b7c011112bbd1dc29e1"
@@ -13421,7 +13519,7 @@ which-module@^2.0.0:
   resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a"
   integrity sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=
 
-which@^1.2.9, which@^1.3.1:
+which@1, which@^1.2.9, which@^1.3.1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
   integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==
@@ -13664,6 +13762,11 @@ xml-name-validator@^3.0.0:
   resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
   integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==
 
+xml@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
+  integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=
+
 xmlchars@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"
@@ -13679,6 +13782,11 @@ y18n@^4.0.0:
   resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b"
   integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==
 
+yallist@^2.1.2:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
+  integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=
+
 yallist@^3.0.2:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd"
@@ -13717,11 +13825,6 @@ yargs-parser@^18.1.2:
     camelcase "^5.0.0"
     decamelize "^1.2.0"
 
-yargs-parser@^20.2.3:
-  version "20.2.9"
-  resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee"
-  integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==
-
 yargs@^13.3.2:
   version "13.3.2"
   resolved "https://registry.yarnpkg.com/yargs/-/yargs-13.3.2.tgz#ad7ffefec1aa59565ac915f82dccb38a9c31a2dd"

From 4a1897327498c7e532b8420d604eeb4ab005e13f Mon Sep 17 00:00:00 2001
From: reesporte 
Date: Fri, 29 Oct 2021 16:16:59 -0500
Subject: [PATCH 34/40] use prettier to format changed files

---
 lattice/src/shared/DataTable/DataTable.tsx    | 50 +++++++++----------
 .../src/shared/utils/formatTableCell.test.tsx | 17 +++----
 lattice/src/shared/utils/formatTableCell.tsx  | 42 +++++++---------
 3 files changed, 50 insertions(+), 59 deletions(-)

diff --git a/lattice/src/shared/DataTable/DataTable.tsx b/lattice/src/shared/DataTable/DataTable.tsx
index 94510f86d..a5246cd49 100644
--- a/lattice/src/shared/DataTable/DataTable.tsx
+++ b/lattice/src/shared/DataTable/DataTable.tsx
@@ -1,18 +1,18 @@
-import React, { FC, Fragment, useEffect, useRef, useState } from 'react';
-import ArrowDropDownIcon from '@material-ui/icons/ArrowDropDown';
-import classNames from 'classnames';
-import OrderBy from 'lodash/orderBy';
-import Table from '@material-ui/core/Table';
-import TableBody from '@material-ui/core/TableBody';
-import TableCell from '@material-ui/core/TableCell';
-import TableHead from '@material-ui/core/TableHead';
+import React, { FC, Fragment, useEffect, useRef, useState } from "react";
+import ArrowDropDownIcon from "@material-ui/icons/ArrowDropDown";
+import classNames from "classnames";
+import OrderBy from "lodash/orderBy";
+import Table from "@material-ui/core/Table";
+import TableBody from "@material-ui/core/TableBody";
+import TableCell from "@material-ui/core/TableCell";
+import TableHead from "@material-ui/core/TableHead";
 // import TablePagination from '@material-ui/core/TablePagination';
-import TableRow from '@material-ui/core/TableRow';
-import Typography from '@material-ui/core/Typography';
-import { ColumnInfo } from 'proto/pilosa_pb';
-import { Pager } from 'shared/Pager';
-import { formatTableCell } from 'shared/utils/formatTableCell';
-import css from './DataTable.module.scss';
+import TableRow from "@material-ui/core/TableRow";
+import Typography from "@material-ui/core/Typography";
+import { ColumnInfo } from "proto/pilosa_pb";
+import { Pager } from "shared/Pager";
+import { formatTableCell } from "shared/utils/formatTableCell";
+import css from "./DataTable.module.scss";
 
 type TableProps = {
   headers: ColumnInfo.AsObject[];
@@ -25,11 +25,11 @@ export const DataTable: FC = ({
   headers,
   data,
   loading = false,
-  autoWidth = false
+  autoWidth = false,
 }) => {
   const [sortedData, setSortedData] = useState(data);
   const [sort, setSort] = useState(headers[0]?.name);
-  const [sortDir, setSortDir] = useState<'asc' | 'desc'>('asc');
+  const [sortDir, setSortDir] = useState<"asc" | "desc">("asc");
   const [page, setPage] = useState(1);
   const [rowsPerPage, setRowsPerPage] = useState(10);
   const resultsRef = useRef(null);
@@ -54,8 +54,8 @@ export const DataTable: FC = ({
     setTimeout(() => {
       if (resultsRef.current) {
         resultsRef.current.scrollIntoView({
-          behavior: 'smooth',
-          block: 'start'
+          behavior: "smooth",
+          block: "start",
         });
       }
     }, 0);
@@ -63,15 +63,13 @@ export const DataTable: FC = ({
 
   const onSortClick = (name: string) => {
     if (sort === name) {
-      setSortDir(sortDir === 'desc' ? 'asc' : 'desc');
+      setSortDir(sortDir === "desc" ? "asc" : "desc");
     } else {
       setSort(name);
-      setSortDir('asc');
+      setSortDir("asc");
     }
   };
 
-
-
   return (
     
       

@@ -86,14 +84,14 @@ export const DataTable: FC = ({
                 >
                    onSortClick(col.name)}
                   >
                     {col.name}
                     
                   
@@ -112,7 +110,7 @@ export const DataTable: FC = ({
                 .map((row, rowIdx) => (
                   
                     {headers.map((col, colIdx) => (
                        = ({
                         className={css.tableCell}
                       >
                         {formatTableCell(row, col)}
-                            
+                      
                     ))}
                     {autoWidth ?  : null}
                   
diff --git a/lattice/src/shared/utils/formatTableCell.test.tsx b/lattice/src/shared/utils/formatTableCell.test.tsx
index cbbc26ca3..2578a34ae 100644
--- a/lattice/src/shared/utils/formatTableCell.test.tsx
+++ b/lattice/src/shared/utils/formatTableCell.test.tsx
@@ -1,9 +1,8 @@
-import { formatTableCell } from './formatTableCell';
+import { formatTableCell } from "./formatTableCell";
 import React from "react";
 import { render, unmountComponentAtNode } from "react-dom";
 import { act } from "react-dom/test-utils";
 
-
 let container = null;
 beforeEach(() => {
   // setup a DOM element as a render target
@@ -19,8 +18,8 @@ afterEach(() => {
 });
 
 it("it renders strings in quotes", () => {
-  let row = {thing:"quoted string!"};
-  let col = {name: "thing", datatype: "[]string"};
+  let row = { thing: "quoted string!" };
+  let col = { name: "thing", datatype: "[]string" };
 
   act(() => {
     render(formatTableCell(row, col), container);
@@ -29,8 +28,8 @@ it("it renders strings in quotes", () => {
 });
 
 it("renders objects as stringified", () => {
-  let row = {thing:{val:"quoted string!"}};
-  let col = {name: "thing", datatype: "object"};
+  let row = { thing: { val: "quoted string!" } };
+  let col = { name: "thing", datatype: "object" };
 
   act(() => {
     render(formatTableCell(row, col), container);
@@ -41,13 +40,11 @@ it("renders objects as stringified", () => {
 });
 
 it("it puts timestamps in MM/DD/YYYY hh:mm:ss a format", () => {
-  let row = {thing:1635452050094};
-  let col = {name: "thing", datatype: "timestamp"};
+  let row = { thing: 1635452050094 };
+  let col = { name: "thing", datatype: "timestamp" };
 
   act(() => {
     render(formatTableCell(row, col), container);
   });
   expect(container.textContent).toBe("10/28/2021 08:14:10 pm");
 });
-
-
diff --git a/lattice/src/shared/utils/formatTableCell.tsx b/lattice/src/shared/utils/formatTableCell.tsx
index 9dd428ef8..3c6de098b 100644
--- a/lattice/src/shared/utils/formatTableCell.tsx
+++ b/lattice/src/shared/utils/formatTableCell.tsx
@@ -1,27 +1,23 @@
-import moment from 'moment';
-import css from '../DataTable/DataTable.module.scss';
+import moment from "moment";
+import css from "../DataTable/DataTable.module.scss";
 export const formatTableCell = (row: any, col: any) => {
-     if (typeof row[col.name] === 'object') {
-        return (
+  if (typeof row[col.name] === "object") {
+    return (
       
         {JSON.stringify(row[col.name], null, 2)}
-      
 )
-     } else if (row[col.name] !== undefined) {
-        if (col.datatype === '[]string') {
-            return (
-                 
-                    {'"' + (row[col.name]) + '"'}
-                 
-            )
-        }
-        return (
-      
-        {col.datatype === 'timestamp' && row[col.name]
-          ? moment
-              .utc(row[col.name])
-              .format('MM/DD/YYYY hh:mm:ss a')
-          : row[col.name].toLocaleString()}
-       )
+
+ ); + } else if (row[col.name] !== undefined) { + if (col.datatype === "[]string") { + return {'"' + row[col.name] + '"'}; } - return null -} + return ( + + {col.datatype === "timestamp" && row[col.name] + ? moment.utc(row[col.name]).format("MM/DD/YYYY hh:mm:ss a") + : row[col.name].toLocaleString()} + + ); + } + return null; +}; From 21969b163708231d15fdbe7820d6f792f0d4fb56 Mon Sep 17 00:00:00 2001 From: Samir Patel <48686912+54mir@users.noreply.github.com> Date: Fri, 29 Oct 2021 16:52:09 -0500 Subject: [PATCH 35/40] Add unit test --- api.go | 11 +++++++++++ server/handler_test.go | 26 +++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/api.go b/api.go index f7d9001f4..3d65dba70 100644 --- a/api.go +++ b/api.go @@ -127,6 +127,17 @@ func NewAPI(opts ...apiOption) (*API, error) { return api, nil } +// Setter for API options. +func (api *API) SetAPIOpetions(opts ...apiOption) error { + for _, opt := range opts { + err := opt(api) + if err != nil { + return errors.Wrap(err, "applying option") + } + } + return nil +} + // validAPIMethods specifies the api methods that are valid for each // cluster state. var validAPIMethods = map[disco.ClusterState]map[apiMethod]struct{}{ diff --git a/server/handler_test.go b/server/handler_test.go index ea6c74e4f..e7212892d 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -32,7 +32,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/boltdb" "github.com/molecula/featurebase/v2/encoding/proto" "github.com/molecula/featurebase/v2/http" @@ -344,6 +344,30 @@ func TestHandler_Endpoints(t *testing.T) { } }) + t.Run("SchemaDetailsOff", func(t *testing.T) { + cmd.API.SetAPIOpetions(pilosa.OptAPISchemaDetailsOn(false)) + defer cmd.API.SetAPIOpetions(pilosa.OptAPISchemaDetailsOn(true)) + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + var bodySchema pilosa.Schema + if err := json.Unmarshal(w.Body.Bytes(), + &bodySchema); err != nil { + t.Fatalf("unexpected unmarshalling error: %v", err) + } + + for _, i := range bodySchema.Indexes { + for _, f := range i.Fields { + if f.Cardinality != nil { + t.Fatalf("expected nil cardinality, got: %v", *f.Cardinality) + } + } + } + }) + t.Run("Import", func(t *testing.T) { indexInfo, err := cmd.API.Schema(context.Background(), false) if err != nil { From d828d73eaef91e4060f5a9a1dcb6bb8f22bd7fdf Mon Sep 17 00:00:00 2001 From: Samir Patel <48686912+54mir@users.noreply.github.com> Date: Fri, 29 Oct 2021 19:04:21 -0500 Subject: [PATCH 36/40] Add APISetOptions test for coverage --- api.go | 7 ++----- api_test.go | 9 ++++++++- server/handler_test.go | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/api.go b/api.go index 3d65dba70..3d4961626 100644 --- a/api.go +++ b/api.go @@ -128,12 +128,9 @@ func NewAPI(opts ...apiOption) (*API, error) { } // Setter for API options. -func (api *API) SetAPIOpetions(opts ...apiOption) error { +func (api *API) SetAPIOptions(opts ...apiOption) error { for _, opt := range opts { - err := opt(api) - if err != nil { - return errors.Wrap(err, "applying option") - } + opt(api) } return nil } diff --git a/api_test.go b/api_test.go index 50a59a03e..9aafae717 100644 --- a/api_test.go +++ b/api_test.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/boltdb" "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/server" @@ -846,6 +846,13 @@ func TestAPI_IDAlloc(t *testing.T) { }) } +func TestAPI_SetAPIOptions(t *testing.T) { + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster.GetNode(0) + cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false)) +} + type mutexCheckIndex struct { index *pilosa.Index indexName string diff --git a/server/handler_test.go b/server/handler_test.go index e7212892d..e4c28bece 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -345,8 +345,8 @@ func TestHandler_Endpoints(t *testing.T) { }) t.Run("SchemaDetailsOff", func(t *testing.T) { - cmd.API.SetAPIOpetions(pilosa.OptAPISchemaDetailsOn(false)) - defer cmd.API.SetAPIOpetions(pilosa.OptAPISchemaDetailsOn(true)) + cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false)) + defer cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(true)) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil)) if w.Code != gohttp.StatusOK { From 163492801f8287cd8e6e6da867cc9962a8ae3062 Mon Sep 17 00:00:00 2001 From: Samir Patel <48686912+54mir@users.noreply.github.com> Date: Sun, 31 Oct 2021 23:18:53 -0500 Subject: [PATCH 37/40] Add test to test endpoint code directly --- api_test.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/api_test.go b/api_test.go index 9aafae717..c06c9f8a2 100644 --- a/api_test.go +++ b/api_test.go @@ -846,11 +846,24 @@ func TestAPI_IDAlloc(t *testing.T) { }) } -func TestAPI_SetAPIOptions(t *testing.T) { - cluster := test.MustRunCluster(t, 1) +func TestAPI_SchemaDetailsOff(t *testing.T) { + cluster := test.MustRunCluster(t, 2) defer cluster.Close() cmd := cluster.GetNode(0) cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false)) + schema, err := cmd.API.SchemaDetails(context.Background()) + if err != nil { + t.Fatalf("getting schema: %v", err) + } + + for _, i := range schema { + for _, f := range i.Fields { + if f.Cardinality != nil { + t.Fatalf("expected nil cardinality, got: %v", *f.Cardinality) + } + } + } + } type mutexCheckIndex struct { From faa831928c997de01b9b7f56e9a4200a0c93a28b Mon Sep 17 00:00:00 2001 From: Samir Patel <48686912+54mir@users.noreply.github.com> Date: Sun, 31 Oct 2021 23:48:43 -0500 Subject: [PATCH 38/40] Add some error handling --- api.go | 5 ++++- api_test.go | 5 ++++- server/handler_test.go | 14 +++++++++++--- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/api.go b/api.go index 3d4961626..458164f6e 100644 --- a/api.go +++ b/api.go @@ -130,7 +130,10 @@ func NewAPI(opts ...apiOption) (*API, error) { // Setter for API options. func (api *API) SetAPIOptions(opts ...apiOption) error { for _, opt := range opts { - opt(api) + err := opt(api) + if err != nil { + return errors.Wrap(err, "setting API option") + } } return nil } diff --git a/api_test.go b/api_test.go index c06c9f8a2..378aee9fd 100644 --- a/api_test.go +++ b/api_test.go @@ -850,7 +850,10 @@ func TestAPI_SchemaDetailsOff(t *testing.T) { cluster := test.MustRunCluster(t, 2) defer cluster.Close() cmd := cluster.GetNode(0) - cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false)) + err := cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false)) + if err != nil { + t.Fatalf("could not toggle schema details to off: %v", err) + } schema, err := cmd.API.SchemaDetails(context.Background()) if err != nil { t.Fatalf("getting schema: %v", err) diff --git a/server/handler_test.go b/server/handler_test.go index e4c28bece..96b0f12aa 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -345,8 +345,11 @@ func TestHandler_Endpoints(t *testing.T) { }) t.Run("SchemaDetailsOff", func(t *testing.T) { - cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false)) - defer cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(true)) + err := cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false)) + if err != nil { + t.Fatalf("setting schema details option") + } + w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil)) if w.Code != gohttp.StatusOK { @@ -357,8 +360,8 @@ func TestHandler_Endpoints(t *testing.T) { if err := json.Unmarshal(w.Body.Bytes(), &bodySchema); err != nil { t.Fatalf("unexpected unmarshalling error: %v", err) - } + } for _, i := range bodySchema.Indexes { for _, f := range i.Fields { if f.Cardinality != nil { @@ -366,6 +369,11 @@ func TestHandler_Endpoints(t *testing.T) { } } } + + err = cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(true)) + if err != nil { + t.Fatalf("could not toggle schema details to on: %v", err) + } }) t.Run("Import", func(t *testing.T) { From 705efc1b8a26b7fdfd0821c431e5e0880b0d224b Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 1 Nov 2021 09:25:07 -0500 Subject: [PATCH 39/40] refactor for easier reading, add more tests --- .../src/shared/utils/formatTableCell.test.tsx | 31 +++++++++++++++++-- lattice/src/shared/utils/formatTableCell.tsx | 13 ++++---- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/lattice/src/shared/utils/formatTableCell.test.tsx b/lattice/src/shared/utils/formatTableCell.test.tsx index 2578a34ae..5a8a0e211 100644 --- a/lattice/src/shared/utils/formatTableCell.test.tsx +++ b/lattice/src/shared/utils/formatTableCell.test.tsx @@ -17,7 +17,7 @@ afterEach(() => { container = null; }); -it("it renders strings in quotes", () => { +it("renders strings in quotes", () => { let row = { thing: "quoted string!" }; let col = { name: "thing", datatype: "[]string" }; @@ -39,7 +39,7 @@ it("renders objects as stringified", () => { }`); }); -it("it puts timestamps in MM/DD/YYYY hh:mm:ss a format", () => { +it("puts timestamps in MM/DD/YYYY hh:mm:ss a format", () => { let row = { thing: 1635452050094 }; let col = { name: "thing", datatype: "timestamp" }; @@ -48,3 +48,30 @@ it("it puts timestamps in MM/DD/YYYY hh:mm:ss a format", () => { }); expect(container.textContent).toBe("10/28/2021 08:14:10 pm"); }); + +it("renders timestamps with no value as LocaleString", () => { + let row = { thing: false }; + let col = { name: "thing", datatype: "timestamp" }; + + act(() => { + render(formatTableCell(row, col), container); + }); + expect(container.textContent).toBe(row.thing.toLocaleString()); +}); + +it("renders non-timestamp, non-string, non-objects as LocaleString", () => { + let row = { thing: "idk" }; + let col = { name: "thing", datatype: "idk" }; + + act(() => { + render(formatTableCell(row, col), container); + }); + expect(container.textContent).toBe(row.thing.toLocaleString()); +}); + +it("returns null for undefined objects", () => { + let row = {}; + let col = { name: "thing", datatype: "" }; + + expect(formatTableCell(row, col)).toBeNull(); +}); diff --git a/lattice/src/shared/utils/formatTableCell.tsx b/lattice/src/shared/utils/formatTableCell.tsx index 3c6de098b..b21c10713 100644 --- a/lattice/src/shared/utils/formatTableCell.tsx +++ b/lattice/src/shared/utils/formatTableCell.tsx @@ -11,13 +11,12 @@ export const formatTableCell = (row: any, col: any) => { if (col.datatype === "[]string") { return {'"' + row[col.name] + '"'}; } - return ( - - {col.datatype === "timestamp" && row[col.name] - ? moment.utc(row[col.name]).format("MM/DD/YYYY hh:mm:ss a") - : row[col.name].toLocaleString()} - - ); + if (col.datatype === "timestamp" && row[col.name]) { + return ( + {moment.utc(row[col.name]).format("MM/DD/YYYY hh:mm:ss a")} + ); + } + return {row[col.name].toLocaleString()}; } return null; }; From 510dd4dfb7859cd8fffb9d037a7e61c67ef3f3f4 Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 1 Nov 2021 09:36:36 -0500 Subject: [PATCH 40/40] un-prettify for less lines of code changed :) --- lattice/src/shared/DataTable/DataTable.tsx | 50 +++++++++++----------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/lattice/src/shared/DataTable/DataTable.tsx b/lattice/src/shared/DataTable/DataTable.tsx index a5246cd49..94510f86d 100644 --- a/lattice/src/shared/DataTable/DataTable.tsx +++ b/lattice/src/shared/DataTable/DataTable.tsx @@ -1,18 +1,18 @@ -import React, { FC, Fragment, useEffect, useRef, useState } from "react"; -import ArrowDropDownIcon from "@material-ui/icons/ArrowDropDown"; -import classNames from "classnames"; -import OrderBy from "lodash/orderBy"; -import Table from "@material-ui/core/Table"; -import TableBody from "@material-ui/core/TableBody"; -import TableCell from "@material-ui/core/TableCell"; -import TableHead from "@material-ui/core/TableHead"; +import React, { FC, Fragment, useEffect, useRef, useState } from 'react'; +import ArrowDropDownIcon from '@material-ui/icons/ArrowDropDown'; +import classNames from 'classnames'; +import OrderBy from 'lodash/orderBy'; +import Table from '@material-ui/core/Table'; +import TableBody from '@material-ui/core/TableBody'; +import TableCell from '@material-ui/core/TableCell'; +import TableHead from '@material-ui/core/TableHead'; // import TablePagination from '@material-ui/core/TablePagination'; -import TableRow from "@material-ui/core/TableRow"; -import Typography from "@material-ui/core/Typography"; -import { ColumnInfo } from "proto/pilosa_pb"; -import { Pager } from "shared/Pager"; -import { formatTableCell } from "shared/utils/formatTableCell"; -import css from "./DataTable.module.scss"; +import TableRow from '@material-ui/core/TableRow'; +import Typography from '@material-ui/core/Typography'; +import { ColumnInfo } from 'proto/pilosa_pb'; +import { Pager } from 'shared/Pager'; +import { formatTableCell } from 'shared/utils/formatTableCell'; +import css from './DataTable.module.scss'; type TableProps = { headers: ColumnInfo.AsObject[]; @@ -25,11 +25,11 @@ export const DataTable: FC = ({ headers, data, loading = false, - autoWidth = false, + autoWidth = false }) => { const [sortedData, setSortedData] = useState(data); const [sort, setSort] = useState(headers[0]?.name); - const [sortDir, setSortDir] = useState<"asc" | "desc">("asc"); + const [sortDir, setSortDir] = useState<'asc' | 'desc'>('asc'); const [page, setPage] = useState(1); const [rowsPerPage, setRowsPerPage] = useState(10); const resultsRef = useRef(null); @@ -54,8 +54,8 @@ export const DataTable: FC = ({ setTimeout(() => { if (resultsRef.current) { resultsRef.current.scrollIntoView({ - behavior: "smooth", - block: "start", + behavior: 'smooth', + block: 'start' }); } }, 0); @@ -63,13 +63,15 @@ export const DataTable: FC = ({ const onSortClick = (name: string) => { if (sort === name) { - setSortDir(sortDir === "desc" ? "asc" : "desc"); + setSortDir(sortDir === 'desc' ? 'asc' : 'desc'); } else { setSort(name); - setSortDir("asc"); + setSortDir('asc'); } }; + + return (
@@ -84,14 +86,14 @@ export const DataTable: FC = ({ > onSortClick(col.name)} > {col.name} @@ -110,7 +112,7 @@ export const DataTable: FC = ({ .map((row, rowIdx) => ( {headers.map((col, colIdx) => ( = ({ className={css.tableCell} > {formatTableCell(row, col)} - + ))} {autoWidth ? : null}