From d5b61ee8e854aacffb52e89ef6111f57fe09e9a2 Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 30 Sep 2021 12:28:33 -0500 Subject: [PATCH 1/5] reduce etcd fsyncs during testing We disable fsync more consistently in testing, including using etcd's already-existing UnsafeNoFsync option to disable fsyncs in the backing store boltdb used by etcd, to reduce runtime of our tests on MacOS significantly. Corresponding to this, we update etcd by one patch to pick up a locally-invented patch which turns out to be nearly-identical to the upstream fix for "disabling fsync makes boltdb not even bother to write some data sometimes", which caused crashes galore. --- etcd/embed.go | 2 ++ go.mod | 55 +++++++++++++++++++++++++++++++++++++++---------- go.sum | 2 ++ idalloc_test.go | 2 +- test/cluster.go | 2 +- test/disco.go | 19 ++++++++++------- 6 files changed, 61 insertions(+), 21 deletions(-) diff --git a/etcd/embed.go b/etcd/embed.go index 67711d4c1..7da2c9806 100644 --- a/etcd/embed.go +++ b/etcd/embed.go @@ -63,6 +63,7 @@ type Options struct { LClientSocket []*net.TCPListener BootstrapTimeout time.Duration + UnsafeNoFsync bool `toml:"no-fsync"` } var ( @@ -228,6 +229,7 @@ func parseOptions(opt Options) *embed.Config { cfg.InitialClusterToken = opt.ClusterName cfg.BootstrapTimeout = opt.BootstrapTimeout cfg.LCUrls = types.MustNewURLs([]string{opt.LClientURL}) + cfg.UnsafeNoFsync = opt.UnsafeNoFsync if opt.AClientURL != "" { cfg.ACUrls = types.MustNewURLs([]string{opt.AClientURL}) } else { diff --git a/go.mod b/go.mod index e590bdd1b..23401e34f 100644 --- a/go.mod +++ b/go.mod @@ -1,63 +1,96 @@ module github.com/molecula/featurebase/v2 -replace go.etcd.io/etcd => github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7 +replace go.etcd.io/etcd => github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d github.com/DataDog/datadog-go v2.2.0+incompatible github.com/HdrHistogram/hdrhistogram-go v1.1.0 // indirect + github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d github.com/beevik/ntp v0.3.0 github.com/benbjohnson/immutable v0.3.0 + github.com/beorn7/perks v1.0.0 github.com/buger/jsonparser v1.1.1 github.com/cespare/xxhash v1.1.0 + github.com/coreos/go-semver v0.3.0 + github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f github.com/davecgh/go-spew v1.1.1 - github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f + github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/dustin/go-humanize v1.0.0 github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 + github.com/go-ole/go-ole v1.2.4 github.com/go-test/deep v1.0.7 github.com/gogo/protobuf v1.3.2 + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.3.3 + github.com/google/btree v1.0.0 github.com/google/go-cmp v0.5.5 - github.com/google/uuid v1.1.4 // indirect + github.com/google/uuid v1.1.4 github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.0 + github.com/gorilla/websocket v1.4.2 + github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/grpc-gateway v1.9.5 github.com/improbable-eng/grpc-web v0.13.0 + github.com/jonboulle/clockwork v0.1.0 + github.com/json-iterator/go v1.1.7 github.com/lib/pq v1.8.0 + github.com/matttproud/golang_protobuf_extensions v1.0.1 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd + github.com/modern-go/reflect2 v1.0.1 github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b github.com/opentracing/opentracing-go v1.1.0 github.com/pelletier/go-toml v1.4.0 github.com/pkg/errors v0.9.1 + github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.1.0 + github.com/prometheus/common v0.7.0 + github.com/prometheus/procfs v0.0.2 github.com/prometheus/prom2json v1.3.0 github.com/rakyll/statik v0.1.7 - github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect - github.com/rs/cors v1.7.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 + github.com/rs/cors v1.7.0 github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil/v3 v3.20.11 + github.com/sirupsen/logrus v1.4.2 + github.com/soheilhy/cmux v0.1.4 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.7.0 + github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 github.com/uber/jaeger-client-go v2.25.0+incompatible - github.com/uber/jaeger-lib v2.4.0+incompatible // indirect + github.com/uber/jaeger-lib v2.4.0+incompatible + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 github.com/zeebo/blake3 v0.1.1 go.etcd.io/bbolt v1.3.5 go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b + go.uber.org/atomic v1.4.0 + go.uber.org/multierr v1.1.0 + go.uber.org/zap v1.10.0 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7 golang.org/x/mod v0.4.2 - golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect + golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect - golang.org/x/text v0.3.5 // indirect + golang.org/x/sys v0.0.0-20210510120138-977fb7262007 + golang.org/x/text v0.3.5 + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 + google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a google.golang.org/grpc v1.28.0 gopkg.in/yaml.v2 v2.3.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 - sigs.k8s.io/yaml v1.2.0 // indirect + sigs.k8s.io/yaml v1.2.0 vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible ) diff --git a/go.sum b/go.sum index 5f4d1be61..5d6ee4b64 100644 --- a/go.sum +++ b/go.sum @@ -232,6 +232,8 @@ github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s= github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7 h1:hufElvtCighE0G2VFJYDGWCY8JlmCWZ1FmXvlf25yUQ= github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7/go.mod h1:1X1h4BZ44WjM0LJof1gKKLap1OA4RsicGCDRtACTkLI= +github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c h1:YnU+8kIrr/7IDGtIYncawklAs54EWihED4DBIm+kjAA= +github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c/go.mod h1:1X1h4BZ44WjM0LJof1gKKLap1OA4RsicGCDRtACTkLI= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= diff --git a/idalloc_test.go b/idalloc_test.go index dff559146..b28a56604 100644 --- a/idalloc_test.go +++ b/idalloc_test.go @@ -40,7 +40,7 @@ func TestIDAlloc(t *testing.T) { }() // Open bolt. - db, err := bolt.Open(f.Name(), 0666, &bolt.Options{Timeout: 1 * time.Second}) + db, err := bolt.Open(f.Name(), 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: true}) if err != nil { t.Errorf("opening bolt: %v", err) return diff --git a/test/cluster.go b/test/cluster.go index 126245a46..137aa6c83 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -607,7 +607,7 @@ func prependTestServerOpts(opts []server.CommandOption) []server.CommandOption { pilosa.OptServerNodeDownRetries(5, 100*time.Millisecond), pilosa.OptServerStorageConfig(&storage.Config{ Backend: pilosa.CurrentBackendOrDefault(), - FsyncEnabled: true, + FsyncEnabled: false, }), ), } diff --git a/test/disco.go b/test/disco.go index 7aebd8f89..d00c06a31 100644 --- a/test/disco.go +++ b/test/disco.go @@ -111,6 +111,7 @@ func GetPortsGenConfigs(tb testing.TB, nodes []*Command) error { LPeerSocket: []*net.TCPListener{peerListener}, LClientSocket: []*net.TCPListener{clientListener}, BootstrapTimeout: 50 * time.Millisecond, + UnsafeNoFsync: true, } peerUrls[i] = fmt.Sprintf("%s=%s", name, peerURL) } @@ -144,14 +145,16 @@ func GenPortsConfig(tb testing.TB, ports []Ports) []*server.Config { BindGRPC: fmt.Sprintf(":%d", ports[i].Grpc), GRPCListener: ports[i].LsnG, Etcd: etcd.Options{ - Dir: discoDir, - LClientURL: lClientURL, - AClientURL: lClientURL, - LPeerURL: lPeerURL, - APeerURL: lPeerURL, - HeartbeatTTL: 5, - LPeerSocket: []*net.TCPListener{lsnP}, - LClientSocket: []*net.TCPListener{lsnC}, + Dir: discoDir, + LClientURL: lClientURL, + AClientURL: lClientURL, + LPeerURL: lPeerURL, + APeerURL: lPeerURL, + HeartbeatTTL: 5, + LPeerSocket: []*net.TCPListener{lsnP}, + LClientSocket: []*net.TCPListener{lsnC}, + BootstrapTimeout: 50 * time.Millisecond, + UnsafeNoFsync: true, }, } cfgs[i].Cluster.Name = clusterName From 9ac4a5a8f47d382e7676e0821cb0fc6a9e8c3309 Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 30 Sep 2021 14:40:11 -0500 Subject: [PATCH 2/5] don't necessarily fsync RBF databases even on close when fsync is disabled In test runs, we open, and close, *huge* numbers of databases. Even the single fsync on close for these ends up being expensive on some hosts. *cough* Apple. At least in theory, writes delivered to the disk are just as written whether or not you've hit fsync, as long as the machine doesn't power off before getting to them. In the circumstances where we disable fsync, that's fine. Since we already have an fsync function for "fsync if it's not disabled", use that. --- rbf/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rbf/db.go b/rbf/db.go index 92efffd72..f1623ded8 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -274,7 +274,7 @@ func (db *DB) Close() (err error) { // Close writer handler. if db.file != nil { - err = db.file.Sync() + err = db.fsync(db.file) if err != nil { return } From e774acb4a075e3e9cdc104f7517a7114f6ffa5b8 Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 30 Sep 2021 14:51:12 -0500 Subject: [PATCH 3/5] disable a few more fsyncs in boltdb boltdb has a couple of places where it fsyncs even when fsync is disabled, this turns out to cost an amazing amount of time over several thousand databases in our test run. In theory, they are rare circumstances compared to updates; in practice, when you open 256 partition key translation databases per server opened and most of them never get written to, not so much. --- go.mod | 1 + go.sum | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/go.mod b/go.mod index 23401e34f..ab18495a3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/molecula/featurebase/v2 replace go.etcd.io/etcd => github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c +replace go.etcd.io/bbolt => github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554 require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d diff --git a/go.sum b/go.sum index 5d6ee4b64..2d81af164 100644 --- a/go.sum +++ b/go.sum @@ -287,6 +287,10 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seebs/bbolt v0.0.0-20210930171653-b02e799f10a9 h1:T3XzfA3QYkfNOLAi7p44L8RdGkLB0wnGjb+Uf8bvA9I= +github.com/seebs/bbolt v0.0.0-20210930171653-b02e799f10a9/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554 h1:88K0ffxhVphUHxlqW4ewOaXdnJByH4LcCuvYfv0QI/M= +github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA= github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -446,6 +450,7 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 214a1492a8d5ff3cf084273ceadfc5e48bfd5edb Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 30 Sep 2021 14:41:48 -0500 Subject: [PATCH 4/5] kill off a ton more fsyncs Performance of tests on MacOS has been atrocious for a while, and a lot of that is fsync, so we're trying to make that optional. To test all of this, I modified RBF to panic if anything tried to open an RBF database without disabling fsync, and ran the tests that way, and tracked down the various places this could still happen. There's a lot of places in our tree where we were creating test holders which were not getting created with fsync disabled, which results in a surprisingly large number of points at which we end up calling fsync in tests, which makes tests much slower than they need to be. There's also a bunch of places where the flags don't get propagated correctly; for instance, storage.fsync didn't propagate to the RBFConfig. We add an "fsync enabled" flag to OpenTranslateStoreFunc, so we can tell translation stores that we don't need syncing, so the server's config can be passed on appropriately. More of the test code that sets things up is correctly configuring that flag by default. We also change the barely-used bolt storage backend to support this as well. With this done, the only calls to fsync left in a run of `go test -short` in the top-level directory are from the zap logger in etcd, and consumed around 0.03 seconds. The overall impact is that `go test -short` went from "takes enough more than 10 minutes that i don't know how long it takes" to about 2.5 minutes. --- bolt.go | 10 +++++++-- bolt_test.go | 3 ++- boltdb/translate.go | 26 +++++++++++++----------- boltdb/translate_test.go | 2 +- cluster_internal_test.go | 5 ++++- dbshard_internal_test.go | 1 + executor_internal_test.go | 2 +- executor_test.go | 2 +- field.go | 2 +- field_internal_test.go | 2 ++ field_test.go | 6 +++--- fragment_internal_test.go | 4 ++-- holder.go | 6 +++--- holder_internal_test.go | 3 +++ holder_test.go | 17 +++++++++++++++- idalloc.go | 14 +++++++------ index.go | 2 +- index_internal_test.go | 2 +- index_test.go | 2 +- internal/clustertests/pause_node_test.go | 2 +- server.go | 3 +++ server_internal_test.go | 4 +++- test/holder.go | 5 ++++- test/index.go | 5 ++++- translate.go | 4 ++-- utils_internal_test.go | 2 +- view_internal_test.go | 2 +- 27 files changed, 92 insertions(+), 46 deletions(-) diff --git a/bolt.go b/bolt.go index e679b1400..12435abef 100644 --- a/bolt.go +++ b/bolt.go @@ -135,8 +135,12 @@ func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storag if !DirExists(path) { PanicOn(os.MkdirAll(dir, 0755)) } + fsyncEnabled := true + if cfg != nil { + fsyncEnabled = cfg.FsyncEnabled + } - db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize}) + db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !fsyncEnabled}) if err != nil { return nil, errors.Wrapf(err, fmt.Sprintf("open bolt path '%v'", path)) } @@ -187,6 +191,7 @@ func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storag openTx: make(map[*BoltTx]bool), DeleteEmptyContainer: true, + fsyncEnabled: cfg.FsyncEnabled, } r.unprotectedRegister(w) @@ -226,7 +231,7 @@ func (w *BoltWrapper) CloseDB() error { func (w *BoltWrapper) OpenDB() error { w.muDb.Lock() defer w.muDb.Unlock() - db, err := bolt.Open(w.path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize}) + db, err := bolt.Open(w.path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !w.fsyncEnabled}) if err != nil { return err } @@ -315,6 +320,7 @@ type BoltWrapper struct { doAllocZero bool DeleteEmptyContainer bool + fsyncEnabled bool // for tracking whether our initial config wanted fsync on openTx map[*BoltTx]bool } diff --git a/bolt_test.go b/bolt_test.go index 5c83ae1af..1dc03b7d0 100644 --- a/bolt_test.go +++ b/bolt_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/molecula/featurebase/v2/roaring" + "github.com/molecula/featurebase/v2/storage" . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck ) @@ -88,7 +89,7 @@ func mustOpenEmptyBoltWrapper(path string) (w *BoltWrapper, cleaner func()) { var err error fn := path PanicOn(os.RemoveAll(fn)) - ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, nil) + ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, &storage.Config{FsyncEnabled: false}) PanicOn(err) w = ww.(*BoltWrapper) diff --git a/boltdb/translate.go b/boltdb/translate.go index 076c77f37..35ef2b08f 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -55,8 +55,8 @@ const ( ) // OpenTranslateStore opens and initializes a boltdb translation store. -func OpenTranslateStore(path, index, field string, partitionID, partitionN int) (pilosa.TranslateStore, error) { - s := NewTranslateStore(index, field, partitionID, partitionN) +func OpenTranslateStore(path, index, field string, partitionID, partitionN int, fsyncEnabled bool) (pilosa.TranslateStore, error) { + s := NewTranslateStore(index, field, partitionID, partitionN, fsyncEnabled) s.Path = path if err := s.Open(); err != nil { return nil, err @@ -88,22 +88,24 @@ type TranslateStore struct { once sync.Once closing chan struct{} - readOnly bool - writeNotify chan struct{} + readOnly bool + fsyncEnabled bool + writeNotify chan struct{} // File path to database file. Path string } // NewTranslateStore returns a new instance of TranslateStore. -func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore { +func NewTranslateStore(index, field string, partitionID, partitionN int, fsyncEnabled bool) *TranslateStore { return &TranslateStore{ - index: index, - field: field, - partitionID: partitionID, - partitionN: partitionN, - closing: make(chan struct{}), - writeNotify: make(chan struct{}), + index: index, + field: field, + partitionID: partitionID, + partitionN: partitionN, + closing: make(chan struct{}), + writeNotify: make(chan struct{}), + fsyncEnabled: fsyncEnabled, } } @@ -120,7 +122,7 @@ func (s *TranslateStore) Open() (err error) { if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path)) - } else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil { + } else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: !s.fsyncEnabled}); err != nil { return errors.Wrapf(err, "open file: %s", err) } diff --git a/boltdb/translate_test.go b/boltdb/translate_test.go index a42d68d50..61f2e61c3 100644 --- a/boltdb/translate_test.go +++ b/boltdb/translate_test.go @@ -393,7 +393,7 @@ func MustNewTranslateStore(tb testing.TB) *boltdb.TranslateStore { panic(err) } - s := boltdb.NewTranslateStore("I", "F", 0, topology.DefaultPartitionN) + s := boltdb.NewTranslateStore("I", "F", 0, topology.DefaultPartitionN, false) s.Path = f.Name() return s } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 8af84efed..cc55ae9aa 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -155,7 +155,10 @@ func newIndexWithTempPath(tb testing.TB, name string) *Index { if err != nil { panic(err) } - h := NewHolder(path, nil) + cfg := DefaultHolderConfig() + cfg.StorageConfig.FsyncEnabled = false + cfg.RBFConfig.FsyncEnabled = false + h := NewHolder(path, cfg) PanicOn(h.Open()) index, err := h.CreateIndex(name, IndexOptions{}) testhook.Cleanup(tb, func() { diff --git a/dbshard_internal_test.go b/dbshard_internal_test.go index 81d64436c..9482d1504 100644 --- a/dbshard_internal_test.go +++ b/dbshard_internal_test.go @@ -330,6 +330,7 @@ func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) { cfg := mustHolderConfig() cfg.StorageConfig.Backend = "rbf" + cfg.StorageConfig.FsyncEnabled = false holder := NewHolder(tmpdir, cfg) defer holder.Close() diff --git a/executor_internal_test.go b/executor_internal_test.go index 4aa37971c..b752957ef 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -28,7 +28,7 @@ import ( func TestExecutor_TranslateRowsOnBool(t *testing.T) { path, _ := testhook.TempDirInDir(t, *TempDir, "pilosa-executor-") - holder := NewHolder(path, nil) + holder := NewHolder(path, mustHolderConfig()) defer holder.Close() e := &executor{ diff --git a/executor_test.go b/executor_test.go index 414bf47b5..adcf64dbb 100644 --- a/executor_test.go +++ b/executor_test.go @@ -6882,7 +6882,7 @@ func TestMissingKeyRegression(t *testing.T) { c := test.MustRunCluster(t, 1, []server.CommandOption{server.OptCommandServerOptions( pilosa.OptServerStorageConfig(&storage.Config{ Backend: "roaring", - FsyncEnabled: true, + FsyncEnabled: false, }))}) defer c.Close() diff --git a/field.go b/field.go index a6c4b5e75..4edccfb3f 100644 --- a/field.go +++ b/field.go @@ -648,7 +648,7 @@ func (f *Field) writeAvailableShards() { func (f *Field) applyTranslateStore() error { // Instantiate & open translation store. var err error - f.translateStore, err = f.OpenTranslateStore(f.TranslateStorePath(), f.index, f.name, -1, -1) + f.translateStore, err = f.OpenTranslateStore(f.TranslateStorePath(), f.index, f.name, -1, -1, f.holder.cfg.StorageConfig.FsyncEnabled) if err != nil { return errors.Wrap(err, "opening field translate store") } diff --git a/field_internal_test.go b/field_internal_test.go index 88ca4b7d7..080836baa 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -244,6 +244,8 @@ func NewTestField(t testing.TB, opts FieldOption) *TestField { cfg := DefaultHolderConfig() cfg.StorageConfig.Backend = CurrentBackendOrDefault() + cfg.StorageConfig.FsyncEnabled = false + cfg.RBFConfig.FsyncEnabled = false h := NewHolder(path, cfg) PanicOn(h.Open()) diff --git a/field_test.go b/field_test.go index 38056e8ac..0bc3b60cb 100644 --- a/field_test.go +++ b/field_test.go @@ -159,7 +159,7 @@ func TestField_NameRestriction(t *testing.T) { if err != nil { panic(err) } - field, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", ".meta", pilosa.OptFieldTypeDefault()) + field, err := pilosa.NewField(pilosa.NewHolder(path, mustHolderConfig()), path, "i", ".meta", pilosa.OptFieldTypeDefault()) if field != nil { t.Fatalf("unexpected field name %s", err) } @@ -192,13 +192,13 @@ func TestField_NameValidation(t *testing.T) { panic(err) } for _, name := range validFieldNames { - _, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault()) + _, err := pilosa.NewField(pilosa.NewHolder(path, mustHolderConfig()), path, "i", name, pilosa.OptFieldTypeDefault()) if err != nil { t.Fatalf("unexpected field name: %s %s", name, err) } } for _, name := range invalidFieldNames { - _, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault()) + _, err := pilosa.NewField(pilosa.NewHolder(path, mustHolderConfig()), path, "i", name, pilosa.OptFieldTypeDefault()) if err == nil { t.Fatalf("expected error on field name: %s", name) } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index a7915fcc3..9b4ac461a 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -3166,7 +3166,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) { origF.Close() fi.Close() - h := NewHolder(fi.Name(), nil) + h := NewHolder(fi.Name(), mustHolderConfig()) PanicOn(h.Open()) idx, err := h.CreateIndex("i", IndexOptions{}) PanicOn(err) @@ -5146,7 +5146,7 @@ func TestImportClearRestart(t *testing.T) { PanicOn(tx2.Commit()) - h3 := NewHolder(filepath.Dir(f2.path()), nil) + h3 := NewHolder(filepath.Dir(f2.path()), mustHolderConfig()) testhook.Cleanup(t, func() { h3.Close() }) diff --git a/holder.go b/holder.go index 0daf8416d..dea18e3e7 100644 --- a/holder.go +++ b/holder.go @@ -109,7 +109,7 @@ type Holder struct { OpenTransactionStore OpenTransactionStoreFunc // Func to open the ID allocator. - OpenIDAllocator func(string) (*idAllocator, error) + OpenIDAllocator func(string, bool) (*idAllocator, error) // transactionManager transactionManager *TransactionManager @@ -241,7 +241,7 @@ func DefaultHolderConfig() *HolderConfig { OpenTranslateStore: OpenInMemTranslateStore, OpenTranslateReader: nil, OpenTransactionStore: OpenInMemTransactionStore, - OpenIDAllocator: func(string) (*idAllocator, error) { return &idAllocator{}, nil }, + OpenIDAllocator: func(string, bool) (*idAllocator, error) { return &idAllocator{}, nil }, TranslationSyncer: NopTranslationSyncer, Serializer: GobSerializer, Schemator: disco.InMemSchemator, @@ -623,7 +623,7 @@ func (h *Holder) Open() error { h.transactionManager.Log = h.Logger // Open ID allocator. - h.ida, err = h.OpenIDAllocator(filepath.Join(h.path, "idalloc.db")) + h.ida, err = h.OpenIDAllocator(filepath.Join(h.path, "idalloc.db"), h.cfg.StorageConfig.FsyncEnabled) if err != nil { return errors.Wrap(err, "opening ID allocator") } diff --git a/holder_internal_test.go b/holder_internal_test.go index c87b6bf73..bd35f0b03 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -86,6 +86,7 @@ func makeHolder(tb testing.TB, backend string) (*Holder, string, error) { cfg := mustHolderConfig() if backend != "" { cfg.StorageConfig.Backend = backend + cfg.StorageConfig.FsyncEnabled = false } h := NewHolder(path, cfg) return h, path, h.Open() @@ -265,6 +266,8 @@ func mustHolderConfig() *HolderConfig { _ = MustBackendToTxtype(backend) cfg.StorageConfig.Backend = backend } + cfg.StorageConfig.FsyncEnabled = false + cfg.RBFConfig.FsyncEnabled = false cfg.Schemator = disco.InMemSchemator cfg.Sharder = disco.InMemSharder return cfg diff --git a/holder_test.go b/holder_test.go index 6aeeab79d..4807a997e 100644 --- a/holder_test.go +++ b/holder_test.go @@ -25,11 +25,26 @@ import ( "time" "github.com/molecula/featurebase/v2" + "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/pql" "github.com/molecula/featurebase/v2/test" "github.com/pkg/errors" ) +// mustHolderConfig provides a default test-friendly holder config. +func mustHolderConfig() *pilosa.HolderConfig { + cfg := pilosa.DefaultHolderConfig() + if backend := pilosa.CurrentBackend(); backend != "" { + _ = pilosa.MustBackendToTxtype(backend) + cfg.StorageConfig.Backend = backend + } + cfg.StorageConfig.FsyncEnabled = false + cfg.RBFConfig.FsyncEnabled = false + cfg.Schemator = disco.InMemSchemator + cfg.Sharder = disco.InMemSharder + return cfg +} + func TestHolder_Open(t *testing.T) { t.Run("ErrIndexPermission", func(t *testing.T) { if os.Geteuid() == 0 { @@ -283,7 +298,7 @@ func TestHolder_HasData(t *testing.T) { // Note that we are intentionally not using test.NewHolder, // because we want to create a Holder object with an invalid path, // rather than creating a valid holder with a temporary path. - h := pilosa.NewHolder("bad-path", nil) + h := pilosa.NewHolder("bad-path", mustHolderConfig()) if ok, err := h.HasData(); ok || err != nil { t.Fatal("expected HasData to return false, no err, but", ok, err) diff --git a/idalloc.go b/idalloc.go index 4c6ea423f..ffa07baf8 100644 --- a/idalloc.go +++ b/idalloc.go @@ -53,18 +53,20 @@ func (k IDAllocKey) String() string { } type idAllocator struct { - db *bolt.DB + db *bolt.DB + fsyncEnabled bool } -type OpenIDAllocatorFunc func(path string) (*idAllocator, error) // whyyyyyyyyy +type OpenIDAllocatorFunc func(path string, enableFsync bool) (*idAllocator, error) // whyyyyyyyyy -func OpenIDAllocator(path string) (*idAllocator, error) { - db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second}) +func OpenIDAllocator(path string, enableFsync bool) (*idAllocator, error) { + db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: !enableFsync}) if err != nil { return nil, err } - return &idAllocator{db}, nil + return &idAllocator{db: db, fsyncEnabled: enableFsync}, nil } + func (ida *idAllocator) Replace(reader io.Reader) error { newFile := ida.db.Path() + ".bak" liveFile := ida.db.Path() @@ -92,7 +94,7 @@ func (ida *idAllocator) Replace(reader io.Reader) error { } else { _ = os.Remove(liveFile + ".sav") } - db, err := bolt.Open(liveFile, 0666, &bolt.Options{Timeout: 1 * time.Second}) + db, err := bolt.Open(liveFile, 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: !ida.fsyncEnabled}) ida.db = db return err } diff --git a/index.go b/index.go index 25b7bd5ee..83a4e8e4a 100644 --- a/index.go +++ b/index.go @@ -249,7 +249,7 @@ func (i *Index) open(idx *disco.Index) (err error) { partitionID := partitionID g.Go(func() error { - store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.holder.partitionN) + store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.holder.partitionN, i.holder.cfg.StorageConfig.FsyncEnabled) if err != nil { return errors.Wrapf(err, "opening index translate store: partition=%d", partitionID) } diff --git a/index_internal_test.go b/index_internal_test.go index a8cd51706..0bf4f6909 100644 --- a/index_internal_test.go +++ b/index_internal_test.go @@ -26,7 +26,7 @@ func mustOpenIndex(tb testing.TB, opt IndexOptions) *Index { if err != nil { panic(err) } - h := NewHolder(path, nil) + h := NewHolder(path, mustHolderConfig()) index, err := h.CreateIndex("i", opt) testhook.Cleanup(tb, func() { h.Close() diff --git a/index_test.go b/index_test.go index 149133108..890d481d1 100644 --- a/index_test.go +++ b/index_test.go @@ -261,7 +261,7 @@ func TestIndex_InvalidName(t *testing.T) { if err != nil { panic(err) } - index, err := pilosa.NewIndex(pilosa.NewHolder(path, nil), path, "ABC") + index, err := pilosa.NewIndex(pilosa.NewHolder(path, mustHolderConfig()), path, "ABC") if err == nil { t.Fatalf("should have gotten an error on index name with caps") } diff --git a/internal/clustertests/pause_node_test.go b/internal/clustertests/pause_node_test.go index e9c4e822a..2de330a74 100644 --- a/internal/clustertests/pause_node_test.go +++ b/internal/clustertests/pause_node_test.go @@ -168,7 +168,7 @@ func openTranslateStores(dirPath, index string) (map[int]pilosa.TranslateStore, return nil, err } // open bolt db - ts, err := boltdb.OpenTranslateStore(filePath, index, "", partition, topology.DefaultPartitionN) + ts, err := boltdb.OpenTranslateStore(filePath, index, "", partition, topology.DefaultPartitionN, false) ts.SetReadOnly(true) if err != nil { return nil, err diff --git a/server.go b/server.go index 93d9947a0..fdca4dc39 100644 --- a/server.go +++ b/server.go @@ -339,6 +339,9 @@ func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption { func OptServerStorageConfig(cfg *storage.Config) ServerOption { return func(s *Server) error { s.holderConfig.StorageConfig = cfg + // For historical reasons, RBF's config can ignore the storage config + // in some cases. + s.holderConfig.RBFConfig.FsyncEnabled = s.holderConfig.StorageConfig.FsyncEnabled return nil } } diff --git a/server_internal_test.go b/server_internal_test.go index 7fbddf73e..7a1b7b121 100644 --- a/server_internal_test.go +++ b/server_internal_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/molecula/featurebase/v2/storage" "github.com/molecula/featurebase/v2/testhook" ) @@ -45,8 +46,9 @@ func TestMonitorAntiEntropyZero(t *testing.T) { if err != nil { t.Fatalf("getting temp dir: %v", err) } + cfg := &storage.Config{FsyncEnabled: false, Backend: storage.DefaultBackend} s, err := NewServer(OptServerDataDir(td), - OptServerAntiEntropyInterval(0)) + OptServerAntiEntropyInterval(0), OptServerStorageConfig(cfg)) if err != nil { t.Fatalf("making new server: %v", err) } diff --git a/test/holder.go b/test/holder.go index 079495488..6eb63efac 100644 --- a/test/holder.go +++ b/test/holder.go @@ -38,7 +38,10 @@ func NewHolder(tb testing.TB) *Holder { panic(err) } - h := &Holder{Holder: pilosa.NewHolder(path, nil)} + cfg := pilosa.DefaultHolderConfig() + cfg.StorageConfig.FsyncEnabled = false + cfg.RBFConfig.FsyncEnabled = false + h := &Holder{Holder: pilosa.NewHolder(path, cfg)} return h } diff --git a/test/index.go b/test/index.go index 608bcae1c..46bc7399b 100644 --- a/test/index.go +++ b/test/index.go @@ -33,7 +33,10 @@ func newIndex(tb testing.TB) *Index { if err != nil { panic(err) } - h := pilosa.NewHolder(path, pilosa.DefaultHolderConfig()) + cfg := pilosa.DefaultHolderConfig() + cfg.StorageConfig.FsyncEnabled = false + cfg.RBFConfig.FsyncEnabled = false + h := pilosa.NewHolder(path, cfg) testhook.Cleanup(tb, func() { h.Close() }) diff --git a/translate.go b/translate.go index f449d75be..256dae341 100644 --- a/translate.go +++ b/translate.go @@ -197,7 +197,7 @@ TranslatorSummary{ } // OpenTranslateStoreFunc represents a function for instantiating and opening a TranslateStore. -type OpenTranslateStoreFunc func(path, index, field string, partitionID, partitionN int) (TranslateStore, error) +type OpenTranslateStoreFunc func(path, index, field string, partitionID, partitionN int, fsyncEnabled bool) (TranslateStore, error) // GenerateNextPartitionedID returns the next ID within the same partition. func GenerateNextPartitionedID(index string, prev uint64, partitionID, partitionN int) uint64 { @@ -407,7 +407,7 @@ var _ OpenTranslateStoreFunc = OpenInMemTranslateStore // OpenInMemTranslateStore returns a new instance of InMemTranslateStore. // Implements OpenTranslateStoreFunc. -func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partitionN int) (TranslateStore, error) { +func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partitionN int, fsyncEnabled bool) (TranslateStore, error) { return NewInMemTranslateStore(index, field, partitionID, partitionN), nil } diff --git a/utils_internal_test.go b/utils_internal_test.go index a3a6af2a7..76c3aa66f 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -117,7 +117,7 @@ func NewTestClusterWithReplication(tb testing.TB, nNodes, nReplicas, partitionN } // holder - h := NewHolder(path, nil) + h := NewHolder(path, mustHolderConfig()) // cluster availableShardFileFlushDuration.Set(100 * time.Millisecond) diff --git a/view_internal_test.go b/view_internal_test.go index 9258cdd27..e7d2036d8 100644 --- a/view_internal_test.go +++ b/view_internal_test.go @@ -35,7 +35,7 @@ func mustOpenView(tb testing.TB, index, field, name string) *view { CacheSize: DefaultCacheSize, } - h := NewHolder(path, nil) + h := NewHolder(path, mustHolderConfig()) // h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment cim := &CreateIndexMessage{ From 9db87f78d004b979ca2ca4997598540e14182ecc Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 30 Sep 2021 15:44:06 -0500 Subject: [PATCH 5/5] fix go.mod/go.sum --- go.mod | 54 +++++++++++------------------------------------------- go.sum | 8 -------- 2 files changed, 11 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index ab18495a3..ff7c1d715 100644 --- a/go.mod +++ b/go.mod @@ -1,97 +1,65 @@ module github.com/molecula/featurebase/v2 replace go.etcd.io/etcd => github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c + replace go.etcd.io/bbolt => github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554 require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d github.com/DataDog/datadog-go v2.2.0+incompatible github.com/HdrHistogram/hdrhistogram-go v1.1.0 // indirect - github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d github.com/beevik/ntp v0.3.0 github.com/benbjohnson/immutable v0.3.0 - github.com/beorn7/perks v1.0.0 github.com/buger/jsonparser v1.1.1 github.com/cespare/xxhash v1.1.0 - github.com/coreos/go-semver v0.3.0 - github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e - github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f github.com/davecgh/go-spew v1.1.1 - github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f - github.com/dgrijalva/jwt-go v3.2.0+incompatible - github.com/dustin/go-humanize v1.0.0 + github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect + github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 - github.com/go-ole/go-ole v1.2.4 github.com/go-test/deep v1.0.7 github.com/gogo/protobuf v1.3.2 - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.3.3 - github.com/google/btree v1.0.0 github.com/google/go-cmp v0.5.5 - github.com/google/uuid v1.1.4 + github.com/google/uuid v1.1.4 // indirect github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.0 - github.com/gorilla/websocket v1.4.2 - github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 - github.com/grpc-ecosystem/grpc-gateway v1.9.5 github.com/improbable-eng/grpc-web v0.13.0 - github.com/jonboulle/clockwork v0.1.0 - github.com/json-iterator/go v1.1.7 github.com/lib/pq v1.8.0 - github.com/matttproud/golang_protobuf_extensions v1.0.1 - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd - github.com/modern-go/reflect2 v1.0.1 github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b github.com/opentracing/opentracing-go v1.1.0 github.com/pelletier/go-toml v1.4.0 github.com/pkg/errors v0.9.1 - github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.1.0 - github.com/prometheus/common v0.7.0 - github.com/prometheus/procfs v0.0.2 github.com/prometheus/prom2json v1.3.0 github.com/rakyll/statik v0.1.7 - github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 - github.com/rs/cors v1.7.0 + github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect + github.com/rs/cors v1.7.0 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil/v3 v3.20.11 - github.com/sirupsen/logrus v1.4.2 - github.com/soheilhy/cmux v0.1.4 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.7.0 - github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 github.com/uber/jaeger-client-go v2.25.0+incompatible - github.com/uber/jaeger-lib v2.4.0+incompatible - github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 + github.com/uber/jaeger-lib v2.4.0+incompatible // indirect github.com/zeebo/blake3 v0.1.1 go.etcd.io/bbolt v1.3.5 go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b - go.uber.org/atomic v1.4.0 - go.uber.org/multierr v1.1.0 - go.uber.org/zap v1.10.0 - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7 golang.org/x/mod v0.4.2 - golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 + golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210510120138-977fb7262007 - golang.org/x/text v0.3.5 - golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 - google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a + golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect + golang.org/x/text v0.3.5 // indirect google.golang.org/grpc v1.28.0 gopkg.in/yaml.v2 v2.3.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 - sigs.k8s.io/yaml v1.2.0 + sigs.k8s.io/yaml v1.2.0 // indirect vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible ) diff --git a/go.sum b/go.sum index 2d81af164..0959b010c 100644 --- a/go.sum +++ b/go.sum @@ -230,8 +230,6 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y= github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s= -github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7 h1:hufElvtCighE0G2VFJYDGWCY8JlmCWZ1FmXvlf25yUQ= -github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7/go.mod h1:1X1h4BZ44WjM0LJof1gKKLap1OA4RsicGCDRtACTkLI= github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c h1:YnU+8kIrr/7IDGtIYncawklAs54EWihED4DBIm+kjAA= github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c/go.mod h1:1X1h4BZ44WjM0LJof1gKKLap1OA4RsicGCDRtACTkLI= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= @@ -287,8 +285,6 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/seebs/bbolt v0.0.0-20210930171653-b02e799f10a9 h1:T3XzfA3QYkfNOLAi7p44L8RdGkLB0wnGjb+Uf8bvA9I= -github.com/seebs/bbolt v0.0.0-20210930171653-b02e799f10a9/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554 h1:88K0ffxhVphUHxlqW4ewOaXdnJByH4LcCuvYfv0QI/M= github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA= @@ -350,9 +346,6 @@ github.com/zeebo/blake3 v0.1.1 h1:Nbsts7DdKThRHHd+YNlqiGlRqGEF2bE2eXN+xQ1hsEs= github.com/zeebo/blake3 v0.1.1/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4= github.com/zeebo/pcg v1.0.0 h1:dt+dx+HvX8g7Un32rY9XWoYnd0NmKmrIzpHF7qiTDj0= github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -449,7 +442,6 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=