From a4b37273ea27cfb3c254068fd6a9b151b1345e95 Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 4 Feb 2021 10:55:55 -0600 Subject: [PATCH 1/2] remove the rest of the gossip code (except config) --- api.go | 1 - cluster.go | 2 +- cmd/server_test.go | 6 - gossip/gossip.go | 565 ----------------------------------------- server/cluster_test.go | 119 +-------- server/handler_test.go | 6 +- server/server.go | 72 ------ server/server_test.go | 3 - test/cluster.go | 20 +- test/disco.go | 17 +- translator_test.go | 5 - 11 files changed, 16 insertions(+), 800 deletions(-) diff --git a/api.go b/api.go index 4c8221170..b6ceadbb2 100644 --- a/api.go +++ b/api.go @@ -214,7 +214,6 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN) if !snap.IsPrimaryFieldTranslationNode(api.Node().ID) { - fmt.Println("--- DEBUG: forward to coordinator") if err := api.server.defaultClient.CreateIndex(ctx, indexName, options); err != nil { return nil, errors.Wrap(err, "forwarding CreateIndex to coordinator") } diff --git a/cluster.go b/cluster.go index 51aff4e4d..0bb39e80f 100644 --- a/cluster.go +++ b/cluster.go @@ -105,7 +105,7 @@ type cluster struct { // nolint: maligned sharder disco.Sharder // Required for cluster Resize. - Static bool // Static is primarily used for testing in a non-gossip environment. + Static bool // Static is primarily used for testing. holder *Holder broadcaster broadcaster diff --git a/cmd/server_test.go b/cmd/server_test.go index b99d88ab9..f698b9e20 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -201,8 +201,6 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) { bind = ` + nextPort() + ` bind-grpc = ` + nextPort() + ` data-dir = "` + actualDataDir + `" - [gossip] - port = "14321" `, validation: func() error { v := validator{} @@ -218,8 +216,6 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) { cfgFileContent: ` bind = ` + nextPort() + ` bind-grpc = ` + nextPort() + ` - [gossip] - port = "14321" `, validation: func() error { v := validator{} @@ -235,8 +231,6 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) { cfgFileContent: ` bind = ` + nextPort() + ` bind-grpc = ` + nextPort() + ` - [gossip] - port = "14321" `, validation: func() error { v := validator{} diff --git a/gossip/gossip.go b/gossip/gossip.go index 2d3b413d0..10b3e2d0e 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -15,556 +15,9 @@ package gossip import ( - "bytes" - "context" - "fmt" - "io" - "io/ioutil" - "log" - "net" - "os" - "strconv" - "strings" - "sync" - "time" - - "github.com/hashicorp/memberlist" - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/logger" - pnet "github.com/pilosa/pilosa/v2/net" - "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/toml" - "github.com/pilosa/pilosa/v2/topology" - "github.com/pkg/errors" ) -// Ensure GossipMemberSet implements interfaces. -var _ memberlist.Delegate = &memberSet{} - -// memberSet represents a gossip implementation of MemberSet using memberlist. -type memberSet struct { - mu sync.RWMutex - memberlist *memberlist.Memberlist - - broadcasts *memberlist.TransmitLimitedQueue - - papi *pilosa.API - config *config - - Logger logger.Logger - - // stdLogger is only used when passed into memberlist library things that take a std library logger rather than an interface. - stdLogger *log.Logger - // logOutput is similar to stdLogger in that it's passed to memberlist things which can't take a pilosa Logger. - logOutput io.Writer - - transport *Transport - - eventReceiver *eventReceiver -} - -// Open implements the MemberSet interface to start network activity. -func (g *memberSet) Open() (err error) { - g.mu.Lock() - defer g.mu.Unlock() - - g.memberlist, err = memberlist.Create(g.config.memberlistConfig) - - if err != nil { - return errors.Wrap(err, "creating memberlist") - } - - g.broadcasts = &memberlist.TransmitLimitedQueue{ - NumNodes: func() int { - g.mu.RLock() - defer g.mu.RUnlock() - return g.memberlist.NumMembers() - }, - RetransmitMult: 3, - } - - var uris = make([]*pnet.URI, len(g.config.gossipSeeds)) - for i, addr := range g.config.gossipSeeds { - uris[i], err = pnet.NewURIFromAddress(addr) - if err != nil { - return fmt.Errorf("new uri from address: %s", err) - } - } - - var nodes = make([]*topology.Node, len(uris)) - for i, uri := range uris { - nodes[i] = &topology.Node{URI: *uri} - } - - err = g.joinWithRetry(pnet.URIs(topology.Nodes(nodes).URIs()).HostPortStrings()) - if err != nil { - return errors.Wrap(err, "joinWithRetry") - } - return nil -} - -// Close attempts to gracefully leave the cluster, and finally calls shutdown -// after (at most) a timeout period. -func (g *memberSet) Close() error { - defer g.eventReceiver.Close() - - leaveErr := g.memberlist.Leave(5 * time.Second) - shutdownErr := g.memberlist.Shutdown() - if leaveErr != nil || shutdownErr != nil { - return fmt.Errorf("leaving: '%v', shutting down: '%v'", leaveErr, shutdownErr) - } - return nil -} - -// joinWithRetry wraps the standard memberlist Join function in a retry. -func (g *memberSet) joinWithRetry(hosts []string) error { - err := retry(60, 2*time.Second, func() error { - _, err := g.memberlist.Join(hosts) - return err - }) - return err -} - -// retry periodically retries function fn a specified number of attempts. -func retry(attempts int, sleep time.Duration, fn func() error) (err error) { // nolint: unparam - for i := 0; ; i++ { - err = fn() - if err == nil { - return - } - if i >= (attempts - 1) { - break - } - time.Sleep(sleep) - log.Println("retrying after error:", err) - } - return fmt.Errorf("after %d attempts, last error: %s", attempts, err) -} - -//////////////////////////////////////////////////////////////// - -type config struct { - gossipSeeds []string - memberlistConfig *memberlist.Config -} - -// memberSetOption describes a functional option for GossipMemberSet. -type memberSetOption func(*memberSet) error - -// WithTransport is a functional option for providing a transport to NewMemberSet. -func WithTransport(transport *Transport) memberSetOption { - return func(g *memberSet) error { - g.transport = transport - return nil - } -} - -// WithLogger is a functional option for providing a Go logger to NewMemberSet. -// If the memberSet's transport is nil, this logger will be used when creating -// one. If WithLogOutput is not used, this logger will be passed to memberlist -// for it to use internally. This logger is not used for logging by code in this -// (gossip) package - for that, use the WithPilosaLogger option. -func WithLogger(logger *log.Logger) memberSetOption { - return func(g *memberSet) error { - g.stdLogger = logger - return nil - } -} - -// WithLogOutput allows one to pass a Writer which will in turn be passed to -// memberlist for use in logging. -func WithLogOutput(o io.Writer) memberSetOption { - return func(g *memberSet) error { - g.logOutput = o - return nil - } -} - -// WithPilosaLogger allows one to configure a memberSet with a logger of their -// choice which satisfies the pilosa logger interface. -func WithPilosaLogger(l logger.Logger) memberSetOption { - return func(g *memberSet) error { - g.Logger = l - return nil - } -} - -// NewMemberSet returns a new instance of GossipMemberSet based on options. The -// logging options which can be passed to NewMemberSet are complicated for -// historical reasons - please pass WithPilosaLogger, and either WithLogOutput -// or WithLogger. If you pass WithLogOutput, be sure to also pass in a Transport -// using WithTransport. -func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*memberSet, error) { - host := api.Node().URI.Host - g := &memberSet{ - papi: api, - Logger: logger.NopLogger, - } - - // options - for _, opt := range options { - if err := opt(g); err != nil { - return nil, errors.Wrap(err, "executing option") - } - } - - ger := newEventReceiver(g.Logger, api) - g.eventReceiver = ger - - if g.transport == nil { - port, err := strconv.Atoi(cfg.Port) - if err != nil { - return nil, fmt.Errorf("convert port: %s", err) - } - - if g.stdLogger == nil { - if g.logOutput != nil { - g.stdLogger = logger.NewStandardLogger(g.logOutput).Logger() - } else { - g.stdLogger = log.New(os.Stderr, "", log.LstdFlags) - } - } - - // Set up the transport. - transport, err := NewTransport(host, port, g.stdLogger) - if err != nil { - return nil, fmt.Errorf("new tranport: %s", err) - } - - g.transport = transport - } - - port := g.transport.net.GetAutoBindPort() - - var gossipKey []byte - var err error - if cfg.Key != "" { - gossipKey, err = ioutil.ReadFile(cfg.Key) - if err != nil { - return nil, fmt.Errorf("reading gossip key: %s", err) - } - } - - //////////////////// - // memberlist config - conf := memberlist.DefaultWANConfig() - conf.Transport = g.transport.net - conf.Name = api.Node().ID - conf.BindAddr = api.Node().URI.Host - conf.BindPort = port - // AdvertisePort - if cfg.AdvertisePort != "" { - if p, err := strconv.Atoi(cfg.Port); err != nil { - return nil, fmt.Errorf("convert advertise port: %s", err) - } else { - conf.AdvertisePort = p - } - } else { - conf.AdvertisePort = port - } - // AdvertiseHost - if cfg.AdvertiseHost != "" { - conf.AdvertiseAddr = cfg.AdvertiseHost - } else { - conf.AdvertiseAddr = hostToIP(api.Node().URI.Host) - } - // - conf.TCPTimeout = time.Duration(cfg.StreamTimeout) - conf.SuspicionMult = cfg.SuspicionMult - conf.PushPullInterval = time.Duration(cfg.PushPullInterval) - conf.ProbeTimeout = time.Duration(cfg.ProbeTimeout) - conf.ProbeInterval = time.Duration(cfg.ProbeInterval) - conf.GossipNodes = cfg.Nodes - conf.GossipInterval = time.Duration(cfg.Interval) - conf.GossipToTheDeadTime = time.Duration(cfg.ToTheDeadTime) - // - conf.Delegate = g - conf.SecretKey = gossipKey - conf.Events = ger - if g.logOutput != nil { - conf.LogOutput = g.logOutput - } else { - conf.Logger = g.stdLogger - } - - g.config = &config{ - memberlistConfig: conf, - gossipSeeds: cfg.Seeds, - } - - return g, nil -} - -// NodeMeta implementation of the memberlist.Delegate interface. -func (g *memberSet) NodeMeta(limit int) []byte { - buf, err := g.papi.Serializer.Marshal(g.papi.Node()) - if err != nil { - g.Logger.Printf("marshal message error: %s", err) - return []byte{} - } - return buf -} - -// NotifyMsg implementation of the memberlist.Delegate interface -// called when a user-data message is received. -func (g *memberSet) NotifyMsg(b []byte) { - err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(b)) - if err != nil { - g.Logger.Printf("cluster message error: %s", err) - } -} - -// GetBroadcasts implementation of the memberlist.Delegate interface -// called when user data messages can be broadcast. -func (g *memberSet) GetBroadcasts(overhead, limit int) [][]byte { - return g.broadcasts.GetBroadcasts(overhead, limit) - -} - -// LocalState implementation of the memberlist.Delegate interface -// sends this Node's state data. -func (g *memberSet) LocalState(join bool) []byte { - schema, err := g.papi.Schema(context.Background()) - if err != nil { - // just panic, this code will be removed soon - panic(err) - } - m := &pilosa.NodeStatus{ - Node: g.papi.Node(), - Schema: &pilosa.Schema{Indexes: schema}, - } - for _, idx := range m.Schema.Indexes { - is := &pilosa.IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt} - - for _, f := range idx.Fields { - availableShards := roaring.NewBitmap() - if field, _ := g.papi.Field(context.Background(), idx.Name, f.Name); field != nil { - availableShards = field.AvailableShards(false) - } - - fs := &pilosa.FieldStatus{ - Name: f.Name, - CreatedAt: f.CreatedAt, - AvailableShards: availableShards, - } - is.Fields = append(is.Fields, fs) - } - m.Indexes = append(m.Indexes, is) - } - - // Marshal nodestate data to bytes. - buf, err := pilosa.MarshalInternalMessage(m, g.papi.Serializer) - if err != nil { - g.Logger.Printf("error marshalling nodestate data, err=%s", err) - return []byte{} - } - return buf -} - -// MergeRemoteState implementation of the memberlist.Delegate interface -// receive and process the remote side's LocalState. -func (g *memberSet) MergeRemoteState(buf []byte, join bool) { - err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf)) - if err != nil { - g.Logger.Printf("merge state error: %s", err) - } -} - -// eventReceiver is used to enable an application to receive -// events about joins and leaves over a channel. -// -// Care must be taken that events are processed in a timely manner from -// the channel, since this delegate will block until an event can be sent. -type eventReceiver struct { - ch chan memberlist.NodeEvent - closed chan struct{} - papi *pilosa.API - - logger logger.Logger -} - -// newEventReceiver returns a new instance of GossipEventReceiver. -func newEventReceiver(logger logger.Logger, papi *pilosa.API) *eventReceiver { - ger := &eventReceiver{ - ch: make(chan memberlist.NodeEvent, 1), - closed: make(chan struct{}), - logger: logger, - papi: papi, - } - go ger.listen() - return ger -} - -func (g *eventReceiver) NotifyJoin(n *memberlist.Node) { - // copy node to avoid data race - n2 := *n - n2.Meta = make([]byte, len(n.Meta)) - copy(n2.Meta, n.Meta) - - select { - case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2}: - case <-g.closed: - } -} - -func (g *eventReceiver) NotifyLeave(n *memberlist.Node) { - // copy node to avoid data race - n2 := *n - n2.Meta = make([]byte, len(n.Meta)) - copy(n2.Meta, n.Meta) - - select { - case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2}: - case <-g.closed: - } -} - -func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) { - // copy node to avoid data race - n2 := *n - n2.Meta = make([]byte, len(n.Meta)) - copy(n2.Meta, n.Meta) - - select { - case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2}: - case <-g.closed: - } -} - -func (g *eventReceiver) Close() { - // TODO workaround to make tests pass. We are going to delete this code anyways. - select { - case <-g.closed: - return - default: - close(g.closed) - } -} - -func (g *eventReceiver) listen() { - var nodeEventType pilosa.NodeEventType - for { - var e memberlist.NodeEvent - select { - case <-g.closed: - return - case e = <-g.ch: - } - switch e.Event { - case memberlist.NodeJoin: - nodeEventType = pilosa.NodeJoin - case memberlist.NodeLeave: - nodeEventType = pilosa.NodeLeave - case memberlist.NodeUpdate: - nodeEventType = pilosa.NodeUpdate - default: - continue - } - - // Get the node from the event.Node meta data. - var n topology.Node - if err := g.papi.Serializer.Unmarshal(e.Node.Meta, &n); err != nil { - panic("failed to unmarshal event node meta into node") - } - - ne := &pilosa.NodeEvent{ - Event: nodeEventType, - Node: &n, - } - buf, err := pilosa.MarshalInternalMessage(ne, g.papi.Serializer) - if err != nil { - panic(err) - } - if err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf)); err != nil { - g.logger.Printf("receive event error: %s", err) - } - } -} - -// Transport is a gossip transport for binding to a port. -type Transport struct { - //memberlist.Transport - net *memberlist.NetTransport - URI *pnet.URI -} - -// NewTransport returns a NetTransport based on the given host and port. -// It will dynamically bind to a port if port is 0. -// This is useful for test cases where specifying a port is not reasonable. -//func NewTransport(host string, port int) (*memberlist.NetTransport, error) { -func NewTransport(host string, port int, logger *log.Logger) (*Transport, error) { - // memberlist config - conf := memberlist.DefaultWANConfig() - conf.BindAddr = host - conf.BindPort = port - conf.AdvertisePort = port - conf.Logger = logger - - net, err := newTransport(conf) - if err != nil { - return nil, fmt.Errorf("new transport: %s", err) - } - - uri, err := pnet.NewURIFromHostPort(host, uint16(net.GetAutoBindPort())) - if err != nil { - return nil, fmt.Errorf("new uri from host port: %s", err) - } - - return &Transport{ - net: net, - URI: uri, - }, nil -} - -// newTransport returns a NetTransport based on the memberlist configuration. -// It will dynamically bind to a port if conf.BindPort is 0. -func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { - nc := &memberlist.NetTransportConfig{ - BindAddrs: []string{conf.BindAddr}, - BindPort: conf.BindPort, - Logger: conf.Logger, - } - - if conf.BindPort == 0 { - panic("TODO: remove this. problem: gossip conf.BindPort was 0!") - } - - // See comment below for details about the retry in here. - makeNetRetry := func(limit int) (*memberlist.NetTransport, error) { - var err error - for try := 0; try < limit; try++ { - var nt *memberlist.NetTransport - if nt, err = memberlist.NewNetTransport(nc); err == nil { - return nt, nil - } - if strings.Contains(err.Error(), "address already in use") { - conf.Logger.Printf("[DEBUG] Got bind error: %v", err) - continue - } - } - - return nil, fmt.Errorf("failed to obtain an address: %v", err) - } - - // The dynamic bind port operation is inherently racy because - // even though we are using the kernel to find a port for us, we - // are attempting to bind multiple protocols (and potentially - // multiple addresses) with the same port number. We build in a - // few retries here since this often gets transient errors in - // busy unit tests. - limit := 1 - if conf.BindPort == 0 { - limit = 10 - } - - nt, err := makeNetRetry(limit) - if err != nil { - return nil, errors.Wrap(err, "could not set up network transport") - } - - return nt, nil -} - // Config holds toml-friendly memberlist configuration. type Config struct { // Port indicates the port to which pilosa should bind for internal state sharing. @@ -638,21 +91,3 @@ type Config struct { Nodes int `toml:"nodes"` ToTheDeadTime toml.Duration `toml:"to-the-dead-time"` } - -// hostToIP converts host to an IP4 address based on net.LookupIP(). -func hostToIP(host string) string { - // if host is not an IP addr, check net.LookupIP() - if net.ParseIP(host) == nil { - hosts, err := net.LookupIP(host) - if err != nil { - return host - } - for _, h := range hosts { - // this restricts pilosa to IP4 - if h.To4() != nil { - return h.String() - } - } - } - return host -} diff --git a/server/cluster_test.go b/server/cluster_test.go index 23668c64f..cbadc6b65 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -29,7 +29,6 @@ import ( "github.com/pilosa/pilosa/v2/server" "github.com/pilosa/pilosa/v2/test" "github.com/pilosa/pilosa/v2/test/port" - "golang.org/x/sync/errgroup" ) // Ensure program can send/receive broadcast messages. @@ -173,8 +172,6 @@ func TestClusterResize_AddNode(t *testing.T) { m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() - seed := "" - // Create a client for each node. client0 := m0.Client() @@ -188,19 +185,16 @@ func TestClusterResize_AddNode(t *testing.T) { // Configure node1 m1 := test.NewCommandNode(t) - m1.Config.Gossip.Seeds = []string{seed} - if err := port.GetListeners(func(lsns []*net.TCPListener) error { portsCfg := test.GenPortsConfig(test.NewPorts(lsns)) - m1.Config.Gossip.Port = portsCfg[0].Gossip.Port m1.Config.Etcd = portsCfg[0].Etcd m1.Config.Name = portsCfg[0].Name m1.Config.Cluster.Name = portsCfg[0].Cluster.Name m1.Config.BindGRPC = portsCfg[0].BindGRPC return m1.Start() - }, 4, 10); err != nil { + }, 3, 10); err != nil { t.Fatalf("starting second main: %v", err) } defer m1.Close() @@ -219,8 +213,6 @@ func TestClusterResize_AddNode(t *testing.T) { m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() - seed := "" - // Create a client for each node. client0 := m0.Client() @@ -249,19 +241,16 @@ func TestClusterResize_AddNode(t *testing.T) { // Configure node1 m1 := test.NewCommandNode(t) - m1.Config.Gossip.Seeds = []string{seed} - if err := port.GetListeners(func(lsns []*net.TCPListener) error { portsCfg := test.GenPortsConfig(test.NewPorts(lsns)) - m1.Config.Gossip.Port = portsCfg[0].Gossip.Port m1.Config.Etcd = portsCfg[0].Etcd m1.Config.Name = portsCfg[0].Name m1.Config.Cluster.Name = portsCfg[0].Cluster.Name m1.Config.BindGRPC = portsCfg[0].BindGRPC return m1.Start() - }, 4, 10); err != nil { + }, 3, 10); err != nil { t.Fatalf("starting second main: %v", err) } defer m1.Close() @@ -283,8 +272,6 @@ func TestClusterResize_AddNode(t *testing.T) { m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() - seed := "" - // Create a client for each node. client0 := m0.Client() @@ -309,19 +296,17 @@ func TestClusterResize_AddNode(t *testing.T) { // Configure node1 m1 := test.NewCommandNode(t) - m1.Config.Gossip.Seeds = []string{seed} if err := port.GetListeners(func(lsns []*net.TCPListener) error { portsCfg := test.GenPortsConfig(test.NewPorts(lsns)) - m1.Config.Gossip.Port = portsCfg[0].Gossip.Port m1.Config.Etcd = portsCfg[0].Etcd m1.Config.Name = portsCfg[0].Name m1.Config.Cluster.Name = portsCfg[0].Cluster.Name m1.Config.BindGRPC = portsCfg[0].BindGRPC return m1.Start() - }, 4, 10); err != nil { + }, 3, 10); err != nil { t.Fatalf("starting second main: %v", err) } defer m1.Close() @@ -345,8 +330,6 @@ func TestClusterResize_AddNode(t *testing.T) { m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() - seed := "" - // Create a client for each node. client0 := m0.Client() @@ -375,19 +358,17 @@ func TestClusterResize_AddNode(t *testing.T) { // Configure node1 m1 := test.NewCommandNode(t) - m1.Config.Gossip.Seeds = []string{seed} if err := port.GetListeners(func(lsns []*net.TCPListener) error { portsCfg := test.GenPortsConfig(test.NewPorts(lsns)) - m1.Config.Gossip.Port = portsCfg[0].Gossip.Port m1.Config.Etcd = portsCfg[0].Etcd m1.Config.Name = portsCfg[0].Name m1.Config.Cluster.Name = portsCfg[0].Cluster.Name m1.Config.BindGRPC = portsCfg[0].BindGRPC return m1.Start() - }, 4, 10); err != nil { + }, 3, 10); err != nil { t.Fatalf("starting second main: %v", err) } @@ -416,8 +397,6 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() - seed := "" - // Create a client for each node. client0 := m0.Client() @@ -436,17 +415,15 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { // Configure node1 m1 := test.NewCommandNode(t) - m1.Config.Gossip.Seeds = []string{seed} if err := port.GetListeners(func(lsns []*net.TCPListener) error { portsCfg := test.GenPortsConfig(test.NewPorts(lsns)) - m1.Config.Gossip.Port = portsCfg[0].Gossip.Port m1.Config.Etcd = portsCfg[0].Etcd m1.Config.Name = portsCfg[0].Name m1.Config.Cluster.Name = portsCfg[0].Cluster.Name m1.Config.BindGRPC = portsCfg[0].BindGRPC return m1.Start() - }, 4, 10); err != nil { + }, 3, 10); err != nil { t.Fatalf("starting second main: %v", err) } defer m1.Close() @@ -468,8 +445,6 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() - seed := "" - // Create a client for each node. client0 := m0.Client() @@ -498,17 +473,15 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { // Configure node1 m1 := test.NewCommandNode(t) - m1.Config.Gossip.Seeds = []string{seed} if err := port.GetListeners(func(lsns []*net.TCPListener) error { portsCfg := test.GenPortsConfig(test.NewPorts(lsns)) - m1.Config.Gossip.Port = portsCfg[0].Gossip.Port m1.Config.Etcd = portsCfg[0].Etcd m1.Config.Name = portsCfg[0].Name m1.Config.Cluster.Name = portsCfg[0].Cluster.Name m1.Config.BindGRPC = portsCfg[0].BindGRPC return m1.Start() - }, 4, 10); err != nil { + }, 3, 10); err != nil { t.Fatalf("starting second main: %v", err) } errc := make(chan error, 1) @@ -536,8 +509,6 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() - seed := "" - // Create a client for each node. client0 := m0.Client() @@ -566,11 +537,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { // Configure node1 m1 := test.NewCommandNode(t) - m1.Config.Gossip.Seeds = []string{seed} if err := port.GetListeners(func(lsns []*net.TCPListener) error { portsCfg := test.GenPortsConfig(test.NewPorts(lsns)) - m1.Config.Gossip.Port = portsCfg[0].Gossip.Port m1.Config.Etcd = portsCfg[0].Etcd m1.Config.Name = portsCfg[0].Name m1.Config.Cluster.Name = portsCfg[0].Cluster.Name @@ -582,7 +551,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { errc <- err }() return m1.Start() - }, 4, 10); err != nil { + }, 3, 10); err != nil { t.Fatalf("starting second main: %v", err) } defer m1.Close() @@ -604,8 +573,6 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() - seed := "" - // Create a client for each node. client0 := m0.Client() @@ -632,11 +599,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { // Configure node1 m1 := test.NewCommandNode(t) - m1.Config.Gossip.Seeds = []string{seed} if err := port.GetListeners(func(lsns []*net.TCPListener) error { portsCfg := test.GenPortsConfig(test.NewPorts(lsns)) - m1.Config.Gossip.Port = portsCfg[0].Gossip.Port m1.Config.Etcd = portsCfg[0].Etcd m1.Config.Name = portsCfg[0].Name m1.Config.Cluster.Name = portsCfg[0].Cluster.Name @@ -648,7 +613,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { errc <- err }() return m1.Start() - }, 4, 10); err != nil { + }, 3, 10); err != nil { t.Fatalf("starting second main: %v", err) } @@ -664,74 +629,6 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { }) } -// Ensure that redundant gossip seeds are used -func TestCluster_GossipMembership(t *testing.T) { - t.Skip("skipping gossip test") - t.Run("Node0Down", func(t *testing.T) { - // Configure node0 - m0 := test.MustRunCluster(t, 1).GetNode(0) - defer m0.Close() - - seed := "" - - var eg errgroup.Group - - // Configure node1 - m1 := test.NewCommandNode(t) - defer m1.Close() - eg.Go(func() error { - // Pass invalid seed as first in list - m1.Config.Gossip.Seeds = []string{"http://localhost:8765", seed} - if err := port.GetPort(func(p int) error { - m1.Config.Gossip.Port = fmt.Sprintf("%d", p) - return m1.Start() - }, 10); err != nil { - t.Fatalf("starting second main: %v", err) - } - - return nil - }) - - // Configure node1 - m2 := test.NewCommandNode(t) - defer m2.Close() - eg.Go(func() error { - // Pass invalid seed as first in list - m2.Config.Gossip.Seeds = []string{seed, "http://localhost:8765"} - err := port.GetPort(func(p int) error { - m2.Config.Gossip.Port = fmt.Sprintf("%d", p) - return m2.Start() - }, 10) - - if err != nil { - t.Fatalf("starting second main: %v", err) - } - defer m2.Close() - return nil - }) - - if err := eg.Wait(); err != nil { - t.Fatal(err) - } - - state0, err0 := m0.API.State() - state1, err1 := m1.API.State() - state2, err2 := m2.API.State() - if err0 != nil || !test.CheckClusterState(m0, string(pilosa.ClusterStateNormal), 1000) { - t.Fatalf("unexpected node0 cluster state: %s, error: %v", state0, err0) - } else if err1 != nil || !test.CheckClusterState(m1, string(pilosa.ClusterStateNormal), 1000) { - t.Fatalf("unexpected node1 cluster state: %s, error: %v", state1, err1) - } else if err2 != nil || !test.CheckClusterState(m2, string(pilosa.ClusterStateNormal), 1000) { - t.Fatalf("unexpected node2 cluster state: %s, error: %v", state2, err2) - } - - numNodes := len(m0.API.Hosts(context.Background())) - if numNodes != 3 { - t.Fatalf("Expected 3 nodes, got %d", numNodes) - } - }) -} - func TestClusterResize_RemoveNode(t *testing.T) { cluster := test.MustRunCluster(t, 3) defer cluster.Close() diff --git a/server/handler_test.go b/server/handler_test.go index 460a4eff3..b0c6cf325 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -40,7 +40,6 @@ import ( pb "github.com/pilosa/pilosa/v2/proto" "github.com/pilosa/pilosa/v2/server" "github.com/pilosa/pilosa/v2/test" - "github.com/pilosa/pilosa/v2/test/port" ) func TestHandler_PostSchemaCluster(t *testing.T) { @@ -1405,10 +1404,7 @@ func TestCluster_TranslateStore(t *testing.T) { ), ) - if err := port.GetPort(func(p int) error { - cluster.GetIdleNode(0).Config.Gossip.Port = fmt.Sprintf("%d", p) - return cluster.GetIdleNode(0).Start() - }, 10); err != nil { + if err := cluster.GetIdleNode(0).Start(); err != nil { t.Fatalf("starting node 0: %v", err) } defer cluster.GetIdleNode(0).Close() diff --git a/server/server.go b/server/server.go index 00ab4d547..37b1b17c6 100644 --- a/server/server.go +++ b/server/server.go @@ -20,7 +20,6 @@ package server import ( - "bytes" "context" "crypto/tls" "io" @@ -47,7 +46,6 @@ import ( petcd "github.com/pilosa/pilosa/v2/etcd" "github.com/pilosa/pilosa/v2/gcnotify" "github.com/pilosa/pilosa/v2/gopsutil" - "github.com/pilosa/pilosa/v2/gossip" "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/logger" pnet "github.com/pilosa/pilosa/v2/net" @@ -72,10 +70,6 @@ type Command struct { // Configuration. Config *Config - // Gossip transport - gossipTransport *gossip.Transport - gossipMemberSet io.Closer - // Standard input/output *pilosa.CmdIO @@ -84,7 +78,6 @@ type Command struct { // done will be closed when Command.Close() is called done chan struct{} - // Passed to the Gossip implementation. logOutput io.Writer logger loggerLogger @@ -233,11 +226,6 @@ func (m *Command) UpAndDown() (err error) { return errors.Wrap(err, "setting up server") } - // SetupNetworking (so we'll have profiling) - err = m.setupNetworking() - if err != nil { - return errors.Wrap(err, "setting up networking") - } go func() { err := m.Handler.Serve() if err != nil { @@ -469,35 +457,6 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "new handler") } -// setupNetworking sets up internode communication based on the configuration. -func (m *Command) setupNetworking() error { - gossipPort, err := strconv.Atoi(m.Config.Gossip.Port) - if err != nil { - return errors.Wrap(err, "parsing port") - } - - // get the host portion of addr to use for binding - gossipHost := m.listenURI.Host - m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger()) - if err != nil { - return errors.Wrap(err, "getting transport") - } - - gossipMemberSet, err := gossip.NewMemberSet( - m.Config.Gossip, - m.API, - gossip.WithLogOutput(&filteredWriter{logOutput: m.logOutput, v: m.Config.Verbose}), - gossip.WithPilosaLogger(m.logger), - gossip.WithTransport(m.gossipTransport), - ) - if err != nil { - return errors.Wrap(err, "getting memberset") - } - m.gossipMemberSet = gossipMemberSet - - return errors.Wrap(gossipMemberSet.Open(), "opening gossip memberset") -} - // setupLogger sets up the logger based on the configuration. func (m *Command) setupLogger() error { var f *logger.FileWriter @@ -539,13 +498,6 @@ func (m *Command) setupLogger() error { return nil } -// GossipTransport allows a caller to return the gossip transport created when -// setting up the GossipMemberSet. This is useful if one needs to determine the -// allocated ephemeral port programmatically. (usually used in tests) -func (m *Command) GossipTransport() *gossip.Transport { - return m.gossipTransport -} - // Close shuts down the server. func (m *Command) Close() error { select { @@ -558,9 +510,6 @@ func (m *Command) Close() error { eg.Go(m.Server.Close) eg.Go(m.API.Close) eg.Go(m.pgserver.Close) - if m.gossipMemberSet != nil { - eg.Go(m.gossipMemberSet.Close) - } if closer, ok := m.logOutput.(io.Closer); ok { // If closer is os.Stdout or os.Stderr, don't close it. if closer != os.Stdout && closer != os.Stderr { @@ -617,27 +566,6 @@ func getListener(uri pnet.URI, tlsconf *tls.Config) (ln net.Listener, err error) return ln, nil } -type filteredWriter struct { - v bool - logOutput io.Writer -} - -// Write forwards the write to logOutput if verbose is true, or it doesn't -// contain [DEBUG] or [INFO]. This implementation isn't technically correct -// since Write could be called with only part of a log line, but I don't think -// that actually happens, so until it becomes a problem, I don't think it's -// worth dealing with the extra complexity. (jaffee) -func (f *filteredWriter) Write(p []byte) (n int, err error) { - if bytes.Contains(p, []byte("[DEBUG]")) || bytes.Contains(p, []byte("[INFO]")) { - if f.v { - return f.logOutput.Write(p) - } - } else { - return f.logOutput.Write(p) - } - return len(p), nil -} - // ParseConfig parses s into a Config. func ParseConfig(s string) (Config, error) { var c Config diff --git a/server/server_test.go b/server/server_test.go index 64f92ecac..ea1b08e70 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -26,7 +26,6 @@ import ( "os" "reflect" "sort" - "strconv" "strings" "testing" "time" @@ -977,8 +976,6 @@ func TestClusterQueriesAfterRestart(t *testing.T) { config := cmd1.Command.Config config.Bind = cmd1.API.Node().URI.HostPort() - // this isn't necessary, but makes the test run way faster - config.Gossip.Port = strconv.Itoa(int(cmd1.Command.GossipTransport().URI.Port)) cmd1.Command = server.NewCommand(cmd1.Stdin, cmd1.Stdout, cmd1.Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore))) cmd1.Command.Config = config err = cmd1.Start() diff --git a/test/cluster.go b/test/cluster.go index 74b2d0988..26e4a4985 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -401,22 +401,6 @@ func (c *Cluster) Start() error { }() portsCfg := GenPortsConfig(sliceOfPorts) - var gossipSeeds []string - for i, cc := range c.Nodes { - i := i - // get the bind uri to use as the host portion of the gossip seed. - uri, err := pilosa.AddressWithDefaults(cc.Config.Bind) - if err != nil { - return errors.Wrap(err, "processing bind address") - } - - cc.Config.Gossip.Port = portsCfg[i].Gossip.Port - gossipHost := uri.Host - gossipPort := cc.Config.Gossip.Port - - gossipSeeds = append(gossipSeeds, fmt.Sprintf("%s:%s", gossipHost, gossipPort)) - } - for i, cc := range c.Nodes { cc := cc cc.Config.Etcd = portsCfg[i].Etcd @@ -425,14 +409,12 @@ func (c *Cluster) Start() error { cc.Config.BindGRPC = portsCfg[i].BindGRPC eg.Go(func() error { - cc.Config.Gossip.Seeds = gossipSeeds - return cc.Start() }) } return eg.Wait() - }, 4*len(c.Nodes), 10) + }, 3*len(c.Nodes), 10) if err != nil { return err diff --git a/test/disco.go b/test/disco.go index 46328a24e..a903774c1 100644 --- a/test/disco.go +++ b/test/disco.go @@ -22,7 +22,6 @@ import ( "time" "github.com/pilosa/pilosa/v2/etcd" - "github.com/pilosa/pilosa/v2/gossip" "github.com/pilosa/pilosa/v2/server" ) @@ -33,8 +32,7 @@ type Ports struct { LsnP *net.TCPListener PortP int - Grpc int - Gossip int //TODO remove + Grpc int } func (ports *Ports) Close() error { @@ -65,10 +63,7 @@ func GenPortsConfig(ports []Ports) []*server.Config { } cfgs[i] = &server.Config{ - Name: name, - Gossip: gossip.Config{ - Port: fmt.Sprint(ports[i].Gossip), - }, + Name: name, BindGRPC: fmt.Sprintf(":%d", ports[i].Grpc), Etcd: etcd.Options{ Dir: discoDir, @@ -101,20 +96,18 @@ func NewPorts(lsn []*net.TCPListener) []Ports { ports[i] = lsn[i].Addr().(*net.TCPAddr).Port } - for i := 0; i < n; i = i + 4 { + for i := 0; i < n; i = i + 3 { out = append(out, Ports{ LsnC: lsn[i], PortC: ports[i], LsnP: lsn[i+1], PortP: ports[i+1], - Grpc: ports[i+2], - Gossip: ports[i+3], + Grpc: ports[i+2], }) - // make Grpc and Gossip ports available to + // make Grpc port available to // be rebound. lsn[i+2].Close() - lsn[i+3].Close() } return out diff --git a/translator_test.go b/translator_test.go index ddaff794b..518da91ff 100644 --- a/translator_test.go +++ b/translator_test.go @@ -263,17 +263,12 @@ func TestTranslation_Reset(t *testing.T) { if err := node0.SoftOpen(); err != nil { t.Fatal(err) } - gossipSeeds := []string{} - - node1.Config.Gossip.Seeds = gossipSeeds if err := node1.SoftOpen(); err != nil { t.Fatal(err) } - node2.Config.Gossip.Seeds = gossipSeeds if err := node2.SoftOpen(); err != nil { t.Fatal(err) } - node3.Config.Gossip.Seeds = gossipSeeds if err := node3.SoftOpen(); err != nil { t.Fatal(err) } From c8c59b649d3b66ebe545c60d3edafc6b6d68ba95 Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 4 Feb 2021 21:34:15 -0600 Subject: [PATCH 2/2] remove pilosa-fsck --- cmd/pilosa-fsck/Makefile | 36 - cmd/pilosa-fsck/fsck.go | 989 ------------------ cmd/pilosa-fsck/fsck_test.go | 448 -------- .../release-pilosa-fsck/.gitignore | 1 - cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md | 252 ----- .../release-pilosa-fsck/backups.tar.gz | Bin 248477 -> 0 bytes .../release-pilosa-fsck/example.sh | 21 - cmd/pilosa-fsck/vprint.go | 177 ---- 8 files changed, 1924 deletions(-) delete mode 100644 cmd/pilosa-fsck/Makefile delete mode 100644 cmd/pilosa-fsck/fsck.go delete mode 100644 cmd/pilosa-fsck/fsck_test.go delete mode 100644 cmd/pilosa-fsck/release-pilosa-fsck/.gitignore delete mode 100644 cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md delete mode 100644 cmd/pilosa-fsck/release-pilosa-fsck/backups.tar.gz delete mode 100755 cmd/pilosa-fsck/release-pilosa-fsck/example.sh delete mode 100644 cmd/pilosa-fsck/vprint.go diff --git a/cmd/pilosa-fsck/Makefile b/cmd/pilosa-fsck/Makefile deleted file mode 100644 index 1b1dcf14c..000000000 --- a/cmd/pilosa-fsck/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -.PHONY: install build release - -CLONE_URL=github.com/pilosa/pilosa -VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) -LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null) -VARIANT = Molecula -VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH) -BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))) -BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) -BUILD_TIME := $(shell date -u +%FT%T%z) -SHARD_WIDTH = 20 -COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) -LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT)" -GOOS = $(shell go env GOOS) - -# Install pilosa-fsck -install: - go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) - -# Compile pilosa-fsck -build: - go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) - -REL = release-pilosa-fsck.$(COMMIT).$(GOOS) - -release: - mkdir $(REL) - cd release-pilosa-fsck; tar cf - . |(cd ../$(REL); tar xf - ) - go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) -o $(REL)/pilosa-fsck - tar cf - $(REL) | gzip > $(REL).tar.gz - rm -rf $(REL) - mv $(REL).tar.gz ../.. - -clean: - find . -name pilosa-fsck | xargs rm -f - rm -f release-pilosa-fsck*.tar.gz diff --git a/cmd/pilosa-fsck/fsck.go b/cmd/pilosa-fsck/fsck.go deleted file mode 100644 index fc98fe574..000000000 --- a/cmd/pilosa-fsck/fsck.go +++ /dev/null @@ -1,989 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "fmt" - "io" - "io/ioutil" - "log" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "time" - - "github.com/dustin/go-humanize" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/boltdb" - "github.com/pilosa/pilosa/v2/internal" - "github.com/pilosa/pilosa/v2/server" - "github.com/pilosa/pilosa/v2/topology" - "github.com/pkg/errors" - "github.com/zeebo/blake3" -) - -// pilosa-fsck : -// an external customer tool (originally for Q2) to do 2 jobs: -// Given a set of cluster backups (and their .id and .topology files) -// mounted on the same file system, we can: -// 1) scan for fragment differences between the primary and its replicas (default); or -// 2) repair those differences by overwriting the replcas with the primary fragments (if -fix is given). -// -// pilosa-chk is deliberately NOT a part of pilosa so that it can run without -// forcing a customer to upgrade or downgrade their installed version. - -// FsckConfig configures the dumpcols() and/or read() runs. -type FsckConfig struct { - Fix bool // -fix - FixCol bool // -fixcol - - Colkeydump bool // -col - JustThisIndex string // -index - - // -col column key dump only options: - // Dir string - // PartitionID int - // ShowHeader bool - // ShowKey bool - // ShowID bool - - // not flags, just the Args() left after all other flags. Should be the list - // of pilosa (holder) directories for the cluster. - Dirs []string - - Verbose bool // -v - Quiet bool // -q - - // manual workaround for not having PilosaConfigPath, if really need be. - ReplicaN int // -replicas - PilosaConfigPath string // -config - - ParallelReaders int // -readers - - topo *pilosa.Topology -} - -// call DefineFlags before myflags.Parse() -func (cfg *FsckConfig) DefineFlags(fs *flag.FlagSet) { - fs.BoolVar(&cfg.Fix, "fix", false, "(warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. Implies -fixcol") - fs.BoolVar(&cfg.FixCol, "fixcol", false, "(warning: alters the backed-up node images on disk) repair string key translation tables. Skip repair of index data.") - //fs.BoolVar(&cfg.Verbose, "v", false, "be very verbose during analysis") - fs.BoolVar(&cfg.Quiet, "q", false, "be very quiet") - - fs.IntVar(&cfg.ReplicaN, "replicas", 0, "(required) manually entered replicaN; the number of replicas maintained in the cluster. Must be the same as the [cluster] 'replicas = R' entry in the pilosa.conf file for the cluster.") - - fs.IntVar(&cfg.ParallelReaders, "readers", 10, "how many parallel readers to use to scan at once. 0 means do everything possible in parallel. 1 means serialize everything through a single reader. Can be adjusted to control memory consumption.") - - fs.StringVar(&cfg.PilosaConfigPath, "config", "", "(required: -replicas or -config, with -config preferred) path to the pilosa.conf for the cluster (e.g. /etc/pilosa.conf)") - - fs.StringVar(&cfg.JustThisIndex, "index", "", "(optional) restrict to just this index. Otherwise we default to all indexes.") - - fs.Usage = func() { - fmt.Fprintf(os.Stderr, "pilosa-fsck version: %v\n\n", pilosa.VersionInfo()) - fmt.Fprintf(os.Stderr, `Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa - - -fix - (warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. - - -replicas R - (required) R is a positive integer, giving the replicaN or replicator factor for the cluster. This is - the number of replicas maintained in the cluster. Must be the same as the - [cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node. - - -index index_name - (optional) restrict to just this index. Otherwise we default to all indexes. - - -readers PR - how many parallel readers to use to scan at once. PR==0 means do everything - possible in parallel. PR==1 means serialize everything through a single reader. - Adjust PR to control memory consumption if needed. As a practical limit, setting - PR > 10000 will have no effect. (default is 10). - - -q - be very quiet during analysis and repair - -`) - fmt.Fprintf(os.Stderr, ` -Welcome to pilosa-fsck. This is a scan and repair -tool that is modeled after the classic unix file -system utility fsck. - -WARNING: DO NOT RUN ON A LIVE SYSTEM. - -The most important point to remember is that analysis -and repair must be done *offline*. - -Just as fsck must be run on an unmounted disk, -pilosa-fsck must be run on a backup. It must -not be run on the directories where a live Pilosa system -is serving queries. Instead, take a backup first. -A backup is a set of N Pilosa data directories that have been -copied from your live system. They must all -be visible and mounted on one filesystem together. - -pilosa-fsck can be run in scan-mode (without -fix), -or in repair-mode with -fix. The console output -supplies a log documenting the analysis -and showing what data changes would have been made. - -REQUIRED COMMAND LINE ARGUMENTS - -The paths to all the top-level Pilosa -data directories in a cluster must be given on the command -line. The -replicas R flag is also always required. It -must be correct for your cluser. Here R is the same as -the [cluster] stanza "replicas = R" line from your -pilosa.conf. - -Example: - -Suppose you are ready to run pilosa-fsck: -you have taken a backup of your four node Pilosa -cluster and stored it all on one filesystem with -all nodes visible and uncompressed. This -is a pre-requisite to running pilosa-fsck. -Let's suppose we have replication R = 3 set. -In this example, have stored our backed-up directories in - -/backup/molecula - -and the four node backups are in -subdirectories node1/ node2/ node3/ node4/ under this: - -/backup/molecula/node1/ -/backup/molecula/node1/.pilosa/.id -/backup/molecula/node1/.pilosa/.topology -/backup/molecula/node1/.pilosa/myindex - -/backup/molecula/node2/ -/backup/molecula/node2/.pilosa/.id -/backup/molecula/node2/.pilosa/.topology -/backup/molecula/node2/.pilosa/myindex - -/backup/molecula/node3/ -/backup/molecula/node3/.pilosa/.id -/backup/molecula/node3/.pilosa/.topology -/backup/molecula/node3/.pilosa/myindex - -/backup/molecula/node4/ -/backup/molecula/node4/.pilosa/.id -/backup/molecula/node4/.pilosa/.topology -/backup/molecula/node4/.pilosa/myindex - -NOTE: your .pilosa directories need not be named .pilosa. They can -be something else, such as when the -d flag to pilosa server was used. -The .id file, the .topology file, and the index directories must be -found directly underneath. - -Then a typical invocation to scan a cluster backup for issues: - -$ cd /backup/molecula/ -$ pilosa-fsck -replicas 3 node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log - -A typical invocation to repair the replication in the same backup: - -$ pilosa-fsck -replicas 3 -fix node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log - -In both cases, the .id and .topology files must -be present in the backups. - -Without -fix, no modifications will be made to the backups. Only -by running with -fix will repairs be made. The user can safely -always run with -fix to repair only if needed. - -A zero error code will be returned to the shell if no repairs were needed. - -A zero error code will be also be returned to the shell if -repairs were needed and they were accomplished under -fix. - -A non-zero error code indicates that repairs were needed but -were not made. -`) - } -} - -// call c.ValidateConfig() after myflags.Parse() -func (c *FsckConfig) ValidateConfig() error { - if c.Fix { - c.FixCol = true - } - if c.ReplicaN == 0 && c.PilosaConfigPath == "" { - return fmt.Errorf("must supply -replicas with the replica count from your pilosa.conf (positive integer count)") - } - - if c.ReplicaN == 0 && c.PilosaConfigPath != "" { - - if !FileExists(c.PilosaConfigPath) { - return fmt.Errorf(" -config path '%v' does not exist", c.PilosaConfigPath) - } - by, err := ioutil.ReadFile(c.PilosaConfigPath) - if err != nil { - return fmt.Errorf("error: could not read the -config path '%v': '%v'", c.PilosaConfigPath, err) - } - srvcfg, err := server.ParseConfig(string(by)) - if err != nil { - //vv("warning: -config path '%v' problem, could not parse toml: '%v'", c.PilosaConfigPath, err) - - // fall back to manual parsing of config - lines := strings.Split(string(by), "\n") - clusterStart := -1 - for i, line := range lines { - if strings.Contains(line, `[cluster]`) { - clusterStart = i - } - if i > clusterStart { - if strings.Contains(line, "replicas") { - split := strings.Split(line, "=") - ns := strings.TrimSpace(split[1]) - n, err := strconv.Atoi(ns) - if err != nil { - return fmt.Errorf("error: could not parse the replicaN from line %v in -config path '%v' (%v): '%v'", i+1, c.PilosaConfigPath, line, err) - } - c.ReplicaN = n - } - } - } - } else { - c.ReplicaN = srvcfg.Cluster.ReplicaN - } - if c.ReplicaN == 0 { - return fmt.Errorf("error: -config path '%v' did not list the Replica count: cannot be 0. See the [cluster] section, the 'replicas = R' line.", c.PilosaConfigPath) - } - //vv("c.ReplicaN = %v", c.ReplicaN) - } - return nil -} - -var ProgramName = "pilosa-fsck" - -func main() { - - myflags := flag.NewFlagSet(ProgramName, flag.ContinueOnError) - cfg := &FsckConfig{} - cfg.DefineFlags(myflags) - cfg.Verbose = true - - err := myflags.Parse(os.Args[1:]) - if err != nil { - fmt.Fprintf(os.Stderr, "\n%v\n", err.Error()) - os.Exit(1) - } - err = cfg.ValidateConfig() - if err != nil { - fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err) - os.Exit(1) - } - dirs := myflags.Args() - nDir := len(dirs) - if nDir <= 0 && !cfg.Colkeydump { - fmt.Fprintf(os.Stderr, "error: %v command line arguments missing error: provide all of the top-level pilosa directories for the cluster as command line arguments.\n", ProgramName) - os.Exit(1) - } - - cmdline := strings.Join(os.Args, " ") - - // make sure all the dir are distinct - dup := make(map[string]bool) - for _, dir := range dirs { - if dup[dir] { - fmt.Fprintf(os.Stderr, "%v error: duplicate data directory '%v' given in command line '%v'. Each backup directory must be distinct.\n", ProgramName, dir, cmdline) - os.Exit(1) - } else { - dup[dir] = true - } - } - - fmt.Fprintf(os.Stdout, "#!/bin/bash\n\n# pilosa-fsck version: %v\n", pilosa.VersionInfo()) - cwd, err := os.Getwd() - if err != nil { - fmt.Fprintf(os.Stderr, "error: could not read current dir: '%v'\n", err) - os.Exit(1) - } - fmt.Fprintf(os.Stdout, "# cwd: %v\n", cwd) - fmt.Fprintf(os.Stdout, "# command line: %v\n", cmdline) - t0 := time.Now() - fmt.Fprintf(os.Stdout, "# started at %v\n\n", t0.Format(RFC3339MsecTz0)) - defer func() { - fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0)) - }() - cfg.Dirs = dirs - - fixNeeded, err := cfg.Run() - if err != nil { - fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0)) - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if fixNeeded && !cfg.Fix { - fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0)) - fmt.Fprintf(os.Stderr, "# pilosa-fsck exiting with non-zero error code because a repair is needed, but -fix was not given.\n") - os.Exit(1) - } -} - -func (cfg *FsckConfig) Run() (fixNeeded bool, err error) { - - // if cfg.Colkeydump { - // cfg.dumpcols() - //} - - perNodeIndexMaps, clusterNodes, ats, err := cfg.read() - if err != nil { - return false, err - } - - if cfg.FixCol { - err := cfg.RepairTranslationStores(ats) - if err != nil { - return false, fmt.Errorf("error fixing key translation stores with cfg.RepairTranslationStores(): '%v'\n", err) - } - } - - //vv("perNodeIndexMaps='%#v', clusterNodes='%#v'", perNodeIndexMaps, clusterNodes) - - fixme, reports, err := cfg.analyze(clusterNodes, perNodeIndexMaps, ats) - if err != nil { - return false, fmt.Errorf("error in FsckConfig.analyze(): '%v'", err) - } - fixNeeded = ats.RepairNeeded || fixme - for _, report := range reports { - fmt.Printf("%v\n", report) - } - if len(reports) == 0 { - fmt.Fprintf(os.Stderr, "pilosa-fsck: no index found to analyze. cmdline was: %v\n", strings.Join(os.Args, " ")) - } - return -} - -var _ = (&FsckConfig{}).dumpAts - -func (cfg *FsckConfig) dumpAts(ats *pilosa.AllTranslatorSummary) { - fmt.Printf("# dumpAts: RepairNeeded=%v\n", ats.RepairNeeded) - for _, sum := range ats.Sums { - fmt.Printf("# sum = '%#v'\n", sum) - } - -} - -type group struct { - elem []*pilosa.TranslatorSummary - partitionID int -} - -func (g *group) String() (s string) { - for i, e := range g.elem { - s += fmt.Sprintf("partition %v, group elem [%v] out of %v: %v\n", g.partitionID, i, len(g.elem), e.String()) - } - return -} - -func indexesFromAts(ats *pilosa.AllTranslatorSummary) (indexes []string) { - indexMap := make(map[string]bool) - for _, sum := range ats.Sums { - if !indexMap[sum.Index] { - indexMap[sum.Index] = true - indexes = append(indexes, sum.Index) - } - } - sort.Strings(indexes) - return -} - -func (cfg *FsckConfig) RepairTranslationStores(ats *pilosa.AllTranslatorSummary) (err error) { - - verbose := cfg.Verbose - - // group by index first. then repair. - indexes := indexesFromAts(ats) - - for _, index := range indexes { - - if !cfg.DoingIndex(index) { - continue - } - - m := make(map[int]*group) - for _, sum := range ats.Sums { - - if !sum.IsColKey || sum.Index != index { - continue - } - grp := m[sum.PartitionID] - if grp == nil { - grp = &group{ - partitionID: sum.PartitionID, - } - m[sum.PartitionID] = grp - } - grp.elem = append(grp.elem, sum) - } - - for partitionID, group := range m { - _ = partitionID - prim := -1 - keyCount := 0 - for k, e := range group.elem { - if e.IsPrimary { - prim = k - } - keyCount += e.KeyCount - } - if prim == -1 { - panic(fmt.Sprintf("no primary found for group '%v'", group.String())) - } - - primary := group.elem[prim] - primaryChecksum := primary.Checksum - for _, e := range group.elem { - if e.IsPrimary { - continue - } - // is e a replica? not necessarily! have to check. - if !e.IsReplica { - //if verbose { - // since this will happen even on a fix point, where it is already empty, - // we don't report it again. - //fmt.Printf("# non-replica should have no data: creating an empty translation store here at '%v'\n", e.StorePath) - //} - err := os.RemoveAll(e.StorePath) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() os.RemoveAll(e.StorePath='%v')", e.StorePath)) - } - store, err := boltdb.OpenTranslateStore(e.StorePath, e.Index, e.Field, e.PartitionID, topology.DefaultPartitionN) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() create empty boldtdb: boltdb.OpenTranslateStore e.StorePath='%v'", e.StorePath)) - } - err = store.Close() - if err != nil { - return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() closing empty boltdb at path '%v'", e.StorePath)) - } - continue - } - // INVAR: e is a replica for this paritionID. - // Copy from primary if checksums are different. - if e.Checksum != primaryChecksum { - from := group.elem[prim].StorePath - dest := e.StorePath - if verbose { - fmt.Printf("# e.Checksum '%v' != primaryChecksum '%v': copying from primary translation store '%v' -> '%v'\n", e.Checksum, primaryChecksum, from, dest) - } - err := cp(from, dest) - if err != nil { - return fmt.Errorf("error: could not copy from primary '%v' to replica translation store '%v': '%v' ... try to keep going...\n", from, dest, err) - } - } - } - } - } - return nil -} - -/* -func (cfg *FsckConfig) dumpcols() { - - verbose := cfg.Verbose - quiet := cfg.Quiet - _, _ = verbose, quiet - - dir := cfg.Dir - index := cfg.Index - partitionID := cfg.PartitionID - showKey := cfg.ShowKey - showID := cfg.ShowID - - if !quiet { - fmt.Printf("# dumpcols: opening dir '%v'... this may take a few minutes...\n", dir) - } - holder := pilosa.NewHolder(dir, nil) - holder.OpenTranslateStore = boltdb.OpenTranslateStore - err := holder.Open() - if err != nil { - log.Fatal(err) - } - if cfg.ShowHeader { - fmt.Println("# columnKey columId") - } - id_key := make(map[uint64]string) - key_id := make(map[string]uint64) - for _, idx := range holder.Indexes() { - fmt.Printf("# Looking '%v'\n", idx.Name()) - if idx.Name() == index { - store := idx.TranslateStore(partitionID) - fmt.Printf("# Key By ID partitionID = %v\n", partitionID) - err := store.KeyWalker(func(key string, col uint64) { - key_id[key] = col - if showKey { - fmt.Printf("# '%v' %v shard: %v partition: %v\n", key, col, col/pilosa.ShardWidth, partitionID) - } - }) - panicOn(err) - } - } - for _, idx := range holder.Indexes() { - if idx.Name() == index { - store := idx.TranslateStore(partitionID) - //fmt.Printf("# ID ByKey\n") - err := store.IDWalker(func(key string, col uint64) { - id_key[col] = key - if showID { - fmt.Printf("# '%v' %v\n", key, col) - } - }) - panicOn(err) - } - } - fmt.Printf("# k: %d i: %d\n", len(key_id), len(id_key)) - fmt.Println("id_key") - for k, v := range id_key { - l, ok := key_id[v] - if ok { - if k != l { - fmt.Printf("# X: %v %v %v\n", k, l, v) - } - } else { - fmt.Printf("# key not in id %v\n", v) - } - } - fmt.Println("key_id") - for k, v := range key_id { - l, ok := id_key[v] - if ok { - if k != l { - fmt.Printf("# T: %v %v %v\n", k, l, v) - } - } else { - fmt.Printf("# id not in key %v\n", v) - } - } -} -*/ - -func (cfg *FsckConfig) read() (perNodeIndexMaps []map[string]*pilosa.IndexFragmentSummary, clusterNodes []string, final *pilosa.AllTranslatorSummary, err error) { - - final = pilosa.NewAllTranslatorSummary() - - dirs := cfg.Dirs - for _, dir := range dirs { - idx2frag, nodeID, atsNode, err := cfg.readOneDir(dir) - if err != nil { - return nil, nil, nil, err - } - final.Append(atsNode) - clusterNodes = append(clusterNodes, nodeID) - perNodeIndexMaps = append(perNodeIndexMaps, idx2frag) - } - return -} - -func (cfg *FsckConfig) readOneDir(dir string) (idx2frag map[string]*pilosa.IndexFragmentSummary, nodeID string, atsNode *pilosa.AllTranslatorSummary, err error) { - - verbose := cfg.Verbose - quiet := cfg.Quiet - - if !quiet { - fmt.Printf("# opening dir '%v'... this may take a few minutes...\n\n", dir) - } - - jmphasher := &topology.Jmphasher{} - partitionN := topology.DefaultPartitionN - replicaN := cfg.ReplicaN - topo, err := loadTopology(dir, jmphasher, partitionN, replicaN) - if err != nil { - return nil, "", nil, err - } - cfg.topo = topo - //vv("topo = '%#v'", topo) - nodeIDs := topo.GetNodeIDs() - //vv("nodeIDs = '%#v'", nodeIDs) - nNodes := len(nodeIDs) - nDir := len(cfg.Dirs) - if nDir != nNodes { - return nil, "", nil, fmt.Errorf("command line had %v directories (%#v) but the .topology had %v nodes (%#v)", nDir, cfg.Dirs, nNodes, nodeIDs) - } - - holder := pilosa.NewHolder(dir, nil) - holder.OpenTranslateStore = boltdb.OpenTranslateStore - - nodeID, err = holder.LoadNodeID() - panicOn(err) - //vv("nodeID = '%v'", nodeID) - err = holder.Open() - - if err != nil { - log.Fatal(err) - } - - if !quiet { - fmt.Printf("\n# calculating hashes of row and column key translation maps on data from dir '%v'...\n", dir) - } - var indexes []*pilosa.Index - - const checkKeys = true - atsNode = pilosa.NewAllTranslatorSummary() - for _, idx := range holder.Indexes() { - - if !cfg.DoingIndex(idx.Name()) { - continue - } - - //vv("calling idx.ComputeTranslatorSummary(verbose, checkKeys=%v, cfg.FixCol='%v')", checkKeys, cfg.FixCol) - - asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, cfg.FixCol, topo, nodeID, cfg.ParallelReaders) - if err != nil { - log.Fatal(err) - } - atsNode.Append(asum) - indexes = append(indexes, idx) - } - atsNode.Sort() - - hasher := blake3.New() - if !quiet { - fmt.Printf("\n# summary of col/row translations in dir: %v:\n", dir) - } - for _, sum := range atsNode.Sums { - if !quiet { - fmt.Printf("# index: %v partitionID: %v blake3-%v keyCount: %v idCount: %v\n", sum.Index, sum.PartitionID, sum.Checksum, sum.KeyCount, sum.IDCount) - } - _, _ = hasher.Write([]byte(sum.Checksum)) - } - - var buf [16]byte - _, _ = hasher.Digest().Read(buf[0:]) - - if !quiet { - fmt.Printf("# all-checksum = blake3-%x\n", buf) - } - - // fragment analysis - - showBits := false - showOpsLog := false - idx2frag = make(map[string]*pilosa.IndexFragmentSummary) // on this node. - for _, idx := range indexes { - if verbose { - fmt.Printf("# ==============================\n") - fmt.Printf("# index: %v\n", idx.Name()) - fmt.Printf("# ==============================\n") - } - frgsum := idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog, topo, verbose) - frgsum.Dir = dir - frgsum.NodeID = nodeID - idx2frag[idx.Name()] = frgsum - } - - _ = holder.Close() - - //vv("idx2frag = '%v'", idx2frag) // tons of output. see 1234.out.full for examaple. - - return -} - -func (cfg *FsckConfig) DoingIndex(index string) bool { - if cfg.JustThisIndex == "" { - // scan all indexes - return true - } - if index == cfg.JustThisIndex { - // scan just this one - return true - } - return false -} - -// from cluster.go:1924 -func loadTopology(holderDir string, hasher topology.Hasher, partitionN, replicaN int) (*pilosa.Topology, error) { - - buf, err := ioutil.ReadFile(filepath.Join(holderDir, ".topology")) - if err != nil { - return nil, err - } - - var pb internal.Topology - err = proto.Unmarshal(buf, &pb) - if err != nil { - return nil, err - } - - return pilosa.DecodeTopology(&pb, hasher, partitionN, replicaN, nil) -} - -func (cfg *FsckConfig) analyze(clusterNodes []string, perNodeIndexMaps []map[string]*pilosa.IndexFragmentSummary, ats *pilosa.AllTranslatorSummary) (fixNeeded bool, reports []string, err error) { - - verbose := cfg.Verbose - quiet := cfg.Quiet - _, _ = verbose, quiet - - allIndex := make(map[string]bool) - for _, mp := range perNodeIndexMaps { - for index := range mp { - allIndex[index] = true - } - } - if !quiet { - vv("allIndex = '%#v'", allIndex) - } - for index := range allIndex { - if !quiet { - vv("on index '%v'", index) - } - nodes2fragsum := make(map[string]*pilosa.IndexFragmentSummary) - for _, mp := range perNodeIndexMaps { - sum := mp[index] - if sum == nil { - continue - } - nodes2fragsum[sum.NodeID] = sum - } - fixme, report, err := cfg.analyzeThisIndex(index, nodes2fragsum, ats) - if err != nil { - return false, reports, fmt.Errorf("error in analyze of index '%v': '%v'", index, err) - } - fixNeeded = fixNeeded || fixme - reports = append(reports, report) - } - return fixNeeded, reports, nil -} - -func (cfg *FsckConfig) analyzeThisIndex( - index string, - nodes2fragsum map[string]*pilosa.IndexFragmentSummary, - ats *pilosa.AllTranslatorSummary, -) (fixNeeded bool, report string, err error) { - - verbose := cfg.Verbose - quiet := cfg.Quiet - _, _ = verbose, quiet - - var removedBytes int64 - var copiedBytes int64 - var changedFiles int64 - var totalFiles int64 - var overwrittenBytes int64 - var totalBytes int64 - - if !quiet { - vv("top of analyzeThisIndex(index='%v'); len of nodes2fragsum = %v; nodes2fragsum='%#v'", - index, len(nodes2fragsum), nodes2fragsum) - } - - // Create a snapshot of the cluster to use for node/partition calculations. - snap := topology.NewClusterSnapshot(cfg.topo, cfg.topo.Hasher, cfg.topo.ReplicaN) - - for node, sum := range nodes2fragsum { - if !quiet { - fmt.Printf("# on node '%v'\n", node) - } - // do they disagree on who is the primary? - // for each fragment, do they disagree on the checksum? - - // Q: which nodes are supposed to have data, and which - // nodes are not supposed to have data? - - // loopFragSum: - for relpath, fragsum := range sum.RelPath2fsum { - fragsum.NodeID = node - totalFiles++ - //vv("checking %v on node %v", relpath, node) - - replicas, nonReplicas := snap.ReplicasForPrimary(fragsum.Primary) - _, _ = replicas, nonReplicas - //vv("replicas = '%#v'", replicas) - //vv("nonReplicas = '%#v'", nonReplicas) - - err := cfg.verifyReplicasAvailable(replicas, nonReplicas, nodes2fragsum, fragsum) - if err != nil { - return fixNeeded, "", err - } - - // find the primary's checksum - primaryChecksum := "" - var primaryFragSum *pilosa.FragSum - for node, isPrimary := range replicas { - if isPrimary { - primarySum := nodes2fragsum[node] - primaryFragSum = primarySum.RelPath2fsum[relpath] - if primaryFragSum == nil { - - // This seems clear indication that we have the topology wrong. - // When the topology is right, there are NO errors of this kind. - // - msg := fmt.Sprintf("# ugh. BAD. Stopping because any fix will be wrong. We see wrong -replica %v param, OR the .id files are mis-assigned with respect to the topology file. Could not find primary FragSum for relpath = '%v'. replicas = '%#v', nonReplicas = '%#v'\n", cfg.ReplicaN, relpath, replicas, nonReplicas) - vv(msg) - fmt.Fprintf(os.Stderr, "%v\n", msg) - panic(msg) // stop. the fixes are going to be wrong. - } else { - primaryChecksum = primaryFragSum.Checksum - primaryFragSum.NodeID = node - primaryFragSum.ScanDone = true - } - break - } - } - if primaryChecksum == "" { - return fixNeeded, "", fmt.Errorf("could not find primary replica??? replicas='%#v', nodes2fragsum='%v'; for fragsum='%#v'", replicas, nodes2fragsum, fragsum) - } - - // is this a non-replica? - _, isNon := nonReplicas[fragsum.NodeID] - if isNon { - removedBytes += FileSize(fragsum.AbsPath) - changedFiles++ - - //vv("yes, is nonReplica: fragsum.NodeID='%v'", fragsum.NodeID) - if !quiet { - fmt.Printf("rm %v #### REPAIR REMOVE data from non-replica at node '%v' (fragsum='%#v') vs. primary (%#v)\n\n", fragsum.AbsPath, node, fragsum, primaryFragSum) - } - if cfg.Fix { - err := os.Remove(fragsum.AbsPath) - if err != nil { - return fixNeeded, "", fmt.Errorf("error removing non-replica extra fragment '%v': '%v'", fragsum.AbsPath, err) - } - } - } else { - presz := FileSize(fragsum.AbsPath) - totalBytes += presz - - checksum := fragsum.Checksum - if checksum != primaryChecksum { - copiedBytes += FileSize(primaryFragSum.AbsPath) - changedFiles++ - overwrittenBytes += presz - - if !quiet { - fmt.Printf("cp %v %v #### REPAIR OVERWRITE replica at node '%v' (%#v) from primary '%v' (%#v)\n", primaryFragSum.AbsPath, fragsum.AbsPath, node, fragsum, primaryFragSum.NodeID, primaryFragSum) - } - if cfg.Fix { - err := cp(primaryFragSum.AbsPath, fragsum.AbsPath) - if err != nil { - return fixNeeded, "", fmt.Errorf("error copying from '%v' to '%v': '%v'", - primaryFragSum.AbsPath, fragsum.AbsPath, err) - } - } - } - } - fragsum.ScanDone = true - } - } - nDir := len(nodes2fragsum) - - keyCount, idCount := cfg.getKeyIDCounts(index, ats) - - fixNeeded = changedFiles > 0 || ats.RepairNeeded - var actionTaken string - var wouldBe string - if cfg.Fix || cfg.FixCol { - if fixNeeded { - actionTaken = "*REPAIRS WERE MADE TO THE BACKUPS*" - wouldBe = "sync repairs made:" - } else { - wouldBe = "" - actionTaken = "NO REPAIR NEEDED." - } - } else { - if fixNeeded { - wouldBe = "sync actions that would be taken under -fix:" - actionTaken = "*REPAIRS NEEDED BUT WERE NOT APPLIED* ; pilosa-fsck -fix was omitted." - } else { - wouldBe = "" - actionTaken = "NO REPAIR NEEDED." - } - } - var fragUpdate string - if changedFiles > 0 { - fragUpdate = fmt.Sprintf(` -# %v -# copied bytes: %v -# file bytes overwritten: %v -# new bytes added: %v -# new bytes is %0.01f%% of %v total bytes -# removed %v bytes from non-replicas -# changed file count %v (%0.01f%%; total files=%v) -# -`, wouldBe, humanize.Comma(copiedBytes), humanize.Comma(overwrittenBytes), humanize.Comma(copiedBytes-overwrittenBytes), 100*float64(copiedBytes-overwrittenBytes)/float64(totalBytes), humanize.Comma(totalBytes), humanize.Comma(removedBytes), changedFiles, 100*float64(changedFiles)/float64(totalFiles), humanize.Comma(totalFiles)) - } - - report = fmt.Sprintf(` -# ======================================================== -# pilosa-fsck final report -# -# run with -fix: %v -# -# index examined: '%v' -# -# nodes examined: %v -# -replicas %v replication factor used -# -# feature data examined: %v bytes -# feature files examined: %v files -# -# key-translation-stores examined: %v -# key-count: %v over all replicas -# id-count: %v over all replicas -# -# %v -# %v -# ======================================================== -`, - cfg.Fix, index, nDir, cfg.ReplicaN, humanize.Comma(totalBytes), humanize.Comma(totalFiles), humanize.Comma(int64(nDir*topology.DefaultPartitionN)), humanize.Comma(int64(keyCount)), humanize.Comma(int64(idCount)), actionTaken, fragUpdate) - return -} - -func (cfg *FsckConfig) verifyReplicasAvailable(replicas, nonReplicas map[string]bool, nodes2fragsum map[string]*pilosa.IndexFragmentSummary, fragsum *pilosa.FragSum) error { - for node := range replicas { - if nodes2fragsum[node] == nil { - return fmt.Errorf("error: node '%v' needed for a replica set was not availabe. Did you give ALL the directories for your cluster on the command line at once? In nodes2fragsum '%#v' (replicas: '%#v'; non-replicas '%#v') for fragsum '%v'", node, nodes2fragsum, replicas, nonReplicas, fragsum) - } - } - return nil -} - -func cp(fromPath, toPath string) (err error) { - tmpTo := toPath + ".fsck.tmp" - toFd, err := os.Create(tmpTo) - if err != nil { - return err - } - defer toFd.Close() - fromFd, err := os.Open(fromPath) - if err != nil { - return err - } - defer fromFd.Close() - - _, err = io.Copy(toFd, fromFd) - if err != nil { - return err - } - err = toFd.Close() - if err != nil { - return err - } - return os.Rename(tmpTo, toPath) -} - -func (cfg *FsckConfig) getKeyIDCounts(index string, ats *pilosa.AllTranslatorSummary) (keyCount, idCount int) { - for _, sum := range ats.Sums { - if sum.Index == index { - keyCount += sum.KeyCount - idCount += sum.IDCount - } - } - return -} diff --git a/cmd/pilosa-fsck/fsck_test.go b/cmd/pilosa-fsck/fsck_test.go deleted file mode 100644 index 2555215cc..000000000 --- a/cmd/pilosa-fsck/fsck_test.go +++ /dev/null @@ -1,448 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "fmt" - "io/ioutil" - "reflect" - "strconv" - "testing" - "time" - - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/boltdb" - "github.com/pilosa/pilosa/v2/hash" - "github.com/pilosa/pilosa/v2/http" - "github.com/pilosa/pilosa/v2/server" - "github.com/pilosa/pilosa/v2/test" -) - -func Test_Repair(t *testing.T) { - t.Skip("I don't quite understand what this test is doing and will need help adjusting it to pass again.") - // a) setup 1 primary + 3 replicas of disagree-ing cluster dirs. - - nNodes := 4 - nReplicas := 3 - - name := t.Name() - var nodeid []string - for i := 0; i < nNodes; i++ { - // work around a bug in the test.MustRunCluster that corrupts - // the .topology file if we only join name with one "_" underscore. - nodeid = append(nodeid, name+"__"+strconv.Itoa(i)) - } - - c := test.MustRunCluster(t, nNodes, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerNodeID(nodeid[0]), - pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), - pilosa.OptServerReplicaN(nReplicas), - )}, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerNodeID(nodeid[1]), - pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), - pilosa.OptServerReplicaN(nReplicas), - )}, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerNodeID(nodeid[2]), - pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), - pilosa.OptServerReplicaN(nReplicas), - )}, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerNodeID(nodeid[3]), - pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), - pilosa.OptServerReplicaN(nReplicas), - )}, - ) - // note: do not defer c.Close() here. We manually close below. - - var nodes []*test.Command - var dirs []string - for i := 0; i < nNodes; i++ { - nd := c.GetNode(i) - nodes = append(nodes, nd) - dirs = append(dirs, nd.Server.Holder().Path()) - } - - ctx := context.Background() - - index := []string{"rick", "morty"} - fieldName := []string{"f", "flying_car"} - idx := make([]*pilosa.Index, len(index)) - field := make([]*pilosa.Field, len(index)) - var err error - - for i := range index { - - idx[i], err = nodes[0].API.CreateIndex(ctx, index[i], pilosa.IndexOptions{Keys: true, TrackExistence: true}) - if err != nil { - t.Fatalf("creating index: %v", err) - } - if idx[i].CreatedAt() == 0 { - t.Fatal("index createdAt is empty") - } - - field[i], err = nodes[0].API.CreateField(ctx, index[i], fieldName[i], pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) - if err != nil { - t.Fatalf("creating field: %v", err) - } - if field[i].CreatedAt() == 0 { - t.Fatal("field createdAt is empty") - } - } - - rowID := uint64(1) - timestamp := int64(0) - - for i := range index { - - // Generate some keyed records. - rowIDs := []uint64{} - timestamps := []int64{} - N := 10 - for j := 1; j <= N; j++ { - rowIDs = append(rowIDs, rowID) - timestamps = append(timestamps, timestamp) - } - - var colKeys []string - switch i { - case 0: - // Keys are sharded so ordering is not guaranteed. - colKeys = []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"} - colKeys = colKeys[:N] - case 1: - colKeys = []string{"col11", "col12"} - N = len(colKeys) - rowIDs = rowIDs[:N] - timestamps = timestamps[:N] - } - - // Import data with keys to the coordinator (node0) and verify that it gets - // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) - req := &pilosa.ImportRequest{ - Index: index[i], - IndexCreatedAt: idx[i].CreatedAt(), - Field: fieldName[i], - FieldCreatedAt: field[i].CreatedAt(), - - // even though this says Shard: 0, that won't matter. The column keys - // get hashed and that decides the actual shard. - Shard: 0, - RowIDs: rowIDs, - ColumnKeys: colKeys, - Timestamps: timestamps, - } - - qcx := nodes[0].API.Txf().NewQcx() - - if err := nodes[0].API.Import(ctx, qcx, req); err != nil { - t.Fatal(err) - } - panicOn(qcx.Finish()) - //qcx.Reset() - - pql := fmt.Sprintf("Row(%s=%d)", fieldName[i], rowID) - - // Query node0. - if res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index[i], Query: pql}); err != nil { - t.Fatal(err) - } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { - t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys) - } - - // Query node1. - if err := test.RetryUntil(5*time.Second, func() error { - if res, err := nodes[1].API.Query(ctx, &pilosa.QueryRequest{Index: index[i], Query: pql}); err != nil { - return err - } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { - return fmt.Errorf("unexpected column keys: %#v", keys) - } - return nil - }); err != nil { - t.Fatal(err) - } - } - // end of setup. - - // partitionID in use: 6, 31, 57, 133, 185, 235 - targetPartition := 31 // which partitionID we mess with. - targetNode := nodes[0] // this is the first replica. - targetIndex := index[0] - // 0 first replica - // 1 second replica - // 2 -- not a replica - // 3 primary - - cfg := &FsckConfig{ - Fix: false, - FixCol: false, - Quiet: true, - //Verbose: true, - ReplicaN: nReplicas, - Dirs: dirs, - ParallelReaders: 5, - } - panicOn(cfg.ValidateConfig()) - - // for this test, mess up a replica that is not the primary. - - h := targetNode.API.Holder() - idx[0] = h.Index(index[0]) - store := idx[0].TranslateStore(targetPartition) - fwd, rev := getFwdRev(store, targetPartition) - //vv("targetPartition=%v, store.PartitionID=%v, before corruption, fwd='%#v', rev='%#v'", targetPartition, store.PartitionID, fwd, rev) - - // # fsck_test.go:288 2020-10-01T13:39:57.718995-05:00 partition 31, key 'col5' -> db00001 - presz := len(rev) - delete(rev, fwd["col5"]) - postsz := len(rev) - - if postsz == presz { - panic("did not delete any key!") - } - - bolt := store.(*boltdb.TranslateStore) - //vv("pre corruption, bolt = '%v'", fileChecksum(bolt.Path)) - //bolt.DumpBolt("pre-corruption") - - if err := bolt.SetFwdRevMaps(nil, fwd, rev); err != nil { - t.Fatal(err) - } - //vv("post corruption, bolt = '%v'", fileChecksum(bolt.Path)) - //bolt.DumpBolt("post-corruption") - - //fwd3, rev3 := getFwdRev(store, targetPartition) - //vv("after corruption, fwd='%#v', rev='%#v'", fwd3, rev3) - - targetIndex1 := "morty" - targetPartition1 := 226 // for "col11" - // # fsck_test.go:248 2020-10-06T20:24:33.755576-05:00 on k=47, idx[1]: targetPartition=47, store.PartitionID=0x4abe160, before corruption, fwd1='map[string]uint64{"col12":0xcf00001}', rev1='map[uint64]string{0xcf00001:"col12"}' - //# fsck_test.go:248 2020-10-06T20:24:35.608568-05:00 on k=226, idx[1]: targetPartition=226, store.PartitionID=0x4abe160, before corruption, fwd1='map[string]uint64{"col11":0xcc00001}', rev1='map[uint64]string{0xcc00001:"col11"}' - idx[1] = h.Index(index[1]) - store1 := idx[1].TranslateStore(targetPartition1) - fwd1, rev1 := getFwdRev(store1, targetPartition1) - //vv("on k=%v, idx[1]: targetPartition=%v, store.PartitionID=%v, before corruption, fwd1='%#v', rev1='%#v'", k, targetPartition1, store.PartitionID, fwd1, rev1) - - presz1 := len(rev1) - delete(rev1, fwd1["col11"]) - postsz1 := len(rev1) - - if postsz1 == presz1 { - panic("did not delete any key!") - } - bolt1 := store1.(*boltdb.TranslateStore) - if err := bolt1.SetFwdRevMaps(nil, fwd1, rev1); err != nil { - t.Fatal(err) - } - - // done corrupting. - for _, nd := range nodes { - nd.Command.Close() - } - //panicOn(bolt.Open()) - //bolt.DumpBolt("post-corruption, after Close. bolt:") - //bolt.Close() - - //chksums := getChecksums(dirs, cfg, targetPartition) - //vv("post corruption, pre repair chksums = '%#v'", chksums) - - // first we check that the corruption can be detected - // by our test with the checksums. - - chk, err := check(dirs, cfg, targetIndex, targetPartition) - _ = chk - //vv("pre-fix, chk='%v'; err='%v'", chk, err) - - if err == nil { - panic("expected to see checksums not match! but no corruption detected.") - } - - chk1, err := check(dirs, cfg, targetIndex1, targetPartition1) - _ = chk1 - //vv("pre-fix, chk1='%v'; err='%v'", chk1, err) - - if err == nil { - panic("expected to see checksums not match! but no corruption detected.") - } - - // b) running in reporting mode only should report that a fix is needed. - fixNeeded, err := cfg.Run() - panicOn(err) - if !fixNeeded { - panic("fix should be needed now, before repair") - } - - // c) run the fix. - cfg.Fix = true - cfg.FixCol = true - - fixNeeded, err = cfg.Run() - panicOn(err) - if !fixNeeded { - panic("fix should be marked needed if repair was made") - } - - // d) check that the replicas all look like the primary. - - //chksums = getChecksums(dirs, cfg, targetPartition) - //vv("after repair chksums = '%#v'", chksums) - - chk, err = check(dirs, cfg, targetIndex, targetPartition) - _ = chk - //vv("chk = '%v' after repair; err='%v'", chk, err) - panicOn(err) - - chk1, err = check(dirs, cfg, targetIndex1, targetPartition1) - _ = chk1 - //vv("chk = '%v' after repair; err='%v'", chk, err) - panicOn(err) - - // e) run again, should see no fix needed. - fixNeeded, err = cfg.Run() - panicOn(err) - if fixNeeded { - panic("should see no fix needed after the prior repair") - } -} - -func getFwdRev(store pilosa.TranslateStore, partitionID int) (fwd map[string]uint64, rev map[uint64]string) { - fwd = make(map[string]uint64) - rev = make(map[uint64]string) - _ = store.KeyWalker(func(key string, col uint64) { - //vv("partition %v, key '%v' -> %x", partitionID, key, col) - fwd[key] = col - }) - _ = store.IDWalker(func(key string, col uint64) { - //vv("partition %v, id %x -> '%v'", partitionID, col, key) - rev[col] = key - }) - return -} - -func check(dirs []string, cfg *FsckConfig, targetIndex string, targetPartition int) (chksum string, err error) { - //vv("top of check, dirs = '%#v', targetIndex='%v', targetPartition='%v'", dirs, targetIndex, targetPartition) - //defer vv("returning from check()") - - firstChecksum := "" - firstDir := "" - firstStorePath := "" - quiet := cfg.Quiet - defer func() { - cfg.Quiet = quiet - }() - cfg.Quiet = true - for i := range dirs { - dir := dirs[i] - _, _, ats, err := cfg.readOneDir(dir) - panicOn(err) - indexes := indexesFromAts(ats) - //vv("indexes = '%#v'", indexes) - - for _, index := range indexes { - - if index != targetIndex { - continue - } - for _, s := range ats.Sums { - //vv(" s= '%#v'", s) - if s.Index != index { - //vv("skipping s.Index '%v' != index '%v'", s.Index, index) - continue - } - if s.PartitionID != targetPartition { - continue - } - //vv("accepting s.PartitionID(%v) == targetPartition(%v); s.Index '%v'; "+ - //"index '%v'; s.IsPrimary=%v, s.IsReplica=%v, s='%#v'; s.Checksum='%v', firstChecksum='%v'", - //s.PartitionID, targetPartition, s.Index, index, - //s.IsPrimary, s.IsReplica, s, s.Checksum, firstChecksum) - - if s.IsPrimary || s.IsReplica { - chksum := s.Checksum - if firstChecksum == "" { - - firstChecksum = chksum - firstDir = dir - firstStorePath = s.StorePath - - } else { - //vv("targetIndex = '%v'; firstChecksum='%v', chksum='%v'", targetIndex, firstChecksum, chksum) - - if chksum != firstChecksum { - return chksum, fmt.Errorf("bolt chksum on node %v '%v' disagrees with '%v' on '%v'; index='%v'; s.StorePath = '%v'; firstStorePath='%v'", dir, chksum, firstChecksum, firstDir, index, s.StorePath, firstStorePath) - } - } - } - } - } - } - return firstChecksum, nil -} - -// These are here to satisfy the linter in CI while the test is being skipped. -var _ = getFwdRev -var _ = check -var _ = getChecksums - -func getChecksums(dirs []string, cfg *FsckConfig, targetPartition int) (chksum []string) { - - for i := range dirs { - dir := dirs[i] - _, _, ats, err := cfg.readOneDir(dir) - panicOn(err) - - for _, s := range ats.Sums { - if s.PartitionID != targetPartition { - continue - } - chksum = append(chksum, s.Checksum) - } - } - return -} - -/* on shardwidth 20 -# fsck_test.go:211 2020-09-30T17:19:05.823278-05:00 partition 6, key 'col2' -> dc00001 -# fsck_test.go:214 2020-09-30T17:19:05.823309-05:00 partition 6, id dc00001 -> 'col2' -# fsck_test.go:211 2020-09-30T17:19:05.823430-05:00 partition 31, key 'col5' -> db00001 -# fsck_test.go:214 2020-09-30T17:19:05.823447-05:00 partition 31, id db00001 -> 'col5' -# fsck_test.go:211 2020-09-30T17:19:05.823970-05:00 partition 57, key 'col10' -> 5d00001 -# fsck_test.go:214 2020-09-30T17:19:05.823998-05:00 partition 57, id 5d00001 -> 'col10' -# fsck_test.go:211 2020-09-30T17:19:05.827007-05:00 partition 133, key 'col7' -> d900001 -# fsck_test.go:214 2020-09-30T17:19:05.827071-05:00 partition 133, id d900001 -> 'col7' -# fsck_test.go:211 2020-09-30T17:19:05.827549-05:00 partition 185, key 'col3' -> dd00001 -# fsck_test.go:214 2020-09-30T17:19:05.827573-05:00 partition 185, id dd00001 -> 'col3' -# fsck_test.go:211 2020-09-30T17:19:05.827792-05:00 partition 235, key 'col9' -> d700001 -# fsck_test.go:214 2020-09-30T17:19:05.827809-05:00 partition 235, id d700001 -> 'col9' -*/ - -var _ = fileChecksum - -func fileChecksum(path string) string { - by, err := ioutil.ReadFile(path) - panicOn(err) - return hash.Blake3sum16(by) -} diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/.gitignore b/cmd/pilosa-fsck/release-pilosa-fsck/.gitignore deleted file mode 100644 index a08586f1c..000000000 --- a/cmd/pilosa-fsck/release-pilosa-fsck/.gitignore +++ /dev/null @@ -1 +0,0 @@ -pilosa-fsck diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md b/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md deleted file mode 100644 index 598896c8d..000000000 --- a/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md +++ /dev/null @@ -1,252 +0,0 @@ -Design for pilosa-fsck -====================== - -Problem Background ------------------- - -Molecula Pilosa provides replication for fault-tolerance within a Pilosa cluster. - -Three kinds of data are replicated: Roaring bitmap data, Column-Key translation data, -and Row-Key data are replicated. Only the first two, Roaring data and Column-Key -data are relevant here. Broadly, the Roaring bitmap data -forms the central features -- the bits -- of a large, sparse bitmap matrix. -The Column-Keys are the labels for the columns at the top margin of this matrix. - -For speed, the Roaring bitmap data is stored separately from the -Key data. The Roaring data is stored in sharded files -within a directory heirarchy under PILOSA-DATA-DIR/index_name/field_name/... -The Key translation data is stored in sharded BoltDB databases within -the PILOSA-DATA-DIR/index_name/_key directory. - -The current approach to Roaring file replication involves an -eventually consistent mechanism that uses an Anti-Entropy agent to -fix partial or incomplete replication from the primary shard to all -replica shards. - -Unfortunately, the Anti-Entropy agent approach has proved inadequate on two -fronts. First, it does not provide for immediately consistent reads in the -event that the primary is lost. Second, the Anti-Entropy agent itself experienced -out-of-memory issues that have yet to be resolved. - -Therefore, work is now underway to replace this replication -approach with a more consistent design. - -However, in the meantime, for our customers in production with Molecula -Pilosa, we wish to provide a means to re-establish correct replication. -Thus even in the event of a node failure followed by a read from a replica, the -returned read will be correct. - -The pilosa-fsck tool can therefore be seen as a temporary, stop-gap -measure to address immediate issues while the cluster replication -mechanism is replaced. - -The second factor motivating the creation of pilosa-fsck was the discovery -of a bug in the Key-translation process. Unfortunately this was a hard -to reproduce bug. It happened only on the customer's premises, -and only after running the system for a long time, with a -large amount of data, and with various eccentric node failures -and recoveries. - -However, we were able to reproduce a plausible explanation. -Non-primary replicas were creating keys when they should have been -forwarding the request to the primary. Correcting this bug is impetus -for the v2.1.4 release of Molecula Pilosa. - -A fine point here: since we were not able to precisely reproduce the customer's -issue in the development environment, we cannot guarantee with 100% -certainty that we have actually addressed the bug that the customer -was seeing. - -Therefore we also desired an additional insurance -policy. We wished to be able to empower customers to proactively discover any -future Key-translation issues that happen in their on-premise systems. - -To do this, we proposed providing select customers with the pilosa-fsck -tool which can analyze their offline backups for issues. - -Optionally, these issues can also be repaired in-place in the -offline backup on which pilosa-fsck is run. - -The -fix flag repairs both kinds of replication issues. - -Solution Approach: mechanism of action --------------------------------------- - -The pilosa-fsck is run offline on a full set of backups taken from -all nodes in a Pilosa cluster. It runs on a single computer that -must be separate from the production or staging Pilosa environments. - -When run, pilosa-fsck analyzes the differences between the -primary and its replicas. Both the Roaring -files and the Key translation databases are analyzed. -The computer running pilosa-fsck must have the same or more -memory as the Pilosa nodes in the cluster, as it will -"pretend" to be each Pilosa node in turn. However, as each -node's backup is closed before the next node's backup is -opened, we do not require substantially more memory than a single -production node. Short Blake3 cryptographic checksums are -computed for each Roaring fragment and each Key translation -database. These are held in memory (and printed to the log) -for comparing nodes. This comparison forms the heart of -the consistency checks, and is the basis for any subsequent -repair. - -We recommend capturing both stdout and stderr to a log. -Use `&> log` or `2>&1 > log` at the end of the -pilosa-fsck invocation to save a log of the run to disk. - -In a typical cluster, the Replication factor R may be less -than the number of nodes N in the cluster. For example, while -N may be 4, the R may be only 3. In this example, within -each replicated shard, one node will be the primary for -that shard, two nodes will be non-primary replicas, and one -node will be a non-replica. Note that the designation -of primary changes for different Roaring shards within an index, -even on a single node. - -The essence of the the -fix repair operation that pilosa-fsck -can do is this: it will copy from the primary to the -the non-primary replicas. Further, it will remove data from -any non-replica node if it was mistakenly present. - -The pilosa-fsck output log will contain -a sequence of command line 'cp' and 'rm' commands. -These commands are merely a record (with -accompanying justifcation in the comment following the -command) of what actions would be performed to repair -the Roaring file data. - -Only with -fix will the repair actions actually happen -during the pilosa-fsck run. - - -Details: running pilosa-fsck ----------------------------- - -Errors in invocation are reported on stderr and the program will exit with a non-zero -error code if invocation errors are present. A non-zero error code -is returned if a repair is needed and -fix was not given. - -A -fix run will return a zero error code to the shell if the fix was -successfully made; or if no fix was required. - -The log of the run is printed to stdout. - -The -h flag to pilosa-fsck prints a summary of its operation -and a guide to laying out the backup directories. - -The help is reproduced below. - -~~~ -$ pilosa-fsck version: Molecula Pilosa v2.2.1-43-g9dacbccf (Oct 5 2020 1:28PM, 9dacbccf) - -Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa - - -fix - (warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. - - -replicas R - (required) R is a positive integer, giving the replicaN or replicator factor for the cluster. This is - the number of replicas maintained in the cluster. Must be the same as the - [cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node. - - -q - be very quiet during analysis and repair - - -Welcome to pilosa-fsck. This is a scan and repair -tool that is modeled after the classic unix file -system utility fsck. - -WARNING: DO NOT RUN ON A LIVE SYSTEM. - -The most important point to remember is that analysis -and repair must be done *offline*. - -Just as fsck must be run on an unmounted disk, -pilosa-fsck must be run on a backup. It must -not be run on the directories where a live Pilosa system -is serving queries. Instead, take a backup first. -A backup is a set of N Pilosa data directories that have been -copied from your live system. They must all -be visible and mounted on one filesystem together. - -pilosa-fsck can be run in scan-mode (without -fix), -or in repair-mode with -fix. The console output -supplies a log documenting the analysis -and showing what data changes would have been made. - -REQUIRED COMMAND LINE ARGUMENTS - -The paths to all the top-level Pilosa -data directories in a cluster must be given on the command -line. The -replicas R flag is also always required. It -must be correct for your cluser. Here R is the same as -the [cluster] stanza "replicas = R" line from your -pilosa.conf. - -Example: - -Suppose you are ready to run pilosa-fsck: -you have taken a backup of your four node Pilosa -cluster and stored it all on one filesystem with -all nodes visible and uncompressed. This -is a pre-requisite to running pilosa-fsck. -Let's suppose we have replication R = 3 set. -In this example, have stored our backed-up directories in - -/backup/molecula - -and the four node backups are in -subdirectories node1/ node2/ node3/ node4/ under this: - -/backup/molecula/node1/ -/backup/molecula/node1/.pilosa/.id -/backup/molecula/node1/.pilosa/.topology -/backup/molecula/node1/.pilosa/myindex - -/backup/molecula/node2/ -/backup/molecula/node2/.pilosa/.id -/backup/molecula/node2/.pilosa/.topology -/backup/molecula/node2/.pilosa/myindex - -/backup/molecula/node3/ -/backup/molecula/node3/.pilosa/.id -/backup/molecula/node3/.pilosa/.topology -/backup/molecula/node3/.pilosa/myindex - -/backup/molecula/node4/ -/backup/molecula/node4/.pilosa/.id -/backup/molecula/node4/.pilosa/.topology -/backup/molecula/node4/.pilosa/myindex - -NOTE: your .pilosa directories need not be named .pilosa. They can -be something else, such as when the -d flag to pilosa server was used. -The .id file, the .topology file, and the index directories must be -found directly underneath. - -Then a typical invocation to scan a cluster backup for issues: - -$ cd /backup/molecula/ -$ pilosa-fsck -replicas 3 node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log - -A typical invocation to repair the replication in the same backup: - -$ pilosa-fsck -replicas 3 -fix node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log - -In both cases, the .id and .topology files must -be present in the backups. - -Without -fix, no modifications will be made to the backups. Only -by running with -fix will repairs be made. The user can safely -always run with -fix to repair only if needed. - -A zero error code will be returned to the shell if no repairs were needed. - -A zero error code will be also be returned to the shell if -repairs were needed and they were accomplished under -fix. - -A non-zero error code indicates that repairs were needed but -were not made. - -~~~ diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/backups.tar.gz b/cmd/pilosa-fsck/release-pilosa-fsck/backups.tar.gz deleted file mode 100644 index 28b08adbdd6fc719c85ba9f15ccc3010fca43eb8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 248477 zcmd>nX+RU#8n&&y#k!%bidH3V6{h?N~OT2!p4sHjvB zVnM-*5EUfIHYy?_Vgy7~gs_J_KnPi9&UYq=T7gNw+wb15lbatJk`OXE=e*1FzVGue zf(8v}+vC9;Ft8ctxy~<(9P4rVeCvT9r-dA_-8*d1^|6k%wpYJ*EY3I{Z8qSYxj&rK z3z%7x{dpFn@wanf)7H$)>3IH|**Tjz4!*=7Fw9}=sdwIe@0W8&7C8@{qx(+Rwl&{H zB!`{;(xLtLQSG?j?6BCCC6Ygurl$Jw1-_Ra%DPf<>xHvIehOMU#H`?t(xm$HoYF~O zCr)YFy*9piTJYKii{~!pZd=0x24C7$9+B<#z4%glx_pvLwv2J9L`md?+_KsJ4VWSQ zf(7(i%+`lp2r6O$W`!$3#AY>FJj08@d@lkwVC6I^iFFa1+7+%`tEKQ2mHB+o?l&)D zy%iA{&ipx8u5(t<;)wz$BUFZSgQg%$?3ILQrqQoePdGR599A_3FdWt0U6cgU9ty$b zP@M>5JIEGth-Oc;fCqNW`3g)CDorhY@v1h^$!DGt<$aMf3W=} zR=K|A1X05O)qA0dgK~a_&Wmc90u&%X2RvlT2^~FWKH%&uTNbCjE^42x5uP+5QabmWzMW{P%ov2QNE@}>DRWjc#ArwF)LDu;Jqn6Vc@c&hU z0<=Ak&}sXXk2(OHCqxyda-87_EIhr+S~f5zhJqBM@h92oIhQcNCGPTp4hBM!PSp}P z7)=mWSWSi<9#D_`N(2eIK(dqvKE{OwWmQP?rQl2u9T(1H)q(@s7-;A~K&hAyjPz!~ zkKbu-sjvg3sN7B=zQrc;0os;hsnD}b<0n-&3yBQ0D-^WyceouKFkrwZcM&)y z3{--1PK@B$t-_<)uH5-w!zX|EXfhiV)#=xWQ00~d@PW3E_WPkAOCR2?&>JF@2o2{C z5c}*|>~luvU^wf^onpLFk#=+6r~87FkIc!3uNuIglrCVD1MLswx55j^2VUN8de*xu zqPMM`UY%QU>D0G@B_Y?s9~{7~J9Ki}&lw4cUwo6sbfj-E|l&E#)(dbKdM+C)MZZjT&olQ^Bp-pDX@ti9Tj%V>QOu{2N z_;~oN;UnQ6mY&au@izd>a;;71r>067P+}P}A+b`xAVN#WeF9%`<=108(7-+VF1-IV zrp=GJaRqsAL!}9U*I5$sM3&Au2LZ!?;25!a_a(#?i-F|(j=arcq0gs@CSPDWNVMT2 zJvTqL0~fgzvw2?R(1f+QyS+?4P0QW!l<^6Ad;+K~PFD8Idmezcw%+)A-FMsqn8*tom(flh*g;tcmoB0+bk3Wr5~(%SjTl}F(hy%u3#!9 z^L2M`4d4OoN^JsXCy|G;T3d>P?8^#7SzGKyXo0Df+5l(oUOEv2oVGvsz(Sse)acFt zMCdFW7f#!L0lu$dFp9|xyb)!*5J&(4h!WrXdvRO84h2xa7qjI=Y$jmdk^pW?2p=d@ z_${D1PTYF6L&3lA;16`%^jd7wS`=`@cSSZt@~>I>1C)(VW&$fekSLHb_@G6E17}4u z2EZ|2zd9k9#X~dq!Jm@-JNN*7WJ0tFfyMD$fbA0k*HS%z^UHk*v3+a%m?|P z<$XTjCrdDp6VPg4kKd?LuuaiSIl+G1gk7`Ek$@TdmsX%+439;{>jySlfjWQ;65l4y zJAILH#jVQ|lp~5fRN|Vv_)EAsDnf2I^L=Gt*Tgpbg0xKlM4+nw1n(ta(!$F!n-)VFw+N(gSNOsvzs2xt8&pnLE%>qfR z+KQAL+T>lr;2t-gbG5tpaP)zg>ixZ~t9?seAaCh)(zMueV{+;?nzx`mH-y zO~t+J|2DA@q$sRXvzlCl^2QWpPT8}Hva(6#HsBXgI!*|&2mpL=rID1v4^DFMFu}4*X^;_xYJN zgYg|Qgf;(T_^n8PU7pD~X#npvC$zBuYyQAh9Ib=Vq27B_8n#-lFJok4;G5r;OMl85 zR5TR6s%_Y9U!oER$Ida*#|`WGTH*hGfqftGP&=@D&BMGJuSXG{hNt$3Hsta_?JzcI8OXL5GvXGlzB)SB@p21N~B>$vOrMlvEl>xiXBs+ z>6XVwlOH$vQ)9l}Z=@OYu=5H=G4BwJxXl08#S&Rg5NLWRNbJZ!b`+AYWbM1c=Cc>& z`hC?9T7r`XuII*&Eolk93< zt5FD$aSnL!=4-hojT|rQ-Q|GbuH|F`dmM+c0`$39W1#Y$t5Sir3ek3cA^hHlnTn8o zh*E+{8GHC3R2l&N@!Tdt8}(io3Elx;NwZMUlFb~eB(U4>L(kjk*;MvpEgxKa*dYYx zBjgIcom7#J<|-}dmyt_ z?;8nRTZF(qk2GQu@(-1R(oNka4q80tE(t0^kqFXYDxi5Q@qa%K11;DY5bN!6LP4PW z!6t()t3CWH&^N#%`D{no)>UX$2@@12U|?bs$h>=rk7kVcaW#VwA^su>a}5w*CoVdH zMpR)18gIN`!zKfd5sk}<+(PzvcKNrA$cMaF#;-=0Dg&&d14`t9ma*=J7yOBg^4E3` z#O7Py69DPD73AjI@rCL{*A1*84cZl>+X`G^S;zeIX&ruFGJAB)@J9csw%O!vW$GIX zm@EQqQ^hUhBjl}ZqE8udG3<1%*SwT%pA=P!&o@s`+=!lB|D;J8l2r6esvE!G*tNuY z?e4;oma2N$d@>MDg2AZ$oeSOtFb;0gQTom3`kc`-cYsnD!RP#~NCWXjKn%=3Z-|R5S zMQ;C*og^@zBjcO>e+2mO58BLyuezx-m?sNmgTHE##7l{3xc~O*HxKBWv6bV<&^j}t zpsIXf`hiPlJ-50|u*ukF%rQN<&W#9M$0dGTqw`@SazIpF6yA_kBzra{a1i_yeuWN2 z3E?0RpS+x4)D6__6+&t&kOO`WxH)|7(}|P$%mgqAXhD-)HnLpEcbx}(Og^8G@nFau zOJsXhBe8Q;rsp4X+*b+5U;XTT_$$)ol}kW}2m}~LLp$Dc+ar9`)MZ|r^zB`kfXp~E zoTInd@XRnpbj>w~9QJu{HP@InIy;>2J*-z;#dU#!EzSi_>gSW)8Z$Z>-q z&{ll0ukS6(=t76e(&fSM^Ib7oo{6mo%Lp}2`*%=hzl?}=ruLGcxe2_(qm<+TTdJI690!=NMyPiQabSE8ksQXt{411xQJGxk(pZ5K4 zOD{@}tQk!@9>3q)&Z06I_-mQ#E_+V3RpI5Qy?n^p7kV1zLjYbU`Ds>-cbZF4R7j?a zjyaEQ9X8rNcEc!H?v&a2qNvf?@%gS9YiYfb{0rmx(bta&T2d{8R?1U1OfW%L^knBj zDGLmq+iSvpU8%I?{$!H65I6x zor9j;ho}PrZ@ok7!udYedtIY1hp6s2Fl=(G)oD3On1^8Ul8k0%4?Nhium$GSPj#l$ zfQ{fF+GYT7OwI*u3|!Q5_(NzVpWY#uT;PTWg471Uj0%|v9cwAaW;L7-f_(qzvYxjBOjn0w;H4&^Zr zC?4>a!y_AXS0RDlr7}`MD`y;9hgAsH{qdQg%zlEo4d-KCoOYG%Sx{LX2c+iTxIxd~ z%i(l}c&n7J1ulR&w@a z5-4CNl_WLQfsN=vXa;3_a-xG2Y1LV!FYWJ__XJr)@MN|fP=R~q_avcB$UOj)ogX~+ zNfeUs2bhEQ2y+9`yKY6iu#{9}!b#zV&%T}bSh3>rAu^@TT;=r;Qz}74{!iU|RrPO$ zQD=8)u-$QI@z*hAG0$>K7NLLRa!I=$Wa6s3dKo1|((+ zoYe$J*YwMk54T^WwSDCpX2R{tT)(y8JXvY$oqT#|f`MJY@0EQ13BX(#ogyB>$~@Hz zdv__52XEZ))kQG&VX&2JjJB0v>$(Y3pc!bh@U={Hf#sCIK{E6=ZLhv+Dq}mm#f)xi zfL0+bB+iyr1)7>ea8o5Ta$I_DXTFvKta8mGuwCM&;i`|+|2*E^TS!dF!c*)LG$ z!4hQoVG`==xn+LADP`Cc%^PR+zFoZI()7raWsb1Mej?9jB6=eE5Vd`6S)c*-)jnB9 z1qr3Eh7~o@I*x7M^>2FMUd@goDj?dMtbW6Kb$qY*oJM}qemnIw#!k$}DxSpAxuQ_x z89|wc7-+y&H7-?BRfZzj8gPvZ@)nUkvd=0t4Cx$_*1^i{zYu=!ZWJ0-O_n$T7d0{6 zLHg&DxfZ_tnH@j3EL&I{n$iyzM57{~vQ-jJ1<>+>LiAmn`|4YvQM9w`ge$A3YSIUC zY0N9qfNLz2m-Pz_koANhk1?8%XO&US-BQ+g___TR;6iQY}}GU|G*aS;78 zEzIZ{rG1UuYW?E5lIE$_`3*bzJCdqGre+u(C-?t>Or*h)I(ijB7^}~6Esa9-vyKha zf!$?INVbHflq<=E*GP95OV2);x(cfqp@14U2?uECzmj?~>LaNUxt7P){_qba+2icZ zf*qA~Z%|u#?Fsct&J3o1Be#<;HCku;r3R!R=iRSST30jJxz%^45{)V$PBTiWij%Ua zk2C?}*-i-z_!az0`y)9jn>v@&1SjdFrfMgcb_l&iZ>Y|qT|rs36PSHWe-~L8l|gA( zqskz=0R5W>)ikPKGu8z6k#?PODW~K`dx?xVztg!{z{%%-AP#5{Qupta#8zv-ktF^X zItN!~QZ^|qHSW9o+L|E`&KtjIUkYO12xJUPqLC@NhF<-1t1FXib;YL=w=ksyH1dfo zw3F9EN}@o^h#WD4U=;yT{bts=R&)*6DC7h;Jqy4_85$ly;ij+ln_(~D_ zczrbSbLGeAO)Ij^?7d2F)<2V;|6wc+~B^d!?6GE30SD@d&GnszhKPDo>kJv z(CI2`H>&C3bU8UZH>f&g9X)=yg6qO%IGFG^jUN zWK!m1nn79+(kxMPD$O*YpQ&)Anzhsjn8fEhs?e@5{U_KULi_&aTxdQiNDFvIW~0C_ zG@ly@Q8(sfssX58uEkZ#Hi|&JXVlth)u2c+RB5gfMtF*RvHmmD-111sLUBd6;;#-yOB$_Ku3SqQF)(*W^^H?gkxqR7{Xd3u;&}t*Qj1pV zYCzeDUrV`$21_*|7gJ^?QhOK!4H!ts%@wKv4@CXpvNYNvV{b2g}1N6!v@+OH$V zQ?pki=%1hL+O>tMpHoLz?C#&~p(kSlwKFF?@n`Ki{>o$_q);C)R|95z< z)i7KBtO=~TAs+t=&~1Afb=k;F1Ik88D3>}N6KVj*$Q0@s_{dldC|66=3=}WbfMo2` zYt#qQ8xl6qeZHp9m~vpjyWnAeE|Nz4B89RtN<2@8{uq!&q#6npBl;ViY5D4Ca*GcA z!#driAW;I+^Nobjzgd)GKOdX^^k(hsAFe$AY>t`r6pXQW=hhzvPN?|7D0W_F&i+Z9 zX$Ln|#wUN(u*kjq_89MB-s2Bc-nC2~Jak&|p+$l_);qqrI&=J(gt&^Gy5-tSW(&vV zzi11%y%aRcK|xvpj5>R6H(l!`?^kcLogUKAmU`;J3d5pnp%2Uh+;-UrA$z|^ve4q}XJX>D2FLXtqy)potL0E3VZz3*(!0AWg8<l^_J*xr)S^PMc-O{uX@C{{ZG1e`$SZ9Td62w_Uqv`s+H8GPpE}`0=Sq zSa3yIZvzEjObs_}Pzk{N-*49E_FH={(y73bpPZK#x}B5_ zX&Gf!?r;~%6*)ob{_#ZS^-H@7!gHW*XIyTSe40zX2yrQbY@bY$?K9Hqq081_o{qq% zo9)v~%QDT0U_5-u_PO+iUkYN9S(>j>P;-U~YG=VKwoi1QAA0>I+h>=#)E{hwazzR# zSLEI!S7e^m5p4(MisUa2x~t@hv&E z9%I1vSc6TGzB}ULU3t0sG?kZIXDKQ$gj%P}GxYIi^_8 zZF?ke3x}ef3X;^c&2N{7k*{CoVt%Dc%*SBqPtPqyV5e!fm`_n|Z5H0pUgpv*emAv7 zRH=L==A(dOK5;!_KFDpMLK_lemqTJ~&MRWrigB9C9()bA0r~i4ulV@8ARoV74R-(Vbc_6Y;Tux>LL(YjjA!S9$Kr zLoAg*J6%Xk*PdJ3czMRHl(18|kPzGDlf;7jq~=dV1wWi!U#x)Ig=NnvJ&0q3emRjh z{^ETzS6wKQ!I>V^7?QG(2U0AdNQOTtlHp~#BrU?klzCRT^C0|_H+3d1Xo-=bi(iUl zG}Lrd5}i^PLzPHIG8D|JP!)`t;^l9d;TonT)zcl_3Xr^#Q6w!Eh69~T0}a?(wd=?Q9q-e7uc`sBW*)*S>1|UBTh)M&nvg`E zFDs`yj**^e8sHr38lF(q>Y_p*13&$%A2KzCtQJkE@fcArrK-zz7|LHu=en!1i1JK2 zXn>1!Dxd~hcNb~^&^ks<6rc$hq-9Y&RaL<332f7&eW4C)SG??l=(9y260%v*J%Q}7 zQge;mXz6%UddKB0Nr~#9Jv9JF7PbrM+$`yc2Y=@h!SQ`hM1 zM`}Rbd{!ZK%nH?{>E9gStZpSC!wBs=AZ8KN%>k?l^sipIsPU1cARo=ESB0&~)V0%FLkklq;yA>?;#Rjn1ZhyH45eS0Ln7zsm5pTXuPA zkeoaj(0i0O$tZ&QJnBL1dfK1Ih8Mgc}>0}(29B}!io*4?c{ zw}D_N{1q1i7||uNy{W(n4-5dinUG0sYXMjk-^K=Lbil377=^dW;j^%N+LeepCJa;}547$9`1B9xK4YW?Y(#GURn7=@F+0iV88y6K)Gxda zzYyU3iOcmqhVS+tIz%~OSd7A}*@KV%FU;LdbM{n&acRQb_H(?1dKp)1LK7MgbEv&3 zR|6^t=S9>F_ZR378_4vijW9_tOZUjHMoqu~gIn77gRJCI6Dh!Wf$qzAzU(2jVGS|U zfF9N4KHWJ?JvXf2(8|M#P1tO6*d1{nxix1A(-}dbI$e3~PWfeO6L`ouP`aZ?_o?C`tK0*5&Uk#8Kv2OgQBm{RZ2lv z<4JB_x`(hQwg5E)F{zr+IMIqw`ph6rAj+~ZvZMrh3?NZ1?a5r0wonJLTv*T$!7?4fiHbL{R?~zh&tm6TVQCvl1_2dy|DM5Tc_iphO6kDS3bW_XPl5S2Z|@1{9>PZF#=XNPxAHlipc=CTvEV8j z_k2rXV*en>Esa!Rgbdw#O~x9O{xnR8hZ#AvEdpu>{n4SiPeL;6H^Zu7*ov zi$+}KHR{8wrwK9?$^Qf06F?nR`ViWZFna*v3q;i^ zrwtB93@!8f)gvo%=;QnM-W$Ar=&pi$nbXXEy}Zk$3#cX-w?D)w6k?VVeX<;~Fk;imqmZL7mq4xRJcwi=f^gCa}XpW6L?dDUk%2NkmQ zqqMEsXLMYQ@|v0%Xe{WdDDE6wUEt`zx#W2DtV2xhHjnFw1xahqVChf<{{o0mri8w@;Ng-?pg7*k3VU8<`}tt zLA5kwCN92J@)h^crbj33mpo`aU78W@e6n`4>4cM;3;1_8n=QE)y%-RDG1vIUPj}n(?3Wr#KW?zK9vU`pM&PruU1^?)N3Nwm2n@J3Q(l^KSzk2oP?4|= z@46J1|I@@AYm)?>aCu~kNe(;3bpx1tc*qA(+N3&&zn()VOZa%#K}+&Vbn7qA!WIH; z5h+&Jsimw9#Xu1QRc14e7JW^A6(gXOn+Qs|mCJ+Sk8mOVu(X#dXOY2eER>V43~wLc zw!73|1XrAXWP4+x)AgqBe{VW>#&8o;CFOST_RmDC2ryXl-c*fRyubaYc^o^NYim{- z6ge(TTWcHbXcGP)BxuV7v{1NJi#fIIvqQ^PZ>gP=w$4T3l5BlCRJL=~X)|aXyl-y9fm zy_Q=CHnK>S8e$Ko?2w@H>R?Qo<+k<%_<{*jYD60esE@S{kVXGUs6Oaf@=|?JG_qVM zynY6quRH0ei{f8Z$&p+)OaDdj9uc4Ip144?qFJSnrEBf=knc}`EUP~Wy2Z}XwX73g z`g%-W!{)HdTkP_-oxHp?#Cuo4;cBaE*U+|wh4E2i7i_j2o|snaA>1;4v-cTotGvp#BDk+`1 zi2UeY=Ms~ZPhcNEDUtPv(H6=i2rE?Cu7fUKIRmyf?D+yp*F7Mm>zWXFa&L;DUQO#u zy&5mm+X_5mk;wHfRLOG2K7a8ZUQBVq>ir1;Tgngm&n3Uise^g4G#)^uxqlK` z9CbH3H{HP|chi!LZ$pZkx7i;#A>J|1YIc~L(gHEi&RmgdGb$kF!TpUM+!w87JECVA zh$_#`3?dfIo^k)y9L?FkK#c(&sxemS8Am~RR%?G4o890kW%(8tZ%H~@eNne0$kN~H zMP#+r3vL1!dw2gnXolOfT@&n3qDjc|y)&KkCEB~6oaRG$Nf{|GDIZ95%QV2NGIafp?2gla-}20^EIk_Sv)*8l*u>5EDB{LXC4L;M^C1){%+bMaIN&l4rf3-%PI^h> zNA~Mj+X+S;sis-4EzL7PK`cT@gOCZar{ME9Cqrq}I#L={%z#D;I#Ld5^9WtjjVO2;>q!YE@UMR6rn z{9xS~z1T!qsG$Gu&Fx?^s26pLrr6&TqXVhJ2eacV+ErTD|C_fX&lvd7E>reAp7u}TDU7W8ize-vN<*LbWz>{ z`}~B{#rpQV>7tsbaXIWxEj2MxsOH2-6R6$kOigj89dWm)cQ4FlXdNSURPT_RaE`l- zc6#28B5|)x>A}k+(F?}M$R@$He0Fd{uk+c6rCna610;FHKP6|T+*?`pv2JOZ0{Q0X z&qwOU801y9Do};+K)3?^Ee79sH3Nutq-21=9H)nfcs>R|0}sO`z%%o1J`ZG~@SFjJ zP@r?hIjhy{VYL27As+;6ln@4rFC^f{`dGY^xupU7HW4JS2|Ifk^FnrT7OOpOw<4e( z!7B6o*m%fH=a%H90?RK>lLjrC=9qNlftYUlU7?#Tb8}y1p~opZfph>ytL@nlB&zW}d~B zXX)qV`7PGZi(;EQ*$xfd+)r5ZxGA)$U=wS}kywzgU0hp-ng$V}i8#CJMv9OChKYIL z>es`u6+Z`s*6x(x?3gkIHnT`ym-MErItd`ypbKSZ?@o#|Rf0jNh=;L<<4Q&jzYXq8 zaXl*CedDPzAABt>MG3s}N}C8Bsx6mE5}Ej{lBWs@vs|X&uw(7(gji0Sqy^l|XtU!3 zA;1C7;my?Q`nmn40#mxp3sW}bxu)GMS0MRD5-@4vP`DW@*=!35>!Gd}&<<+>hl`^T&kEt6mbW?o6Y*$?n*+EDF^ z0mbH40v*)~*jz0FM;7}44E|C*P+D^N6Vp>C%VaRU~C2n$3g59sb!0zu2~JaE*XyvXjqS%I@B;;%nKy^>*$fefI~ ze*q9zZ5#*O_0`L8^e#XK<0yQj1xN^xqWd*IJU3hoR5e*cKybSIF9`Ji~a zxGk&nby$?qrUiF0&(158pJq51-u1eF+gF*D4Q$s<=DM{whTNH{Hm9??5znDfFX2v>u5)lq z_M}6BflME)S7G8g5COMN*AkWzCP3dxBcDcMEvvDrFHH#rOtA z`YVIm{J}?vhzX29=a z&yRz7l3BGw;9dBw24f{%0~QEHId1^S9z5`2>WJ3xIm->n@9*sAzww0Wmib@&Qa(|l z-ScgdPdt1Lz}de6z-8BB5jNqU$=sGcKRi6O9~_y5JlCDYt#p$|WL9VNa@H;-@?e9d zZqP{PB@eAO8zT$%WF!9guGf&G$|H>U3m$$)sR+Akhb&XZBNm-*JYk>-YD~=D7lH-R)2!9QeXV}BGZcv zda!a9^0lIL4;fr`P89Y6;H){!1oI$S?E-=cVi`m@<5FHe@Ee00Y*ipRqATE)$-ME} zWWpnsegi1*793GJFgr{UkpiqkiBfQCOi_W4{1}+3Y(YQ^ct@#ZWb>787|vnchBKfA z`gcawI|yKP#gS$Z105SB0u0aLqrsKJQgGb2(lrx8xEK+S0oJ17sUVM`7oCe`zVJMf zAwk37C)kM1d_mgGv=a0Qd_K@T9caBYnMU;l5OX(FFGIVns1RDPfL2g-5CehaOGvBn zjs$Q-u4y=9H;}A`@e64%C!Pjq=|Wvt=5a^`i~!QC4Ob{YX+rxsC12QWsf=m}AzD35 zevS|1(3B~lDLbbG<2mhn6vzztfuN-oc|3xV(bO)G1{nJZjLpp|2a4qF#`?gJXtI`) zFL){h@Uq`9me98GpS}a_Jz_w9x{X4hI}7ku_gKp-0@S=2{0yxh`RpTyTG1bmg5 zpSTF*j1;~yyBjfpwS27I2<8*r#@gsI;42WT2tKr=X+n@C@E;D3cL3@GwwjeoBp_O9 zz}p1gmySylz(A-&e#M&|P&XWO$(V|K9C`2n$o0E$W%#dawB3amNMtKzp?6W!1RZBt zcjP1qix8o5z7|??+Mx7W$?0B!Qv?Q&31eydUpOYzfm3%)bi^Aq(FX$0@|e2G%Y;W2 zq=D|at=#%L;hgza!s(b**)IepgptL`0OS7hunkQ}X8oQ0?Nba&=N;CuuW>E+CpyJ> zuhHt(`yfs9dFRshN)e}=iTar++qzyCfJGt6WT|3tOq(D7_F;Ih&2PU%>g%lixyTgl zl~pf)Bdean=6i>S1-_SI)Jpc>_1qRmye^3*zEu)EfiE@bxB83g=O5bjD7Y$%qa60% zvU!IaFaC1}ncS+AHmd_j|#7-0gUdPDIKahnmgn4zy2sd(%RwWP=R# z)II2+XX-~3E1H5ifsAt9S9J%JRcK2tL(Zs3;X2jKcO<7x7v;3P^HaGXxL3=PZui)1q)n*RKo5vbl zdplT9Onz|+cnbE`yH{lim>C!$^^sP#yza>m=sA?Z`1v-vo!^M!vt0A ze%*5N;KW-kCtY#P{gxO=OM8DAq7lKtu-==t2VLpjv|0Ui5z%Tnk^Fu;h+tNfxM|(5 zRh>qD*ncha`_IHcL-VrUwm?0LNkV0#vSFXp--r&b%zZGV{n~0&%puB{ARCp6U#0UI z=uA)q`DB}t2{gz>p%o_2XhP27)vO}lHeM8vrrzY^oPuYL-lr}|42HK`S+#k#Sh$#u zeb`~>XGorETQ3&l2|7sljXo@pq$p4TCNHCE zfqXMJCY=y|i$l2sq9&YF8&i;Nki7%M!A+n3gQETKy0l5)sIOKza-&D-U(;WyjrCFO zrd1d$f5sW!Uq_(oGAGoiUCsZup!a_!5WcLh<}&;32BcMlXpinxm%H`TwM(yE>*PUw znUUt2aG9(Px~%?M0Liq*I;NkNh5r!B`k(Q&okbK)aoyQle@!U#I-94dlhiYkXldIF zG@w^8f?iiScQKixUNzw*-+Q~oT$3Ei%o%DN%6X0On=@yr3HCJOGK~>se+_>mu)Ar^ zm;H3B(h9$pP)D?5hxgY*lptm>cURr^Iv(mg?rzY`4nv_SDwo&39$)gT1!dlod?KW6 zGCLs$$<$?e`M#P+Q_3vXk(IXztGzN^H?Dy9yfVpBZ?e`oZ#L3vmS^%CvpoNo1xj*4 zAh)RB)J>I#AMhT927V*7>(w?-Cy1=ZE{pmq4{aP3+h=8)R?ftXCGBr8i>ySL9XHd2 zV_7Tfw|d4?0c0QdH@)+h&gBQ=v(iTx7>VsbBOeztm4QsqQ$Z_G8{i*em}SKrYX763 zs1b-tcE63f-Kai1&Gbk05xlrxqt#vV?{_?nPJVV;o~nDSUl4F?mnMWv4O=nOSB1s5 z$J$km4Ra1^0;=CBVovOLUOvG!$9|YV|k$t3iz_ zj=wP|<92HZj`cZGNH2U|q^5x*T_b`6rO!@kuwm}us?IjL14rlC0_u}FyigO`RB}0o zDDknUhcsa=`37p>k$$JQ=@kG~s>!IwXg~mDfml&iTFAyK4d^X@Rzckis2RSbvxuNh zz*NJ`ee%T7Da<=@p?~CB1p;ZvW!Co3r_ghm+E|C-#a>)CbXv^sPE#geS4V4tkv&b_ zP;=E}0{v@)=ap*u$~S8S0&P27P4F{RBW6DABPk)v?BoysR&nqjQX9&BpW4Lkn9&bq z&8s>e9hy*QHmRA$($NS2ra%pJ^Ukuwtokh8KYD7Pcp;fU(k`;fFNhGEpx|lr#H^MxdO?Sn6Ku{we8cg^WFGqb?0@9H|LO8=?;8 ztkH;8>m82}$|j6?1{yI2A*)ccXEkE#85ys}0DYwqi)oLkrak)9!%#I29W&F2h8mf= znOF6VsB}nBcdY53Pqn*0sz1>WT8$50V|i02Zs%Z2P2YM6bZWT$w6NdAt$Px}Oamq% z&a+2T#qChN1W_-UZBEnuXrvB- zra;_?_MIj*cCvoU_T*SgSli#(uzz!?1$;>kwRrr08ERpvp;t3KsHV_QO2hT$%3HO@ z+U?(B8UD`@E-dq)JjvNRp(GLQ#~^Y6I|GzBPh7(Y*s#{^@PfTI28NeyM~*$G|8-6I z=f`|TzyGe}ouRnfGzMnr3eE9HG%C{A*A2~lX7@oM<|LWJWs3VBzr6*hEn*CR}n+;XZYptomLEJ6hdw~=k=3Z zp4*4makm!jDv7^6VtIzkoy-a3FAUi)R;F4y=??DfO7Be!we}RLMP>!H$Tr#^RcVp6 zHhL>_k-$JCntMtRurwxnjz5EI&cWbJ-?N>oMtNflaiQOgL%xBC4|eGVDH>=~z3u8B zruRyJCc;avgKF;?1`l*@{IuyeNM)9gz}96(20t|;KS{{Lu#`B$l{2uwsW!050L3%$ zx#=?RvkoLpWZ)pNpX7W7yay@8)A`)FB!R3lJM>PuBhZxhtaq%QhmpHHT5p#yv!8U2+r_)?U9~jZ-KDDcs`~{}|WkHGa*P>PDxPsn9(#4m)g(89eP$W=0Z&$ZS zpdAm21Xfn^>YZz4=Z-qf{g$88QqbfI88{YA7-d#2G|yPT6FH2M3`aXzHaApqhqiYmIVUBV>f z{)!*HM54tR-3phDvQ`VyPBJv=yTizose;4$Q?_g?H!F>JJzkOWj`?o|lwCPkg|yY< zmv*USV2;FP$H*tS@?@?qB}A|wK79_eChFIXnfIoorLuhdyiGUhklWE}>GrAwIhHZ} z%h*$l1WYMbbmg?w3EX_TMJ*qT%0jwc64kQ!cU45SlEM6T&#;ku2B@LG#_2` z1R-kM+L_$pw<;%TY>cqFt6psQb>fuq!w*dnN}?UY1l|b!ZR=3z9Vo1hF=KeNRNI)| z{zEyRdb7Iwyf9K0C~Rs~vW_3y1zC!hl}C#BFA{JDZ=E;Xre+^y`5ia&aK}5P9e#Ia zlY5Z8`^lXRV$^ry!Id+R^{A{#(h^*m&#Uh27A(3bNYiV1N%o0n9#N5f3P)#F3l~fs zAeJcIIv33zVc2pxH?B0%=|+?7#zDV3K$Y3y@QvH{-}YgZ#VpYNmyd5b0w3tPb*pZ~ z--klU<*~Wjx~|MLR|zG1ij9f$d?7=5xZE~UyYaU%S)KkbbTfz*PPUF&%dBfk1>(Z= zdmZ|%_+G%Jwf872M~1WdGp%FbRgUe9hCY(uJN$e87SdR{yq09kRyBFD;4mB+wUxi{s`VDjc z4JkPw8zCLZFKvWrGZ-x&XltNRh&CLA z2lLy>XPC3o1=>RJ#a=>4Y+eK(=(%<4V)lJBrr}%N`u%FiJfNw_O{lHJGWz#b)0)B9 zx}cfY;+kS}LIqPjNM&N_`76=?AZJy`sL`S_Um{ma7Th#K{2Oh0-8GiL;8I8s)T2L}SOja}k4_>Rh+I-Fr3CGH(NS@E3!` zsUsD;$X_rR^+eRP=z_QYqf$|C*^F&A)1f-LDO5)f+`Ob)9X(Fat`~q#Ao?-oIP38@ zqNOIS6A5~UR^Oecjyib0*0guYHEgtkDn#}+I07xQSJ%JktNtMe@>&+Vp!B|I*~Fqv zzhwrOMwjp*(btZ}`iIi_!9Grr0$iW=oApH9vs+Y2tHJy5B55C69RQ0dGdge6!}K?; z-YiP@(bP%*#5@dCS5RfGIssD-`Jxq|#paYs@qF9*S&NFbynP>{Kc{t}#;&#dWm9x4 zhlt0d4|vRn@>!)RpLd74yqMaAb^9mp^C>&%zl8h+oxx2q_wYjF zQS~j}06x%j>sI8r{#i)&E+6*vxMx|{CVbNL%rWvsu}B({#Gcq+x})-uQ}X^+BJj^J zuH!m~Qv*p8zHDcB(HktY2&(KcuBD|HdV?%F0UJk2h?}4^j?N(+zGEk43KF#JWBLQw z7Hb{Vq#%ygFBAiTD~Ha-i1T3V1WMDhX%2Lh)R{I7@Fe70?N>-k3tpc6Hd1f!euf4F zJ%tO5C<{xaD>T6PG*nQ-*jdoR(!J(lp}+(BNkse<=?-D8<2AzcOwgq^J*&P{2W9>9 z%Pt+koz(l)8~VyDMh274b>ZA5`owyj4NH$paSro%R2R zq@u(XPwLqIGVDZSE5UL)+bg*}pk7LQ4d|t^cn)&ZoJ)I6hzFtmB^5B8J&6RSDNBX_ z`~;?kp$^Y12%BK>)ZO=@2k#t|mX%mOO&PDrbsG3`EPXDlX-aR#7pC)vTIZ#(N zjTtN4gl*0Pg)nuYD zNY_R?z>t5hrRtwiNojxpSATtDs+QblbJ|x@8nhGpgTPd^c!!~K4Zy%U75#ON$h!8z z2J*lFO=vi1P?Lse1on;+BI=wn>uC(GaByUgy zqiY2A+O8Ez(MW1K^AOo?uvJLwt~CNtvs)$6^lv~!7poiA9iaiePIHYZyJa=rsWHu+V<$BI{Nq)!ZIW1l!MO&yA#|rdDuG6UM## zc9L^@vaS`S$KGbQP>p*T3t&r*S{aIGHbuE@d}4~do(T@$Hx0r=Qf8VvaLJFfYSW&QskA3KlSQ?vT& z3k)B2p}pxizIzIJs{b*RrxuDVYVduiMxZ8HsG8CLV;V4saDu32Py>CY6X}l)1)yrG zo0>3+fX4LKEUV%MH|8nmKZ-C74D1i&BWH}YgH-s}GYZ~Is#?+Cq`gE*+RI14%k?0@ z<)S)Wv3sR1&;dgs$I+nM3dI%hh5l23V-H9k%*^OjLJ%hb-E`Y{r2-f-uxgUMICH%h zk5LW+-^KZO-i{uAB=~>DsJqd*>3tK!TOcNs`HpOk+WEKJL~w74=D|jv{e!p?8+NKA z)`1+&^k=(L2es&HKnQ6O)NcscO(h7M-s#SJd7@U0K`zpOYb54VFb1*ogLXaeB~VyVW74AX$XZd@mIHPrO2q6C3HRfLy6$#7rReBOQH)K9;E zzGB5*lge{fwe|{shmv4Rmbt$_uI1MIzm(lOr}e?2&Z_Va9l!ZS-}b()_@dUhg};;8s8lW(Z&yzdF;)e4tJmMbhCxvNdkV&8P zb0|y`G<~7=2XBjg7)Mun+0M#y%>3=e`P^MTY5IY=+m=Rtf&$GwbdwnRVG);hnXJ!t`~C3S0y{b{Z`#YC3n|hU0)M%Ad)VV8>e6x_S!1vyk z7>X>|X&iTY^E~FaE`^RuGB%vda$Go(cY+CJTc+W8YzgI3W}nl^F5|c19+&QmcaF}n zUU;dlsqsk4L;-Tc8H&^;S3!}w#2APV*Fn1Gtg@o&cfv`!=JXD6k_PRSa*?)04 zu0x8{C5(lyuNVkL>Sl*MHlFr2U32^41J(A|XNJf&?FyNm*U%C*VR~GH*J%Xlo?TWkgHFP0u_^#1Yrz#V4`4x46=<$goro+BH}QN$TBR#0K+iMa_{+{JAkZn z=k@yM>)tn?Pl>D^?m55nTfU2veW{+OZSaWw8gDb3Io7upLOoe%QNh>$FqJjgM=C@~p!a#BqZTet$K#%AqYD+SJ7e{UlCq z#%&qQQ0FBdGDE{{`MJ0l{%?kX8KpsK&Wv3a6FiY@x!^}quM)8$-mw<+pl8kOPJh$Q zH9o0h#nL6X`B!_;EQu_N6{ZhKcjP~p-fTd$V zPDtu|WXp4KvUz=^ym9Hm*XkP&O6wJqaT`!2v;p<8GcLd<=|Ys2>9{!|ARhWE+AU|= zvwvw?3G!S7pf!0S^i{mLeE0W;OZf>2wB|?yy~iKGUG+4=ChVXi1Uo~4lR&*v4#ET8 z?{~H~Exv|t=8o}gilFEeJPfUjY3cE*b3fs#j))y`$pHE)8ltbdZsLcfrkO{4CtyuX z!+jM!a^U9=qd{Lq&%&76nQI1EWS(|1n#RAf`@((wkZ>ah8@7JUVh8Koq-uNUIcVjJ zlo9V&)Sm*ur@9{F*RiU-d5em7p2=Ed%!v%f%}y^dCwM37yBC|$CfJn=L18+%Lr>yk z9glKBU^=f-)%5AP#>)3jM}PTgL21cUSJMZ`K>>7daLIuV4o=onYJ=`4#Wx0|$K?)R zfg4tN#f8L{pYL)4eU>c~izEnRtiPY>$Mf;L=UGT zbD@L72HxuOhe_`yB-qZWNpU=}2zPMUz65^$?GezyVY|_f-X9KhIjWj6CO7Fzhj*np z)$dxLFBVw5|M8aM)&+{q&N7z9{uxqe9?$BGejqcEg)8=qfEoMOP$l{Ey;}{{Gh+*IvBYAO-yNO}g) z`-f75h7`hg^t^5_At+$#IljL1==5*JXxhg?Ym{l_em>^7zveN2yXs8FLu+Xi$3j=h z-Q^-UFepP8!4zmd2rX|X(h}&vA4m_jR0(!$#Vv22zg;cWw{+3ih<;evVX`HqL@tVgRI&Dj$tP0lsbAWa6H2&U|k(=|EUNC&u z*!jlAibeN!+z31El$VHG-o_#oTDO2E?L`?BNF!npEWd&%k$^^`QK_n_St1*9cNaFK z5SQIbsFsF(CE;!&ckZhqJeq!IEL=T-rDV&06t-~V^uIv5RK#o7RdxqcpzLnvmk}S# zIw+As{)5>h{=>Hk3EI?8fK)*f&{I+R?QO#c{HSicgyk~JCL(-qj zn9>czVK)3DD&#*F?EirsTa+J|NjN|%gsfrmkI53{;{SzZDnl!DTqab=3}Q-~baafa z>i>Zn{I8PC?!T_1E^($pltn54(FjQljm*eDcu3+sCvI_KGbqrlF7ngS|AX?Bq#u+$ zu_n%aEFMu|vP*}k5hC!6cg3k^q85_RJSE0vMBU0Y(EO;G?bSi<_Lg;!j>N~70u@QM zAyP(sjaV24rmN^aTryXeMU9?1V$XTYR?HTg|0$V4B}R=B;=9t{m;!etRxTpku%(rx zZO39p5$XpxVur=-LLe|>y!+tT+vMC2!B~5!1yXhpf2`@&ZuTmImqpF0^+>I z*11p!wnk#b+NLWwuQ}7F z?#g`5M?}-ral&S#g@0Q3;n-S^8cmF?Dr2iHpE6Mi5Q~s4Vt^($iV)fk6$jAODPXi$ zE(c)%6`untGi72BSIK+Jh8++s2&bO|2UDM9wTZ{3V~`4I2{4gbAunRlWpZHIsV@v& z&N#*k!j9F^{TG480-&NrgZGup6(BgV;hG3Ij#gj`E7WKlc!&V$&+m?DQGwMvrq>}- zIx0n^8LvIep<|p8J2EdrKxLyv!2tBcZ*r~%Ybex>H57R-!!e|t_lE=0Rwa_nw}LPeXVmeb(|9qA8{ z{Sq}Q9sy7>{EVL^Ah;Xp&_{Xj@J0WNky(8`qbKX>k3eE$teSX%^C_L}s7~F498zE-Yl)WCG zi*p=s(CGTM(p;Yft5@;q6>3oFlldNZ`_*E6XPTeToW}bB-({_m(D3(kzgG5varZrq zOk?1)H6ec&ULOYT+}H*E?Xy`6%2<(>t)CAyfBYmJsDY8%KD)l@fSa#0u8ld?KIB^b zn*E!>D`IbagV8}Q+<=V4ItVTpCXyp!JxmM?(*R44|3qW~*0j^;p+Hs((BMKa4K)l0 zx%M|za(enLAb^%_cf)xg85IEr{BP$|IrssBseN%U_22n8nEK)XF!c`-wg%96dFWqd zKS1M_Eck0_85+})>UlVitNs}QNE06N zHfTZ~ABMap1PcelTf|@{R!axSXJbHkdbmQ#?zFul!XRoe_D4Dhz7FqbLsZI?pzQ-Sc+!~lVr7(f$l@|dI{!{6zo=TWl_veO#L$_POufR_e zD^d6z8)3YY!}lN~6>uT`DC`0vaMkc9VJZI3b{{@qc$?51EpdKxyn?G{i1fNnp2ZY<8 zm6`yFB+lc9-6;NU0IGhT8wPnGAYr2DkESjE5l7;0qZb;*;<8;+ESkitL#t?PJH?^k*JW~C598@;Ed0akhwB2#40xY1o}=n2FhaagCN_A zY#JwnqE9KRgKSktiJp(Oj&8`TM)DyT2Gt`*5)FbO>lQ!u!=ZVQlqkE53Ur`Rcw|)N z8JPPxb9D&~kdf>;u;sj~3?rg?n(@=gN8KKDF0XQlFOm^ZdWh^7$?V2vVj-8GX9ObI zBoi7KcqS18Vlf4225b^w!4=mA0*wC3N?8MsWVMbXDKaxA9h!PjyC-b7_B?$7l&>q4 z=u`|K;&}e^)Q;_6Cv8F1JS7b)1=WZuAL&s(t5XZwpde8;Z*K#zXfL8~EESc5%4(5+ zg0CKi)`{Hbs{;VPZdXR?!itKi&uC2KNqKbEqVg(s=N>q;UOlct6oAs!-oYeY5(&4R zCbXeQ1B!GB!0;3p$Pq|7aord_{R(M-9L!t;(vhWa^_hIH{Hj;>Zg5Jdiy1oEO(%4> z%NAvT?A<14i}K%7Wg3{@*UNN6%}IZl)e2obYG`mo`W|(r4g!c6s?(E=05T|KC90@M zB^Ba;I@e& zwwy?*kp-D9L!y5F!pfq9N;Us1!cr|UyNdiN!7bu)lnQHXSLP5;8y=gjAfL9ldg)3 zj3nj_1LUcFggWGg?n1*X(n)6yOz*7Hb;vDJW0gtG#Hl)5Wd+DXPLN4KVc;_?jfD)jf*$Z`|*r)*c4*M5zZZ~T_ z2?bCGPV;6r>C$eYM%vRvtla{w9R6Qyj{w~lBnnKfo4=Nx{A{xGtLZ6Ix5STpedC08 z4zg!_m^o_rQPnVWW!2g<3;z-I*#hQ=4Tr46AD!G(b1dC9`F-BX`{&PnwqeqKGut03 zj~M3JDaI-ulrzS%{HO2v4o+vgt-tK>L2a+~mw!UbbR_P=E0=lD6^rYAG_@YOVhOuk zu{QO(Vj1?jVl`qo?YaAPq>s;sISknW-98_;;u|}mvPe&#jD>r)u#GJfd3^}Z#a&YC z`dB}y^~2+!k(+V1k=tdcj{9Vz#(#%nz>L&esqxa2?74F~L^?H9< z;{V2Jz20B9?s%#r_PllpgTnb0wNN;p1BLSujd1?Qy~6ofy~6nkD%^g(=her5M+Jj6 zZBn=YPC`PJZBaX?=cHRhTwIoLS|--t9{|V2F*}idiWO_1E0(Cccl`sS`l|D~OpRTg zkDs^s({cEC-PXkUNng7<6~C$GX>I9_^}Y50(+AiCh)0Q_UEP!;N?&L^7!HjG{h{$7 zsxcm1+{buu?F&|Zdb1XtgXW}N?;rlG+paD>?KFK#bx)2lc##m&pjbZ$Wb`{8$lnX!@zivb0^|7rnlTC; z!&I~^fJj?S{qc|FpOC97QRKtlfRLwgl6dj-cf4?48uaviUO2yTq4S%F8y;OM>gW8X zQGgx|`YAx0;|kE>-3rhTgN~!L!B2mb>3e-${6q-LaD*{$?6O`@w1&C(0((8t==Pn^ z#qa*Ghaza%Bgj}?-VQB$n!7A}=Ji_k==WOoxPzW0!ar~Kaq;88DO7g1i(h$(JG7nD z>me|OL)*#2>F@iXr~27W(qRZB`yRKF?(8154f!m?1IDI776I|Iytv%uvc{{>HrS9q zV%jYDi*2!*W1Zf|s}KRVGN+ZRP43Udk9n8-QZ)V`dfk`Q+bT^ln`Of{BG504Eq&Fs z821aKYy84ud;P+gy?$Y(C~k(*^XlVOco??U^t!zY589Wu&dL0_JCPnbD4X%25&Zx- zgK`G6No9B2q;7=fn#n&fG>YY`vdm})zJ@>b_M8Z#roM{hsJP?c0cqpn=w45>(*r!w z4xNy6nijmi8eIj=H4C7*CLNkuz`pW1)elh;^EQ^fM6s$$LNfpJaVcV{ORo>O7Q%ANii-jUH#sp%=+$$N5WSKacY(f|@}}B4nQ>oV&7nk*X}B9>LAM)Y19XC2@lfM!7nd6<9zGWL zw(E=ui52$ow&Q{SYR=0r=>gUcy-0n0JVW{?`6R!m+pR4ffFiSGPY3FEgYs@iEG^IS z7>feW(&VnLXGw+B&R0h%sxcQLdnwx1(<9ZA+CgwVHo8B7fOMCL;R}8il|#=&?bS9u z7&hY%Mlu|O&hWf$XZW?5cO8nq^7+==!yRq!Z1cN46VJUlZbiw0Hqc=*E@F#cDyEim z1o=M~Bxav{1Ag$a4&4!7sVZpO{r8F$830h0nH!P@4vxQ{F-4^|=9gUz$X|OVIMlA3+RuU7Z`>fc2-4*|$S&=>^}pR40(fuejDLplFTe+N zJ`}7>I}i+l5L|utZro00`cs=+GpE=Ea0(|rK%oB3eiaqd4j&SDtL3P(X!x|)t zsyY|O43^U2O?!D<$U(uiI}_GsvVKXJ48P#%=o2#aqOgZ%v+J7D5crOs*Hf?c*{xT% zp3n1Ja*Gd@;C9%J4JmOm+tA!#g3U%Pvk2=mTn~lU0zCSPk&-SR^K~?a3}N?cpu;-Z z+Sv6n>=xmZG*m*mEa~erVA$;eGPqffpUE}f!U?OPnQvcL!T~+$zkVOHOjq}7FfEI8 z-LKbuh=Z#cv^g2buOnOB`NXYZ6xRx-_r|>EWFXod!0xkp3|) zts`Edbj|o2q_Qx5a~~HMvfUmEr<87d&d_aI(O`l;>FVpm(jI*Ipr)6|EHS*+f;TmX z^tsMi5Ac)LGyr#*&`16MNE$GD#aHe#A8bXXphE<8{FMl%VD^k|BanSSj?%D0#UQaz zpk6Hm2S7jwVw&KjhplU;$4$hlfU(IGLjxsEGNzTs=ppEXXB-^Vu*M6abs!`Eq2hti zLiqRFo;X2YhS9t7UAi9twN;*K{3D^GC5mhnICBXS^rwvJNp#_#)3W!jjNF{R^n&5L zgGOgDU~-RC?%-H;<8pYdtUgsm+$j5piaxmaFu~)~a@eNV-~W2!+KV@pVgOxlf%a7ySwK7l z&zYgK>kW&|$%CdepY&HQ_%F9Cd{7+|xBUiF$)6#&2BTdD!CD5yj2H?OJK&3M=Ecw* zwjrcSN!NRtCG6H{A9PzuWH644hoJ9~uF`?N4_z`=TH5-OJ@^QmU(wemgmihww@uv& zPlInAA82YtfeiUbWz8Q7|FnDO6~A zp6$3n3==EF%`XhqyuMa?q-!FzL!xY#Bk<7OK)*AV6f_NiEz(D41*RjF zpy|_djg{}6j{fq~g3=PQjUi$}WxAjmIt!4$KGNJOXg?7O*O?lKzk2!$7SzzEGK%>D z1wzkj_1*MKnDQLrc@Dz-Q3DPaw#Sz+!fLo)#61%d@LVur+&FCl`735C-%PCuL4h>Q znX!us(?KP_$B9o%A!0tA^bMXodJ%DhC(?-u+f}G_yD|(RD=YTR92H#wR&P!Sh+ld7 zirsRiJ?UyK#tQ9-YTWKR2%GUoQoFxT%}yKVbPZD=&@jDxmOwheSo{)j=ZB% zT?gwvd<@VoK3P`u83~A&gDy4?s8j)C8m|iH8D?$s;?pWXz~7*^xrV0J)2x3wnE;hi z&_?+5q0BHo<|nLPa3$SA*qeFJFQ@)>_)5+^p4L%K7dHax$B-Ve!Tznsx^L)4rao#E zD2K+*NBam>kgk!LdG&vR=JG_9+ne+bMSIqmo{&BrT$4imW0^=lO2>?uL+t8jbYaUQ`FoMSP6>eES$s*W=kY^!PRi`fHQF67?j6Q$*@1?bQ1HCCEruZCA|D z3jdSffQml|jB8x&wzaiVDg7_YnI!gzln=`4vb;YxE|9JPmbC05p>#XfWoM67_Fz5$}or0an=7wT&CFeXSq zH5g82zJrgmc6fDH&kltWx+eg46PY{HM%Kg;Hr$2+jY&)wP!`&?k)49n#@8X7z)zY> ze&K`{;@#R(^tZ^L4}79N;pArAmcbkv@f_(1#m%r2P7+eXfaE%kn8Xd)72Uf zQ=m1FLrh_%0zTjGI&mM%n171=DP4uGnt8z=nNa^lG!Pc*V$Q!tg*j}ZKt!zhg;GKD zYikTKU@eLR4T_181Al!H&=?^`Jcqp-NP!|Gxl_lqd2%}xnkRc};I(7KXf8^mJUzBE zL(lFq>ZmXcamM(>prs{sR0uuCbUjNs1rBAAzb*&}lwcu!muC$^JoOpEzKnDT5FgZy z5COuV1n0X#!`kyZ6m1z{7y0$A{-qtjjw6D-znw^dy4l%KM-SZr&hN-R+WLgR zaYLg4I)s}k*|eX|b_5Y;1BHAF%(aRzI`N<&=Mn`pb&8+_K^?2wn>Td&>bJ2ys*_D} z%{g#ihgR9QgPr!s#xmV?rdP;og|T1hAn%MwIBO>R#HXlHuH#77{`6n(fRupz3edjt zv~2Xwci(=G9hF`f`TE0qcmFnO+v^8+R9go|eYoXT!oD4sju@R;Z96t&?(h|_D&OQf zZj71z_0IIS?HQAk7Js>9@vjxi4V}a9-`kgHcI1`ljhC%mSgc<+t@l~J^KbknuaX}( zX8iE!$ni3>6QKd(8ry0+izUna1I)z77aiYI@wLkm$V9olYLrwFk^H0ghrgBP_Vbwu zSXQSuN0#wXY*L}a`TM!zq$`{Szh+wkg+LtcO=8m-ZxY*x8U9a z$+_KyP3t3)=LW(?ng_4l!~N!S!)Gh6jF7oGh{kbiI)`(e(cC%vTz6#LElzp8$|ANb z$evb#C{uIa#YL<=-5#C~Zv!K+C1Ws5nPx4h(Li0UBy8m=4bF}_U^ax4Y^#K&;L}T-JK|iQARmPW*O%>Q4)CJ>XQ!9p=1{ z70dPhp;V^!lZHL;8d@0>X`kn|!X*E!qSJ4SEF{NkrIk&%tJ(_tdji2K@8&NJG-W=0 z;#g90gZFjSWrK&syAQ7`KVPL+9xJ19uI8L*onHivJ z=90gdu`4XoOIC@-1^e!w1s~D@OG}qq^8k}G;+fY%b3d#sJjjzTP1HEv$|GdcA4 z_s)l(c;xUXzI@?*dKx4jm|D&nboLshTq6J^EjvKXoWI@Jw!A|@O zd?(aOO`SJM&s<6j?q^vy*6BBzO@sHjkDa2E7F-DCJ8h;roD2x5PM(s>aq}&EP+E6& zf<@xI;-$8Uq4AmdF5m6heO`X=Mx95r&DQVY^M8#nY(Z15#P3GN=i05h9G9>BDs^El zE!Db_Ztf5?y2>IlW%p#mr3H)(WeZlEyJ`jQ#gZBwol^cCNaPQ9&6q?p2hR&~(!~lv zsLEL*+Z=R;&zY7y5q{!r^Alp#B#mry?);2gq`YI8kp#MyZRV{mf0VQ$;m2UhlGsfr z{%8m)Vr3EU+Qg|jlvt-)5J4Dgr4Wgdbz3gDl5ifim}V)v6uGEz>a&z-$kP?qKGuJR_{cXJ-T%uomD1Gn3gY`}hOKBYb75^KP_2%R`FFQ@Q;-Mbm zeQsLT~cly^{AL|R}y?rvb zHEm}>`?91A`|}pusqRHlbMMVJw`x2kXH`q* zjBvp{C9m67DzZ*YfqIiqTSrM+Y1~!A3RG%Ww-Uo~T-nuA`x>h3Zq|SL!{{F_81Hn% zxvmF-@A&!8arx5Vjdq!9KUcxZ*Ol2;s}j%F?{_=@jw&WuS5$R|NQwwk zguQpF9yEtmB#?X4&da*3?MiWYXcZI{vffoZdYf;2+SU$z*J&lkeA2G6zZM)+9~C5H zRUtQ17c;7z9lvUwRs1|BUQa?lT>izIxNo4T$(Q-o3ZOiGY6s)%xP3Awpg^G1Xba*e z&+dMn&pK)Oh$kAr+t&zYr8Us@HW(*>1MR^oYst^_Z!j0`9iAMVKK)gaw7Fp4I(3bh zlPunAuK%Fb-ftOvrZYzJCD}^7nL9u@U*nIU2mKM~ay7jxv0(dQNKLJiMrky2G+&V| zg-WA{tZXm*(w^7TaRix@u@HSL;#^eS^4(i&>tzRTdftMTlK0yf4wHl|_=}kZOaCO* zzqs+*@1Q%%;2~nxoxs4qQi2#J_sAkT%g<#|snWk^iMEhD5#D;7( zTvqL?3R8N0>SD6FZtdRMQ;#R%)+Oc9i#Z~z@UVct9X?#%&zooEG3>x6{PC zu%!#7?LN1jq?bb@?mga#3xZh24KWssTDx+^tECkr&w>@r0>ZRu?!5(+h(Qa3{zr-$ zt@$C|S@FJL@rUxmzsikLkR?<6R?pO=ARDgm+3{4ddGk--zzqBL^yxQiV%5lD zgRiS@cD5*-*o!Jw@kLqXw)62qefx1gtoqV5_=&?h=9!7Oi{FNCnK3F?VO1$3E_rk4 zrxTQaf0hz?)7AR0*_9O$@43x1lCIw$glGHBcHi?QG1dj^{+VUoOvArL_iN?FHOR^z z#haH#oO*C$m$^A-GrDVufz8?CJ=2i0{_-Z`_AV5~r$j9l@ql=p4wG`ynTkmyzfqDm zJtqvcWgDU?(LhjcPb02UC^BObvQl)clN!O+!c$Jvs}C%3r$T!-?;c?>MvWe}Rl2B& zI0nt9MicYfW`Clr_NO#fc^O(HAp10*bxTJCqQ*3TH=8&D&4SA$|A5I#sv`}`w3#6B zuW`JreUL(iVSbygnW(89CC=u4;ys&jk_vmaJ~l#YbFUx@B2XO0|B=-F%@tpH&x}kc z;h_U38apnjxTgerU0u{)_>0_M4 zIsc8JsM5uguzhroVW6{BPmZyuzpj~tzRrb`A#4?~c>==5=TZsY;3a?Kvps#kj-5M& z8V|IYcmnJV4=5FyNTs^AEj9Y}?E+ofc04tT1cRTTjF1BnRkyxXKb7Kj$Md zs}Yn|6^3R&l3ATw7xlsWorO_-_@Wsh=O7yQ?5W2w19SFESaH&q-b#cXF3$}rE^hw8 zG5c`3Kky7ew?*H|$zLk+1BuxF)C?fCiF4c)ucB{|E(nyOMq)!WPXc&q1sB1P%6;1~ zHCrjAtJR>D4;YE1GAU1wgDQB82enDkGG0dK($ZA5idW~NR>Wn>cpzv&qe2DD)k&c7 zGAA~hEtOy%8^2SCIxs9fiHCMb0&+yB*Vl{m6`sx}lg~2&6MH5<{$T5>?eU(a4P`&@ z-U;`^Fn04dyH_EZ%4JVSZ~LNA0y6zj%rN}~{Kjsk!Uv+*2wqbRaOO&Bu{Fr!c?@}0 zlY5%CVLnI+0Ra&Y=%`ZwYqqbnOoc$?WY$x|qg)IHZ5Z1h@O>kSB#3;6ln*4l$H*(5 zoeKQ|3F42bJd63j9Wdp&V)SQ4Sz`lJrXTKvmG2ZtKm@N7MPyP*Yy=}$1mu+x6Z=CM zuQ4rvkx~wS=uR0Qyn(U{4dW57tE@D@4!bK)bge=;{>^+aOA^}*R1<2CVgW$L0ogUv zHqJp5V8o8B%aF%g8x%G`9m1|omF|Mk+R!r~z&i>7wQ)?u8&kH{oWCJlz`J%L_jBBt;(y~P1&=X`$*;njhB zxl?fs6Scg<1X4+@cfA;)HMhfESkt%z=}dgU@;r5o2i$M$ql3-cxTW>y{6V@Ncz!q0 z7un+pk9DR9+=2%t6@c<)FM!BP;YmU{*JQvg$tLU;kYtN)g0sQdwBdjU_&`oKxiPJd zlM~SlBsX}qiNI8ia<+GXR3PVz0K)P_QIL4+*X z0Ke%peE;UsPIzKj7vPW3yf1+du&{mQKLT3}F)YWwRTB`&rMz+!F>D9uE#Yz?Qb`dx z$UFeREnzDL-Hm{!5XgT1b%>LAmiPw<1dsWKDRRAKO(%9 zc|i>1^REtvj?RsA5Rkw8whh(>^zQiVfD$ZM^;ZlOv-quv3q?DaBh*?%JLGrt;BJCY zZUxqQ$_~DHd)*{lf<;u#PANzqRBI@a_^b?hl@*c*2@dZlP6MB2j+4rs)9&B1)tF=E!zCo14wR5sD-yXtYPcjo zO~3zUyULi3D)So?k%|}Sc=P*UYMX+m&;**6aVPvlpJc&b7OvW(!c}|rn62*1ZorcJt+15#iBh|^!jMZ8aTcmfxA!T%81?7-aI%G)S3|Zkv(B{Q@}RsRlD@q? zE+olLO$YQWWiao`gY%%ku3co{0;UCLcc=g}n2+VZS61^@!9vD)MXicge9Hqk6`-Q+ z=#p__X2XVxYFmFG>JamkT%+kS{pfUcKual5hl62iwIrK~N`cA(Fth04i$Ea#4NS>F z5<@)j4~?Y**48Y8jsVcGU$8s{!%FiKP_@0Q(998?Qz_^9w(!*n^0<^@lp|@vBqmD020gxFfOeHB94Hi00tb8-WB6q;#71YyzpVjI|jp`3s(i;*KPtx zGse?v19WbI95h#%A8SH@96P9X+PX_vgd#Kyjeab|=6Ga_KuA)M@Ylny6->wMPl1nu z{AB4TTAA|C1$>_OxTN^0Ky6!!V41aIW=Az5Z?JO2vVoEhDi4mi8^TkAW7ii|7vJPt z^LXEbj@)4S^KSOai3cJa+`M}9rEsPSB-B&Winbd zE@;x?bsP^z_j&yjr!HNav3+!H%*0Ft10_z;c*O619a%T|g|^n!@Bu<;MH20WX1E1Z zO7qQ}TY=u=ZXBeXK(>WOC255q;8G|zLf5x8+h?5^q@9r71cg4fefSwzGL7SpECEfy zG~{uw3<})6JN&Dx7;CRk+Icfl>+*&8Lp(d7SXiX?J>Mii86RSbMax%rwulZI%{(Hp$jm zWSyk-h;GmMmxXp>lI8+UJp(V$mF6h66BE$r*!5n0KW+iYR*#zr^w3v$h_pD7mw_q8 zz0!Bt>fo6iE~eqp8O#(hnPAPI8;fED^R&)#je|^_?KgD^8%O#k36Sclw}h}7+khKW4PyI0C^G5F%H_!`jY!CBswmzE-D089yBV5*SN zS*pj-vn2Oyp5#8*k%CmXf*6%W* zYU9vKVwxIGAFttZ<3!B+h^i`=*6~6{tSv=>d{3K&UmQ>6Y=5OJw3G=lUdWsHeGisY zOCKRnr3#+QYE@$+{Vsq3^l9rziJZp0fGjhZF^DHQy$?^)>j4i_gZM6?qMnroEbf)3 zG#;RV7vnIvTYD`bM5Acf6^tykz%rXt;LYMXa|fDxz)y+hsJV21^^RVFEp|8vP|RG5 z(Uimehbqh!v~}ie)Dr$1z0B5T@v^XEYMj|=5ZvMqT*DQ{Rj)i7sNPt*rjL4~Sl!46 z|7KaPQK)PcY6UDNJcC+d>DqAfE~T&@UklO1m7VRXAO#yycV_p>Q&y}1deU{r)%5le zYOKJPhqWmiIlYR8dQ$pVeih1KxHNcm`XPtW5ei&>#>@b3K&+6me;xkIK)t_&V|^q? z`HE5wAkdTQS0~|WV(o(7y*p4G4zJL6Cu&6$3lhNOUr!u|S$%80Twf}}I$HyJB@HuR ztEDzzqynka>qNsj{%DP?lrDLpfk&%FURPwuXkFV?fhm=GZHVUa-cg~sF14bQBfcJ_ z;5ed>f@7pKkq-Xd@&#ApbY^HL-%o<_aL@5}-;m^cvfVIEyR`P;(*rjI-6+MIEh*a>!7O= zdk$(Y_zX*Lq^m7mP%vq`OwUL*wG>&Y%td46PdOie6yK+8OkB5jrKzo{vEkU(&^evq z9kWAeuRAH`XHA8h{J=YAe7#2carrQ;R#Si9UN=G0mc)j2ANQGjJq0pZujSv(h(9iY z`lJNdMtp<6KTXRe)Fo%hIy_Wto_jUUi#YeMP~s&GY&^V-aCIYH>l7_MHCZcflw{Ye zTQOTl>eNP+)G0{2m{`~K1DRWth*ik~t<-?{M*RA5*M?LjUEXh{b@O^o44t%li#F&n z$#TRzPwldXd!Vjv;0-8*HzcWrWc5wGN0!#TE5v71W%(4@yR)rL2~(%d?*w}C+7><> zq)Sh-#3{K>qfGFV3IEE619eBcx@X>0saT!0Gk7!=YpGV<5VAefO)0b?Xfc6UX>^jfBQ{?p$%+EmgUYaxS_R;uhC-2gkealboVyS^_jmw zTowqLlZ13~UMoYm{Tulicq3nGsh$I}fr5@n*dPh`bho%&)xkBqbPR$_WXSj-x(|+( zA{DaUGP}&%Tl!=iqEO6fdOwVR;*ffx2!|<+ydza#$JTxL82mr=9`0*;4}_PGF8$6& zuxUED0S-0kbqqDG;X_8~`tmZm{Pi8$C&Y_ZV7vAtZaEMQ&PoK6o?9qCy+So@6PLTd_nd+fdVD zNu)~GjMbri@8WZ~`%u5?zcjP~bYFR5@#;6t1HF$jql}zWhb7zJ_Q~vMmVfj0=~J8M?y857SVno`RjLlB6m7jN^h8c zC%NiWvC?O|{@3CDkA5jZ3$Lr|>~_>&GB@729ly6*j@|LZN+iHiqVbi%FFyRm`$07I zWW2+Lk7KhNJ(f-tew@!(N*nWQ-sa3;x--|*kr#dQJ8?vsbr4ILvb`d68?;2a!HS#P z<}vAJQAS;IbgFC&5N+LjtFEAGQXz(0Ijy&ofzhfhL z68iq7FpXI~Ei+;CFV^&Z&l<43pZT|6zV5b>GxeuMIO@7{O{FPlkb`K?eo!FMZ;~;; zmR$Y#-Qr4j(+j6qvAYc~1iw~QeWfCoqn3`5>|I#n9mgt;3eR#0^)xJg$)aQdBts z4kV8ln(TdIEd>Y6A;=)x=XP;kO*UHJDrD`!9iR5UT`jFtLv&2M24bLsgZKDuxYKp| z9f?XW{64P&;=~ghUv(|!hoz>`-#SlQ_1=Q%55?=RY6Ob?H+DYYDN#@->}UI9-0pkK zb@}{|JU1_)&7SCR=dum8(Lb-ebw%DfC&EwVnS8!2V-wf8&{~v|pB}pJyFC_~!=MruC$fIj;MD1-SFvSBGc*05O-Gi^2y69K#2mD=pw6 zo(7E#^C(u+FXEW4>jo`u2NeepA3tum18$un7^(1EiDL^IG>9&2qyqc0dlLOTxJ89! zzbj3Y!%1{w+UoN5q<0gVgTJb=3?8*%gzVP%MvcQBZ9(?DpKmL; zw`=Ki1~;UvZqL?^vx>coZ4w#|tMlkhO`^iB_oOz$SPSFBf<+QWf?e1|2kxc`-ljoY zof!eS*P57_3^G5(CYzsRrdv07SNteQu8U<0YJ~aAKT4X_=n*)J?PNE`a>NOVI$i{VD>R%4_&qf*|8+d$$@WQ0FMJ$O+O%s$ z+J*C(yUgD@84~Y9LyB1LTz|`b^Z2Ij;gf{zF6kM*x1giQE}@NGb<^%`Vfqms6S!HG zz%l+k{Y>v^mYaED3$EBcdvp>4IUo*3wRSB8|Cqm@9rU8tQ3^!NlG)O=a7H9E)XiGU;CGtHoZ6t z{-#?jMp&miW3&f2GUPKoFOyq-sq|G@f`SEj+;utbXMPb!l4@`vpCb?fbF&-i(_z_p z80-j9l&T=o$9?+@meS!(d%0zHTeRhge;renf9<{me*4>cqeKdO9-w9R!yT%>@@Fqj z*t2`-g|Uf(y!~#=7W(G-8BLyJFEhugMGrc(T^|t^*NphmS%HoMo=@hLZSA$L(>9}D z-neZ3jpH&b7wX$Dc|Q6daHXBq&pkRi{amxM=i%X#>GoT4crB>(!{L>(BXJ@50jlDv zbm4uC4ek27;%o)P;@NA!l^qbFX5Wq5f(sVOX)7)unY0W&yKp!jnMm+$6OC?mQ>k3 zaY)e`cnqsx90X|ssdY#S``ShfLv@bjrNIeKDFbYV$EIt9 znd>fZ3M4$q=UmX!k3Na?Y*?`{tzO{aEk6jINAIRZMBNoQJp4#H3t|9B|0rV|_B9gK zS^5qJIRc*da59(66LPfXJDPI&!)VgqlCxc00MTB}2)Af8&LDS67l;twkqp?{A$ztn zd!dQe`=ev&qJkis@#HT+W)~FcxIQeXaD5a%A+9C2e9+p#u2@ax_Kx$D3gR`Mw<{DWpJr_1;GuU-RX^idsmX4aDi%tqv6EWO@8VjQlVQdiJOzk&#GdbPEzloV)5f zPvE8?G9NO}L3V-tS}QFWb(nz^c#JoQAEOusR??RsTu0*8$K5v4&(UK-v^NFdR^>~# zkpDQwfNJ9BXmFbJsd*kxM^8YX3OxZX&;=X(`Wg9Au8OaSfC8RDW_TspOllZbUYdY- zJRqRJc)*2Clm+mm5Ufo0OlQJ19ch5`1SFf>xX9V4Ct-O-0fK}T_a<@S!5W`K_6kyA zEWn5m=Nl*xa9CXJlVuG(!iWkzNTm}okS<~Y1y-rF_0+mb#;^bKUNub%(S=rmY+D5- z08s9H6tvTdK_jSh0*(Fg*oSaD_JiO~8sGvk0<9$CfC4G&W=c%M#2vf@&_Xvt!Gz2N zy@#l6BYm1$tn!gPs~!BOB|ro zo4Et9yQn=Fv^#-r16N0ABxE5K!3+wP@1WoQcUcCU5>5=5M4ZCc?a?-cKKv{uJgkf`2z!6Kp7d-(D=~*K zC(4>Y{%My*8gafr1&SK*&(=o6gS;j)wmUxRk}yb3HJqhACkl0Sj-)~p&q}&q>qUz5 zoAo1XC@?u_rt7GugUH{G&683-t~z6UV$h4=H2M2mXH$}{X(2T!+79EBXV|JFJ?JIk z5iu9W@dMKHM2ezbU|wpfiiRR_Q=uFCKs!;S3;#+5f^*R<0RjHinc>z(fg@Q@kI)u) zx^zI?0x7V{CRW-DB-&@@66?Vt8C2+xgzX{j2SAE5`R@>C?Z8RA2NcOAzv{qj7lNK8 z&iVOvqPE5ki9|Ar5FI26Domj1^@JIZi5ckuAy$HGgcnHGbMi9{o36e{>V_7QvYDf; zy&l{^PcV2q?inUHhat+~dLTpElYo1FakpEzxSQBg9Sj6O$^#-$4Ai{m-I`Kk!66m1 z6zQaeVMrjObVLIdC!>wUq=z$oT0|B=bmI;}55hao2t9F3D-%NzBmQNu$gB$FBDloc z-zt6mFN1M!bbwlo0NPW-Q3`)tXzhr`(+eu%NY`Psj^We(T@fD#)GACD$;+7F_krbp zRQ}}vQbVWn_^bnU^fk zbs}$2phFU|Q!6$P-ja$>$p0K;K(da+H7|<%iK`9LwfCE z4_~e45g)dTg%G<=cE#HwGwmbY%03Y_)OA&};_l>%uiModJY z0v$1VFR_GZRPpI&O`?pWWEP=*;;dcbU!$~5iN1$(@APsgAA`^c3S_Mc9cV2>7c-iZTV@|({L^aZo4McY+#cHAd0!E!T6U<-O@85h zF@zC`UXuQr7DFLce?y@RG4n9^J%&QbolcETeJ#G?Lq&aU%%TL};HBK*m*WlPr^VP~ zOMgQ;U#vv_`wZz!9O-=x=@^;n7v4~s+t^v7+LiTkc=K698`8PO_cx?l1sYVp$B-_@ zB2BieugzCUKD)15n|>GhQT~!-=Ct@GIrjHWk|^v&zsEOewu^g;|N6c*Up|Wdh7j{{ z?hPV(Io^U=jFZwf_cu<8L}cjiF-|Ia#?-N4yVS19yP`|4rHUpo0%xc=M+&4DW)Z4oG@5 z3M?dlkP#`w4{{bIQXUT-*(>zGB7NNXq*{B%gV!OE`gn7<@Z~rR{z=7_m1)i+{7Akq z#tlRfp>br$dLq6r^37oOc`B{t2i?024I?>7WGc z3Rw0x$b8Cbxa4eH0J0V1W=e3f7!BhCIVeN`Xh+f?_j7cwki-MrYFhPnwJKgls|N7x z>-0N=qHH;mrWWn}%)h$>+vjqnyGO-BX{HAtOvNMwbO+|7c|;?B=2)66A|q_Ak9^3N`&z3ex8Yx( z2lo>W{FCMgn3eVjj59VgylZl23)P|)qE4QojSkxeULAwSL8{AGRm0ISGn+*T?H~wQ zhVs;0^f|A*a>6T+UTbKw_ldO>99Z-LqzEgXmn45LbAN7H~a8gL9jwkH~>0}6qYLnU3wjym>pdo1eh@->hkfF0)h5G*RM0>Ql3 z;kOuhN2T5fX3l;Neoilhf4|L~(1xA?0kUBVde<%yLv_sOBUbttzXku^^XeRfR(!s# z;NGsK(;3{5vbsH6JI*ThF1AT%NK%G)^k(}Sl-z)hm{_P`une)W@!tr~;Ty+mVDhhX zoVZE3;vngCoAL06c1CTqb1hcWFY1kN1J?~&+zu)ZAU=NFaEDjnM+GC{MT6fh*Lg4H zqDha?deL;L?zmB51Q_+0FhgvtB!Ba^Xs1dW_MnSvrBOj2Vy6m{u`eF$lOhD)@YAtI zsJJ5_`H-`ot{`mrSr}_n0>}*_u}bRDx=6hCW_0lZNZw#5Gd9x(lz}7pZhwjAq{hjp zrHM^Mg^D!lP4c(=_$#FBD>uIrN2FN?v6Ly>`yrmTp7z*^JNQ-UZHpUP)52mcntPqO z>ny4VLO=mfLJ4qTZ(r%ztL55Hg-Z^lKj zToU%sY<69|3WKRw&+CY&STg3ZOkjlXBXHpsOrNqje`AtZHq6;d86qh)8E*$9@Lw0*#G3g+vJ zO)t)Zv8BaggmtsX#u{^D?>Rmr7riCFs}OZ1a}UV0eU<34X{;>{z#mRp*~G zCOaFey}8#_bHz}gB~(V&{v;SK`DO-tK?%(dV%Robn zH@+0d?LP`}@rh^qifxbsfJx@CZ8s3qHCRDgS+L=H*u9Q0UGDblA z(m;M3DbU|>)=_|@!wu^K$c=}I0dCq)C64)_1T(=3oYMhO(|Ej)o{LFLI9))S3S}Wl zjy8OvDAhVw$N54Q+=*1^M`F4F-x})-k{8V`?!-*RsA4LxqPWFcJCd4NNSD79*<;UM z;@IOx1v*XBRbr4`kPRi8%iw=uo(__A#2zZtT*;2tw8kEd!dJxH3M_BxWipSN_vjim zYfLpsjx!8Mv)K-KZ&6fvlv> zss=k&J*}^9tHjznS=~tW@okC&*cW%jB?ttrA3?|W^N#wH8g^l*4pEqhS-3kE$y1&J z>dy&^w&;>(pg`_g&Adk3U*-$!)MH3~Bvn%*wJ`KFi7PeGgOjw5(wj6|e|Z#OhhPdP zt+RBz!JJXIo&o|!&f0rCc;^)=iL`duR;=E<$Va;J^cDZ*4xe&Ten~>gX=VQ9y31Em6B{0p8;AW_&=}-CDo5GXj^pxwfT?_MG*Rv`0TxczUZd z_goEZoIkMgiy4^8*#T+eL01_)JyNz_8Xr$n(5JyosCjiR0_;&iRp_vBlV1HlezSD$ diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/example.sh b/cmd/pilosa-fsck/release-pilosa-fsck/example.sh deleted file mode 100755 index 79fd4cf11..000000000 --- a/cmd/pilosa-fsck/release-pilosa-fsck/example.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set +x -export PATH=.:${PATH} - -# unpack the sample Molecula Pilosa cluster. -tar xf backups.tar.gz - - -# check if repair is needed. -pilosa-fsck -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa - - -# yes, so do the repairs. This can be done first (only) as well. -# -pilosa-fsck -fix -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa - - -# check again if you like -# -pilosa-fsck -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa diff --git a/cmd/pilosa-fsck/vprint.go b/cmd/pilosa-fsck/vprint.go deleted file mode 100644 index 83b1681f7..000000000 --- a/cmd/pilosa-fsck/vprint.go +++ /dev/null @@ -1,177 +0,0 @@ -// home: https://github.com/glycerine/vprint -// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved. -// License: MIT -// -// MIT License -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package main - -import ( - "fmt" - "io" - "os" - "path" - "runtime" - "runtime/debug" - "sync" - "time" -) - -const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00" -const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00" - -// for tons of debug output -var VerboseVerbose bool = false - -// convience functions for . import -var pp = PP -var vv = VV - -var panicOn = PanicOn - -func init() { - // keeper linter happy - _ = pp - _ = vv -} - -func PanicOn(err error) { - if err != nil { - panic(err) - } -} - -func PP(format string, a ...interface{}) { - if VerboseVerbose { - TSPrintf(format, a...) - } -} - -func VV(format string, a ...interface{}) { - TSPrintf(format, a...) -} - -func AlwaysPrintf(format string, a ...interface{}) { - TSPrintf(format, a...) -} - -var tsPrintfMut sync.Mutex - -// time-stamped printf -func TSPrintf(format string, a ...interface{}) { - tsPrintfMut.Lock() - Printf("# %s %s ", FileLine(3), ts()) - Printf(format+"\n", a...) - tsPrintfMut.Unlock() -} - -// get timestamp for logging purposes -func ts() string { - return time.Now().Format(RFC3339UsecTz0) -} - -// so we can multi write easily, use our own printf -var OurStdout io.Writer = os.Stdout - -// Printf formats according to a format specifier and writes to standard output. -// It returns the number of bytes written and any write error encountered. -func Printf(format string, a ...interface{}) (n int, err error) { - return fmt.Fprintf(OurStdout, format, a...) -} - -func FileLine(depth int) string { - _, fileName, fileLine, ok := runtime.Caller(depth) - var s string - if ok { - s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine) - } else { - s = "" - } - return s -} - -func stack() string { - return string(debug.Stack()) -} - -func FileExists(name string) bool { - fi, err := os.Stat(name) - if err != nil { - return false - } - if fi.IsDir() { - return false - } - return true -} - -func DirExists(name string) bool { - fi, err := os.Stat(name) - if err != nil { - return false - } - if fi.IsDir() { - return true - } - return false -} - -func FileSize(name string) int64 { - fi, err := os.Stat(name) - if err != nil { - return 0 - } - return fi.Size() -} - -// Caller returns the name of the calling function. -func Caller(upStack int) string { - // elide ourself and runtime.Callers - target := upStack + 2 - - pc := make([]uintptr, target+2) - n := runtime.Callers(0, pc) - - f := runtime.Frame{Function: "unknown"} - if n > 0 { - frames := runtime.CallersFrames(pc[:n]) - for i := 0; i <= target; i++ { - contender, more := frames.Next() - if i == target { - f = contender - } - if !more { - break - } - } - } - return f.Function -} - -// happy linter: -var _ = DirExists -var _ = FileExists -var _ = Caller -var _ = stack -var _ = RFC3339MsecTz0 -var _ = RFC3339UsecTz0 -var _ = AlwaysPrintf -var _ = FileSize