From adb21589f048ee7bbbfdf66ea8464b54fe8fe0b3 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Wed, 8 Apr 2020 10:09:49 -0500
Subject: [PATCH 1/2] Support multiple dialTargets and cycle through on
 connection reset

---
 api/client/grpc.go | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/api/client/grpc.go b/api/client/grpc.go
index 1c69e7b33..d6613d0de 100644
--- a/api/client/grpc.go
+++ b/api/client/grpc.go
@@ -31,18 +31,19 @@ const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody!
 
 // GRPCClient is a client for working with the gRPC server.
 type GRPCClient struct {
-	dialTarget string
-	tlsConfig  *tls.Config
+	dialTargets []string
+	tlsConfig   *tls.Config
 
-	mu   sync.RWMutex
-	conn *grpc.ClientConn
+	mu          sync.RWMutex
+	conn        *grpc.ClientConn
+	targetIndex int
 }
 
 // NewGRPCClient returns a new instance of GRPCClient.
-func NewGRPCClient(dialTarget string, tlsConfig *tls.Config) (*GRPCClient, error) {
+func NewGRPCClient(dialTargets []string, tlsConfig *tls.Config) (*GRPCClient, error) {
 	c := &GRPCClient{
-		dialTarget: dialTarget,
-		tlsConfig:  tlsConfig,
+		dialTargets: dialTargets,
+		tlsConfig:   tlsConfig,
 	}
 	// resetConn sets GRPCClient.conn when it doesn't
 	// exist yet.
@@ -79,7 +80,8 @@ func (c *GRPCClient) resetConn() error {
 	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
 
 	var err error
-	if c.conn, err = grpc.Dial(c.dialTarget, opts...); err != nil {
+	if c.conn, err = grpc.Dial(c.dialTargets[c.targetIndex], opts...); err != nil {
+		c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets
 		return errors.Wrap(err, "creating new grpc client")
 	}
 

From 8f0ae1b6f553278b2eca177b37e29a7ee478b498 Mon Sep 17 00:00:00 2001
From: Travis <travis@pilosa.com>
Date: Wed, 8 Apr 2020 12:29:15 -0500
Subject: [PATCH 2/2] cycle through dial targets regardless of error

---
 api/client/grpc.go | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/api/client/grpc.go b/api/client/grpc.go
index d6613d0de..7f65f2222 100644
--- a/api/client/grpc.go
+++ b/api/client/grpc.go
@@ -80,14 +80,24 @@ func (c *GRPCClient) resetConn() error {
 	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
 
 	var err error
-	if c.conn, err = grpc.Dial(c.dialTargets[c.targetIndex], opts...); err != nil {
-		c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets
+	if c.conn, err = grpc.Dial(c.dialTargets[c.getTargetIndex()], opts...); err != nil {
 		return errors.Wrap(err, "creating new grpc client")
 	}
 
 	return nil
 }
 
+// getTargetIndex gets the current target index, then increments it for
+// next time. Unprotected.
+func (c *GRPCClient) getTargetIndex() int {
+	if len(c.dialTargets) == 0 {
+		return 0
+	}
+	ret := c.targetIndex
+	c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets
+	return ret
+}
+
 // Close closes any connections the client has opened.
 func (c *GRPCClient) Close() error {
 	c.mu.RLock()