From 0e37e04f47a372f0ff82119ac7945b9e7a3b03bb Mon Sep 17 00:00:00 2001 From: reesporte <45641995+reesporte@users.noreply.github.com> Date: Mon, 25 Apr 2022 14:23:27 -0500 Subject: [PATCH] check for context deadline exceeded in retry client (#2033) you can get a context deadline exceeded from clientv3 if there's heavy load and the etcd server sends a grpc code DeadlineExceeded. this causes the etcd client to not retry connecting and then you end up with nodes that can't recover. Co-authored-by: tgruben Co-authored-by: 54mir <48686912+54mir@users.noreply.github.com> Co-authored-by: seebs Co-authored-by: tgruben Co-authored-by: 54mir <48686912+54mir@users.noreply.github.com> Co-authored-by: seebs --- etcd/embed.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etcd/embed.go b/etcd/embed.go index cff332736..c60456dbf 100644 --- a/etcd/embed.go +++ b/etcd/embed.go @@ -225,7 +225,8 @@ func (e *Etcd) retryClient(fn func(cli *clientv3.Client) error) (err error) { cli = e.newClient(cli) break } - if !strings.Contains(msg, "etcdserver: request timed out") { + // check that the request hasn't timed out. this can happen with either of these errors + if !strings.Contains(msg, "etcdserver: request timed out") && !strings.Contains(msg, "context deadline exceeded") { // not a known error, also not a wrapped timeout return errors.Wrap(err, "non-retryable error") }