From 51744dc77a4e91452c8472132a1596dc66369de0 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@seebs.net>
Date: Thu, 4 Mar 2021 13:32:11 -0600
Subject: [PATCH] check for nil translate store while reading translate entries

If we are using replication, we can be a replica translate store for a
partition, which means we start a translate store reader to replicate
data for it. The translation logic does not admit *stopping* the
translate reader, only "resetting" it (stopping and immediately
restarting), so the translate reader just runs until it hits an error
and terminates, which it does even if perhaps it shouldn't. Oops.

Anyway, one potential failure mode is that if you hit timing just
right, you can end up trying to process translation *while* the
index is being closed, and the index can close its translation stores,
and make them all nil, right before we request a store and try to use
it. Another is a similar error, but during the initial startup of the
translate store readers. Either way, we want to error out of the
process cleanly if this happens.

This could also happen during initial creation, perhaps.

We're aborting translation sync on these errors, because otherwise
we'd continue accepting new keys, and then end up with our highest
known key being higher than some keys we missed; this way the next
restart will restart from the last key we have.
---
 holder.go | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/holder.go b/holder.go
index 38a1df3aa..18ae027ec 100644
--- a/holder.go
+++ b/holder.go
@@ -1883,6 +1883,9 @@ func (s *holderSyncer) initializeIndexTranslateReplication(snap *topology.Cluste
 				}
 
 				store := index.TranslateStore(partitionID)
+				if store == nil {
+					return fmt.Errorf("no store available for index %q, partition %d", index.Name(), partitionID)
+				}
 				offset, err := store.MaxID()
 				if err != nil {
 					return errors.Wrapf(err, "cannot determine max id for %q", index.Name())
@@ -1974,6 +1977,10 @@ func (s *holderSyncer) readIndexTranslateReader(rd TranslateEntryReader) {
 
 		// Apply replication to store.
 		store := idx.TranslateStore(snap.KeyToKeyPartition(entry.Index, entry.Key))
+		if store == nil {
+			s.Holder.Logger.Printf("no translate store suitable for index %q, key %q", entry.Index, entry.Key)
+			return
+		}
 		if err := store.ForceSet(entry.ID, entry.Key); err != nil {
 			s.Holder.Logger.Printf("cannot force set index translation data: %d=%q", entry.ID, entry.Key)
 			return
@@ -1998,6 +2005,10 @@ func (s *holderSyncer) readFieldTranslateReader(rd TranslateEntryReader) {
 
 		// Apply replication to store.
 		store := f.TranslateStore()
+		if store == nil {
+			s.Holder.Logger.Printf("no translate store suitable for index %q, field %q, key %q", entry.Index, entry.Field, entry.Key)
+			return
+		}
 		if err := store.ForceSet(entry.ID, entry.Key); err != nil {
 			s.Holder.Logger.Printf("cannot force set field translation data: %d=%q", entry.ID, entry.Key)
 			return