From 90c424a4d939fb24f1cbb0d46c63d0d47393b224 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@seebs.net>
Date: Wed, 30 Jun 2021 11:41:28 -0500
Subject: [PATCH] separate backup of index data and index translation keys

It's not enough to back up each index's translation keys after
backing up that index's data; we also have to back them up after
backing up any index data from indexes which have foreign key
references to that index. So we do the per-index passes separately.

Since the individual backup data files are being created separately,
the expected output is unchanged for a quiescent database, the only
difference is that the amount of translation info which might be
newer than the data stored for shards is potentially increased.
---
 ctl/backup.go | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/ctl/backup.go b/ctl/backup.go
index 1ce3c8e88..26507c36a 100644
--- a/ctl/backup.go
+++ b/ctl/backup.go
@@ -109,7 +109,16 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) {
 
 	// Backup data for each index.
 	for _, ii := range schema.Indexes {
-		if err := cmd.backupIndex(ctx, ii); err != nil {
+		if err := cmd.backupIndexData(ctx, ii); err != nil {
+			return err
+		}
+	}
+	// Backup translation data. This has to happen separately, because
+	// otherwise a field which uses foreign key translation can reasonably
+	// contain values which got created for the foreign index after we
+	// backed up that index.
+	for _, ii := range schema.Indexes {
+		if err := cmd.backupIndexTranslation(ctx, ii); err != nil {
 			return err
 		}
 	}
@@ -158,16 +167,13 @@ func (cmd *BackupCommand) backupIDAllocData(ctx context.Context) error {
 	return f.Close()
 }
 
-// backupIndex backs up all shards for a given index.
-func (cmd *BackupCommand) backupIndex(ctx context.Context, ii *pilosa.IndexInfo) error {
+// backupIndexTranslation backs up both field and index-wide key translation for
+// the given index. it has to run after the index's data has been backed up,
+// but also after the data for any index which might have a foreign-key
+// relation to this index has been backed up.
+func (cmd *BackupCommand) backupIndexTranslation(ctx context.Context, ii *pilosa.IndexInfo) error {
 	logger := cmd.Logger()
-	logger.Printf("backing up index: %q", ii.Name)
-
-	if err := cmd.backupShards(ctx, ii); err != nil {
-		return err
-	}
-
-	// Back up translation data after bitmap data so we ensure we can translate all data.
+	logger.Printf("backing up index translation: %q", ii.Name)
 	if err := cmd.backupIndexTranslateData(ctx, ii.Name); err != nil {
 		return err
 	}
@@ -182,7 +188,10 @@ func (cmd *BackupCommand) backupIndex(ctx context.Context, ii *pilosa.IndexInfo)
 	return nil
 }
 
-func (cmd *BackupCommand) backupShards(ctx context.Context, ii *pilosa.IndexInfo) error {
+// backupIndexData backs up all shard data for a given index.
+func (cmd *BackupCommand) backupIndexData(ctx context.Context, ii *pilosa.IndexInfo) error {
+	logger := cmd.Logger()
+	logger.Printf("backing up index data: %q", ii.Name)
 	shards, err := cmd.client.AvailableShards(ctx, ii.Name)
 	if err != nil {
 		return fmt.Errorf("cannot find available shards for index %q: %w", ii.Name, err)