scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-07 03:17:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

separate backup of index data and index translation keys

Browse source 
It's not enough to back up each index's translation keys after
backing up that index's data; we also have to back them up after
backing up any index data from indexes which have foreign key
references to that index. So we do the per-index passes separately.

Since the individual backup data files are being created separately,
the expected output is unchanged for a quiescent database, the only
difference is that the amount of translation info which might be
newer than the data stored for shards is potentially increased.
This commit is contained in:
Seebs 2021-06-30 11:41:28 -05:00
parent a8161923de
commit 90c424a4d9
1 changed files with 20 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
ctl/backup.go
View file

@ -109,7 +109,16 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) {
// Backup data for each index.
for _, ii := range schema.Indexes {
if err := cmd.backupIndex(ctx, ii); err != nil {
if err := cmd.backupIndexData(ctx, ii); err != nil {
return err
}
}
// Backup translation data. This has to happen separately, because
// otherwise a field which uses foreign key translation can reasonably
// contain values which got created for the foreign index after we
// backed up that index.
for _, ii := range schema.Indexes {
if err := cmd.backupIndexTranslation(ctx, ii); err != nil {
return err
}
}
@ -158,16 +167,13 @@ func (cmd *BackupCommand) backupIDAllocData(ctx context.Context) error {
return f.Close()
}
// backupIndex backs up all shards for a given index.
func (cmd *BackupCommand) backupIndex(ctx context.Context, ii *pilosa.IndexInfo) error {
// backupIndexTranslation backs up both field and index-wide key translation for
// the given index. it has to run after the index's data has been backed up,
// but also after the data for any index which might have a foreign-key
// relation to this index has been backed up.
func (cmd *BackupCommand) backupIndexTranslation(ctx context.Context, ii *pilosa.IndexInfo) error {
logger := cmd.Logger()
logger.Printf("backing up index: %q", ii.Name)
if err := cmd.backupShards(ctx, ii); err != nil {
return err
}
// Back up translation data after bitmap data so we ensure we can translate all data.
logger.Printf("backing up index translation: %q", ii.Name)
if err := cmd.backupIndexTranslateData(ctx, ii.Name); err != nil {
return err
}
@ -182,7 +188,10 @@ func (cmd *BackupCommand) backupIndex(ctx context.Context, ii *pilosa.IndexInfo)
return nil
}
func (cmd *BackupCommand) backupShards(ctx context.Context, ii *pilosa.IndexInfo) error {
// backupIndexData backs up all shard data for a given index.
func (cmd *BackupCommand) backupIndexData(ctx context.Context, ii *pilosa.IndexInfo) error {
logger := cmd.Logger()
logger.Printf("backing up index data: %q", ii.Name)
shards, err := cmd.client.AvailableShards(ctx, ii.Name)
if err != nil {
return fmt.Errorf("cannot find available shards for index %q: %w", ii.Name, err)