Merge pull request #1727 from tgruben/wip-partial-restore

partial backup/restore

cmd/backup.go

@@ -40,6 +40,7 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 	flags.BoolVar(&cmd.NoSync, "no-sync", false, "disable file sync")
 	flags.IntVar(&cmd.Concurrency, "concurrency", cmd.Concurrency, "number of concurrent backup goroutines")
 	flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.")
+	flags.StringVar(&cmd.Index, "index", "", "index to backup, default backs up all indexes. ")
 	ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
 	return ccmd
 }

ctl/backup.go

@@ -24,7 +24,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/molecula/featurebase/v2"
+	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/http"
 	"github.com/molecula/featurebase/v2/server"
 	"github.com/molecula/featurebase/v2/topology"
@@ -38,6 +38,9 @@ type BackupCommand struct { // nolint: maligned
 	// Destination host and port.
 	Host string `json:"host"`
 
+	// Optional Index filter
+	Index string `json:"index"`
+
 	// Path to write the backup to.
 	OutputDir string
 
@@ -91,6 +94,19 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) {
 	if err != nil {
 		return fmt.Errorf("getting schema: %w", err)
 	}
+	if cmd.Index != "" {
+		for _, idx := range indexes {
+			if idx.Name == cmd.Index {
+				indexes = make([]*pilosa.IndexInfo, 0)
+				indexes = append(indexes, idx)
+				break
+			}
+		}
+		if len(indexes) <= 0 {
+			return fmt.Errorf("Index not found to back up")
+		}
+	}
+
 	schema := &pilosa.Schema{Indexes: indexes}
 
 	// Ensure output directory doesn't exist; then create output directory.

ctl/restore.go

@@ -17,6 +17,7 @@ package ctl
 import (
 	"context"
 	"crypto/tls"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -26,7 +27,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/molecula/featurebase/v2"
+	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/server"
 	"github.com/molecula/featurebase/v2/topology"
 	"golang.org/x/sync/errgroup"
@@ -102,7 +103,6 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) {
 	} else if err := cmd.restoreIDAlloc(ctx, primary); err != nil {
 		return fmt.Errorf("cannot restore idalloc: %w", err)
 	}
-
 	if err := cmd.restoreShards(ctx); err != nil {
 		return fmt.Errorf("cannot restore shards: %w", err)
 	} else if err := cmd.restoreIndexTranslation(ctx); err != nil {
@@ -128,11 +128,47 @@ func (cmd *RestoreCommand) restoreSchema(ctx context.Context, primary *topology.
 	}
 	defer f.Close()
 
-	cmd.Logger().Printf("Load Schema")
-	url := primary.URI.Path("/schema")
-
-	var client http.Client
-	_, err = client.Post(url, "application/json", f)
+	existingSchema, err := cmd.client.Schema(ctx)
+	if len(existingSchema) == 0 {
+		cmd.Logger().Printf("Load Schema")
+		url := primary.URI.Path("/schema")
+		var client http.Client
+		_, err = client.Post(url, "application/json", f)
+	} else {
+		schema := &pilosa.Schema{}
+		if err := json.NewDecoder(f).Decode(schema); err != nil {
+			if err != nil {
+				return err
+			}
+		}
+		exists := func(indexName string) bool {
+			for _, i := range existingSchema {
+				if i.Name == indexName {
+					return true
+				}
+			}
+			return false
+		}
+		logger := cmd.Logger()
+		//NOTE SHOULD ONLY BE ONE
+		for _, index := range schema.Indexes {
+			if exists(index.Name) {
+				return fmt.Errorf("Index Exists %v", index.Name)
+			}
+			logger.Printf("Create INDEX %v", index.Name)
+			err = cmd.client.CreateIndex(ctx, index.Name, index.Options)
+			if err != nil {
+				return err
+			}
+			for _, field := range index.Fields {
+				logger.Printf("Create Field %v", field.Name)
+				err = cmd.client.CreateFieldWithOptions(ctx, index.Name, field.Name, field.Options)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
 	return err
 }
 

testBackupRestore.sh

@@ -33,3 +33,25 @@ else
 	echo "FAIL Single"
 	exit 1
 fi
+
+datagen --source texas_health -e 9999 --pilosa.index newsink --pilosa.batch-size 10000 --pilosa.hosts pilosa0:10101
+before=$(/featurebase chksum --host pilosa0:10101)
+/featurebase backup -o newbackupdir --host pilosa0:10101 --index newsink
+curl -X DELETE -s pilosa0:10101/index/newsink
+/featurebase restore -s newbackupdir --host pilosa0:10101
+after=$(/featurebase chksum --host pilosa0:10101)
+if [ "$before" = "$after" ]; then
+	echo "PASS Cluster Table"
+else
+	echo "FAIL Single Table"
+	exit 1
+fi
+/featurebase restore -s newbackupdir --host pilosax:10101
+single=$(/featurebase chksum --host pilosax:10101)
+if [ "$before" = "$single" ]; then
+	echo "PASS Single Table"
+	exit 0
+else
+	echo "FAIL Single Table"
+	exit 1
+fi