From 7c6423587a65f212a0822697483ef8402b4c2ad4 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Tue, 4 May 2021 12:04:00 -0500
Subject: [PATCH 01/15] skeleton restore

---
 cmd/restore.go |  51 +++++++++++++++++++++
 ctl/restore.go | 122 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 173 insertions(+)
 create mode 100644 cmd/restore.go
 create mode 100644 ctl/restore.go

diff --git a/cmd/restore.go b/cmd/restore.go
new file mode 100644
index 000000000..c8a78f917
--- /dev/null
+++ b/cmd/restore.go
@@ -0,0 +1,51 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2/ctl"
+	"github.com/spf13/cobra"
+)
+
+func newRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	c := ctl.NewRestoreCommand(stdin, stdout, stderr)
+	restoreCmd := &cobra.Command{
+		Use:   "restore [flags] PATH ",
+		Short: "restore a backup",
+		Long: `
+		The restore command will take a backup archive and restore it to a new, clean cluster.
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) > 1 {
+				return fmt.Errorf("too many command line arguments")
+			}
+			c.Path = args[0]
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.Run(context.Background())
+		},
+	}
+	flags := restoreCmd.Flags()
+	flags.StringVarP(&c.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
+
+	return restoreCmd
+}
diff --git a/ctl/restore.go b/ctl/restore.go
new file mode 100644
index 000000000..c4e7a4974
--- /dev/null
+++ b/ctl/restore.go
@@ -0,0 +1,122 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctl
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"io"
+	"os"
+	"strings"
+
+	gohttp "net/http"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/http"
+)
+
+// RestoreCommand represents a command for restoring a backup to
+type RestoreCommand struct {
+	// Filepath to the backup file.
+	Path   string
+	Host   string
+	client *http.InternalClient
+	// Standard input/output
+	*pilosa.CmdIO
+}
+
+// NewRestoreCommand returns a new instance of RestoreCommand.
+func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand {
+	h := &gohttp.Client{}
+	host := "SOMETHING"
+	c, err := http.NewInternalClient(host, h)
+	if err != nil {
+		panic(err)
+	}
+
+	return &RestoreCommand{
+		CmdIO:  pilosa.NewCmdIO(stdin, stdout, stderr),
+		client: c,
+	}
+}
+
+/* helper to allow for both gz and just plan tar
+	f, err := os.Open(cmd.Path)
+	if err != nil {
+		return (err)
+	}
+	defer f.Close()
+	var tarReader *tar.Reader
+	if strings.HasSuffix(cmd.Path, "gz") {
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		tarReader = tar.NewReader(gzf)
+	} else {
+		tarReader = tar.NewReader(f)
+	}
+return
+}
+*/
+func readSchema(path string) string {
+	return "{}"
+}
+
+// Run executes the restore.
+func (cmd *RestoreCommand) Run(ctx context.Context) error {
+	f, err := os.Open(cmd.Path)
+	if err != nil {
+		return (err)
+	}
+	defer f.Close()
+	var tarReader *tar.Reader
+	if strings.HasSuffix(cmd.Path, "gz") {
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		tarReader = tar.NewReader(gzf)
+	} else {
+		tarReader = tar.NewReader(f)
+	}
+	schemaJson := readSchema(cmd.Path)
+	//Push the schema from the archive into the cluster belonging to the host.
+	client := &gohttp.Client{}
+	url := "FIXME"
+	_, err = client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson))
+	if err != nil {
+		return err
+	}
+
+	for {
+		header, err := tarReader.Next()
+		//fmt.Println("What %v", header.Name)
+		_ = header
+		if err == io.EOF {
+			break
+		}
+	}
+	/*	Fetch the cluster nodes from the target host.
+		For each index:
+		Upload the RBF snapshot for each shard to the nodes that own the shard.
+		Upload the index & field translation BoltDB snapshots to each node.
+		If possible, trigger the node to reload itself. Otherwise a restart would be required.
+	*/
+
+	return nil
+}

From 169dc30d62cfa4445834109667986ff6073382e6 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Mon, 10 May 2021 16:35:22 -0500
Subject: [PATCH 02/15] api compiles

---
 api.go          | 40 ++++++++++++++++++++++++++++++++++++++++
 ctl/restore.go  |  8 ++++++++
 http/handler.go | 35 +++++++++++++++++++++++++++++++++++
 3 files changed, 83 insertions(+)

diff --git a/api.go b/api.go
index 37d385791..46cf6980b 100644
--- a/api.go
+++ b/api.go
@@ -17,6 +17,7 @@
 package pilosa
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"encoding/binary"
@@ -26,6 +27,7 @@ import (
 	"io/ioutil"
 	"math"
 	"net/url"
+	"os"
 	"sort"
 	"strconv"
 	"strings"
@@ -38,6 +40,7 @@ import (
 	"github.com/pilosa/pilosa/v2/stats"
 	"github.com/pilosa/pilosa/v2/topology"
 	"github.com/pilosa/pilosa/v2/tracing"
+	"github.com/pilosa/pilosa/v2/vprint"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 )
@@ -2219,6 +2222,43 @@ func (api *API) TranslateFieldDB(ctx context.Context, indexName, fieldName strin
 	return err
 }
 
+// RestoreShard
+func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64, rd io.Reader) error {
+	snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
+	if !snap.OwnsShard(api.server.nodeID, indexName, shard) {
+		return ErrClusterDoesNotOwnShard // TODO (twg)really just node doesn't own shard but leave for now
+	}
+
+	idx := api.holder.Index(indexName)
+	//need to get a dbShard
+	dbs, err := idx.Txf().dbPerShard.GetDBShard(indexName, shard, idx)
+	if err != nil {
+		return err
+	}
+	dbs.Close()
+	//need to find the path to the db
+	//will not work on blue green
+	finalPath := dbs.W[0].Path()
+	tempPath := finalPath + ".tmp"
+	vprint.VV("restore to %v", tempPath)
+	o, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
+	if err != nil {
+		return err
+	}
+	w := bufio.NewWriter(o)
+	//close db if open
+	vprint.VV("restore index:%v shard:%v", indexName, shard)
+	_, err = io.Copy(w, rd)
+	w.Flush()
+	o.Close()
+	if err != nil {
+		_ = os.Remove(tempPath)
+		return err
+	}
+	vprint.VV("Rename %v to %v", tempPath, finalPath)
+	return os.Rename(tempPath, finalPath)
+}
+
 type serverInfo struct {
 	ShardWidth       uint64 `json:"shardWidth"`
 	ReplicaN         int    `json:"replicaN"`
diff --git a/ctl/restore.go b/ctl/restore.go
index c4e7a4974..9329f7303 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -94,6 +94,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 	} else {
 		tarReader = tar.NewReader(f)
 	}
+	//maybe begin transaction?
 	schemaJson := readSchema(cmd.Path)
 	//Push the schema from the archive into the cluster belonging to the host.
 	client := &gohttp.Client{}
@@ -102,6 +103,13 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	//TODO (twg) load schema
+	//TODO (twg) load rbf shard
+	//TODO (twg) load row keys
+	//TODO (twg) load column keys
+	//TODO (twg) load row attributes keys
+	//TODO (twg) load col attributes keys
+	//TODO (twg) load idalloc
 
 	for {
 		header, err := tarReader.Next()
diff --git a/http/handler.go b/http/handler.go
index d5d14b1d6..6f60b18df 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -441,6 +441,7 @@ func newRouter(handler *Handler) http.Handler {
 	router.HandleFunc("/internal/idalloc/reset/{index}", handler.handleResetIDAlloc).Methods("POST").Name("ResetIDAlloc")
 	router.HandleFunc("/internal/idalloc/data", handler.handleIDAllocData).Methods("GET").Name("IDAllocData")
 
+	router.HandleFunc("/internal/restore/{index}/{shardID}", handler.handlePostRestore).Methods("POST").Name("Restore")
 	// endpoints for collecting cpu profiles from a chosen begin point to
 	// when the client wants to stop. Used for profiling imports that
 	// could be long or short.
@@ -2911,3 +2912,37 @@ func (h *Handler) handleIDAllocData(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 }
+
+func (h *Handler) handlePostRestore(w http.ResponseWriter, r *http.Request) {
+	/*
+		if !validHeaderAcceptType(r.Header, "text", "plain") {
+			http.Error(w, "text/plain is not an acceptable response type", http.StatusNotAcceptable)
+		}
+	*/
+	indexName, ok := mux.Vars(r)["index"]
+	if !ok {
+		http.Error(w, "index name is required", http.StatusBadRequest)
+		return
+	}
+	shardID, ok := mux.Vars(r)["shardID"]
+	if !ok {
+		http.Error(w, "shardID is required", http.StatusBadRequest)
+		return
+	}
+	shard, err := strconv.ParseUint(shardID, 10, 64)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to parse shard %v %v err:%v", indexName, shardID, err), http.StatusBadRequest)
+		return
+	}
+	ctx := context.Background()
+	//validate shard for this node
+	err = h.api.RestoreShard(ctx, indexName, shard, r.Body)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to restore shared %v %v err:%v", indexName, shard, err), http.StatusBadRequest)
+		return
+	}
+
+	w.Header().Add("Content-Type", "text/plain")
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte("OK")) //nolint:errcheck
+}

From 9d24fb07b7d52b7f683ad11d6c7db7e43b760deb Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Thu, 13 May 2021 11:28:52 -0500
Subject: [PATCH 03/15] wired in restore command

---
 api.go         | 21 +++++++++++---
 bolt.go        | 17 +++++++++++
 cmd/root.go    |  1 +
 ctl/restore.go | 77 ++++++++++++++++++++++++++++++++++++--------------
 dbshard.go     |  3 ++
 fragment.go    |  3 ++
 lattice        |  2 +-
 rbf.go         | 20 +++++++++++++
 rrtx.go        |  7 +++++
 9 files changed, 125 insertions(+), 26 deletions(-)

diff --git a/api.go b/api.go
index 46cf6980b..a452bf8d5 100644
--- a/api.go
+++ b/api.go
@@ -2235,10 +2235,10 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
 	if err != nil {
 		return err
 	}
-	dbs.Close()
 	//need to find the path to the db
 	//will not work on blue green
-	finalPath := dbs.W[0].Path()
+	db := dbs.W[0]
+	finalPath := db.Path() + "/data"
 	tempPath := finalPath + ".tmp"
 	vprint.VV("restore to %v", tempPath)
 	o, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
@@ -2248,15 +2248,28 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
 	w := bufio.NewWriter(o)
 	//close db if open
 	vprint.VV("restore index:%v shard:%v", indexName, shard)
-	_, err = io.Copy(w, rd)
+	n, err := io.Copy(w, rd)
+	vprint.VV("Written:%v", n)
 	w.Flush()
 	o.Close()
 	if err != nil {
 		_ = os.Remove(tempPath)
 		return err
 	}
+	err = db.CloseDB()
+	if err != nil {
+		return err
+	}
 	vprint.VV("Rename %v to %v", tempPath, finalPath)
-	return os.Rename(tempPath, finalPath)
+	err = os.Rename(tempPath, finalPath)
+	if err != nil {
+		_ = os.Remove(tempPath)
+		return err
+	}
+	api.holder.recalculateCaches()
+
+	return db.OpenDB()
+
 }
 
 type serverInfo struct {
diff --git a/bolt.go b/bolt.go
index 0ce42b0f5..c0dbb600f 100644
--- a/bolt.go
+++ b/bolt.go
@@ -217,6 +217,23 @@ func (w *BoltWrapper) HasData() (has bool, err error) {
 func (w *BoltWrapper) CleanupTx(tx Tx) {
 	// inlined into Rollback and Commit, so this is a no-op, just here to satisfy the interface.
 }
+func (w *BoltWrapper) CloseDB() error {
+	w.muDb.Lock()
+	defer w.muDb.Unlock()
+	w.closed = true
+	return w.db.Close()
+}
+func (w *BoltWrapper) OpenDB() error {
+	w.muDb.Lock()
+	defer w.muDb.Unlock()
+	db, err := bolt.Open(w.path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize})
+	if err != nil {
+		return err
+	}
+	w.db = db
+	w.closed = false
+	return nil
+}
 
 func (tx *BoltTx) IsDone() (done bool) {
 	return atomic.LoadInt64(&tx.unlocked) == 1
diff --git a/cmd/root.go b/cmd/root.go
index a6de7527e..6fd307a04 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -64,6 +64,7 @@ at https://www.pilosa.com/docs/.
 	rc.PersistentFlags().StringP("config", "c", "", "Configuration file to read from.")
 
 	rc.AddCommand(newBackupCommand(stdin, stdout, stderr))
+	rc.AddCommand(newRestoreCommand(stdin, stdout, stderr))
 	rc.AddCommand(newCheckCommand(stdin, stdout, stderr))
 	rc.AddCommand(newConfigCommand(stdin, stdout, stderr))
 	rc.AddCommand(newExportCommand(stdin, stdout, stderr))
diff --git a/ctl/restore.go b/ctl/restore.go
index 9329f7303..2c270f00f 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -16,17 +16,15 @@ package ctl
 
 import (
 	"archive/tar"
-	"bytes"
 	"compress/gzip"
 	"context"
 	"io"
 	"os"
 	"strings"
 
-	gohttp "net/http"
-
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/vprint"
 )
 
 // RestoreCommand represents a command for restoring a backup to
@@ -41,16 +39,18 @@ type RestoreCommand struct {
 
 // NewRestoreCommand returns a new instance of RestoreCommand.
 func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand {
-	h := &gohttp.Client{}
-	host := "SOMETHING"
-	c, err := http.NewInternalClient(host, h)
-	if err != nil {
-		panic(err)
-	}
+	/*
+		h := &gohttp.Client{}
+		host := "SOMETHING"
+		c, err := http.NewInternalClient(host, h)
+		if err != nil {
+			panic(err)
+		}
+	*/
 
 	return &RestoreCommand{
-		CmdIO:  pilosa.NewCmdIO(stdin, stdout, stderr),
-		client: c,
+		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
+		//		client: c,
 	}
 }
 
@@ -95,14 +95,16 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 		tarReader = tar.NewReader(f)
 	}
 	//maybe begin transaction?
-	schemaJson := readSchema(cmd.Path)
-	//Push the schema from the archive into the cluster belonging to the host.
-	client := &gohttp.Client{}
-	url := "FIXME"
-	_, err = client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson))
-	if err != nil {
-		return err
-	}
+	/*
+		schemaJson := readSchema(cmd.Path)
+		//Push the schema from the archive into the cluster belonging to the host.
+		client := &gohttp.Client{}
+		url := "FIXME"
+		_, err = client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson))
+		if err != nil {
+			return err
+		}
+	*/
 	//TODO (twg) load schema
 	//TODO (twg) load rbf shard
 	//TODO (twg) load row keys
@@ -113,11 +115,44 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 
 	for {
 		header, err := tarReader.Next()
-		//fmt.Println("What %v", header.Name)
-		_ = header
 		if err == io.EOF {
 			break
 		}
+		record := strings.Split(header.Name, "/")
+		if len(record) == 1 {
+			switch record[0] {
+			case "schema":
+				vprint.VV("Load Schema")
+			case "idalloc":
+				vprint.VV("Load ids")
+			default:
+				panic("UNKNOWN " + record[0])
+
+			}
+			continue
+		}
+		indexName := record[1]
+		switch record[2] {
+		case "shards":
+			shard := record[3]
+			vprint.VV("shard %v %v", shard, indexName)
+		case "translate":
+			vprint.VV("column keys %v", indexName)
+		case "attributes":
+			vprint.VV("column attributes %v", indexName)
+		case "fields":
+			fieldName := record[3]
+			switch action := record[4]; action {
+			case "translate":
+				vprint.VV("field keys %v %v", indexName, fieldName)
+			case "attributes":
+				vprint.VV("field attributes %v %v", indexName, fieldName)
+			default:
+				panic("unknown:" + action)
+			}
+
+		}
+
 	}
 	/*	Fetch the cluster nodes from the target host.
 		For each index:
diff --git a/dbshard.go b/dbshard.go
index 8e6c92fcb..573f9b6ff 100644
--- a/dbshard.go
+++ b/dbshard.go
@@ -66,6 +66,9 @@ type DBWrapper interface {
 	Path() string
 	HasData() (has bool, err error)
 	SetHolder(h *Holder)
+	//needed for restore
+	CloseDB() error
+	OpenDB() error
 }
 
 type DBRegistry interface {
diff --git a/fragment.go b/fragment.go
index 592701331..915a29556 100644
--- a/fragment.go
+++ b/fragment.go
@@ -2909,6 +2909,9 @@ func (f *fragment) flushCache() error {
 		return errors.Wrap(err, "marshalling")
 	}
 
+	if err := os.MkdirAll(filepath.Dir(f.cachePath()), 0777); err != nil {
+		return errors.Wrap(err, "mkdir")
+	}
 	// Write to disk.
 	if err := ioutil.WriteFile(f.cachePath(), buf, 0666); err != nil {
 		return errors.Wrap(err, "writing")
diff --git a/lattice b/lattice
index 7ea3d77f8..d4bada428 160000
--- a/lattice
+++ b/lattice
@@ -1 +1 @@
-Subproject commit 7ea3d77f89771a06cbe59867f9135436fc8ea3b6
+Subproject commit d4bada428e45823a70432321843c38fcf2a384a0
diff --git a/rbf.go b/rbf.go
index f1b4a98c6..73e8f92d1 100644
--- a/rbf.go
+++ b/rbf.go
@@ -571,6 +571,26 @@ func (w *RbfDBWrapper) Close() error {
 	return w.db.Close()
 }
 
+// needed to handle the special case on reload, the close method unregisters the wrapper and all that is
+// required is the backing file get reloaded
+
+func (w *RbfDBWrapper) CloseDB() error {
+	w.muDb.Lock()
+	defer w.muDb.Unlock()
+	w.closed = true
+	return w.db.Close()
+}
+func (w *RbfDBWrapper) OpenDB() error {
+	w.muDb.Lock()
+	defer w.muDb.Unlock()
+	err := w.db.Open()
+	if err != nil {
+		return err
+	}
+	w.closed = false
+	return nil
+}
+
 var globalNextTxSnRBFTx int64
 
 func (w *RbfDBWrapper) NewTx(write bool, initialIndex string, o Txo) (_ Tx, err error) {
diff --git a/rrtx.go b/rrtx.go
index 288b4f361..9cce080c3 100644
--- a/rrtx.go
+++ b/rrtx.go
@@ -663,6 +663,13 @@ func (w *RoaringWrapper) OpenSnList() (slc []int64) {
 	return nil
 }
 
+func (w *RoaringWrapper) CloseDB() error {
+	return errors.New("CloseDB not supported in roaring")
+}
+func (w *RoaringWrapper) OpenDB() error {
+	return errors.New("OpenDB not supported in roaring")
+}
+
 // statically confirm that RoaringTx satisfies the Tx interface.
 var _ Tx = (*RoaringTx)(nil)
 

From 42b465b80cb88372a3e6d06107f88b3f2afdf18e Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Fri, 14 May 2021 10:39:39 -0500
Subject: [PATCH 04/15] load schema

---
 cmd/restore.go |  8 ++++++++
 ctl/restore.go | 48 ++++++++++++++++++++++++++++++++++++++++++++----
 http/client.go |  1 -
 3 files changed, 52 insertions(+), 5 deletions(-)

diff --git a/cmd/restore.go b/cmd/restore.go
index c8a78f917..f22ebfbc0 100644
--- a/cmd/restore.go
+++ b/cmd/restore.go
@@ -46,6 +46,14 @@ func newRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
 	}
 	flags := restoreCmd.Flags()
 	flags.StringVarP(&c.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
+	flags.StringVarP(&c.Path, "source", "s", "", "pilosa backup file")
+	ctl.SetTLSConfig(
+		flags, "",
+		&c.TLS.CertificatePath,
+		&c.TLS.CertificateKeyPath,
+		&c.TLS.CACertPath,
+		&c.TLS.SkipVerify,
+		&c.TLS.EnableClientVerification)
 
 	return restoreCmd
 }
diff --git a/ctl/restore.go b/ctl/restore.go
index 2c270f00f..438369f08 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -18,23 +18,30 @@ import (
 	"archive/tar"
 	"compress/gzip"
 	"context"
+	"errors"
+	"fmt"
 	"io"
+	gohttp "net/http"
 	"os"
 	"strings"
 
 	"github.com/pilosa/pilosa/v2"
-	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/topology"
 	"github.com/pilosa/pilosa/v2/vprint"
 )
 
 // RestoreCommand represents a command for restoring a backup to
 type RestoreCommand struct {
 	// Filepath to the backup file.
-	Path   string
-	Host   string
-	client *http.InternalClient
+	Path string
+	Host string
+	// Reusable client.
+	client pilosa.InternalClient
+
 	// Standard input/output
 	*pilosa.CmdIO
+	TLS server.TLSConfig
 }
 
 // NewRestoreCommand returns a new instance of RestoreCommand.
@@ -79,6 +86,13 @@ func readSchema(path string) string {
 
 // Run executes the restore.
 func (cmd *RestoreCommand) Run(ctx context.Context) error {
+	// Create a client to the server.
+	client, err := commandClient(cmd)
+	if err != nil {
+		return fmt.Errorf("creating client: %w", err)
+	}
+	cmd.client = client
+
 	f, err := os.Open(cmd.Path)
 	if err != nil {
 		return (err)
@@ -112,7 +126,22 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 	//TODO (twg) load row attributes keys
 	//TODO (twg) load col attributes keys
 	//TODO (twg) load idalloc
+	nodes, err := cmd.client.Nodes(ctx)
+	if err != nil {
+		return err
+	}
+	var primary *topology.Node
+	for _, node := range nodes {
+		if node.IsPrimary {
+			primary = node
+			break
+		}
 
+	}
+	c := &gohttp.Client{}
+	if primary == nil {
+		return errors.New("no primary")
+	}
 	for {
 		header, err := tarReader.Next()
 		if err == io.EOF {
@@ -123,6 +152,14 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 			switch record[0] {
 			case "schema":
 				vprint.VV("Load Schema")
+				url := primary.URI.Path("/schema")
+				vprint.VV("SCHEMA %v", url)
+				//schemaBytes, err := ioutil.ReadAll(tarReader)
+				_, err = c.Post(url, "application/json", tarReader)
+				if err != nil {
+					return err
+				}
+
 			case "idalloc":
 				vprint.VV("Load ids")
 			default:
@@ -163,3 +200,6 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 
 	return nil
 }
+func (cmd *RestoreCommand) TLSHost() string { return cmd.Host }
+
+func (cmd *RestoreCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS }
diff --git a/http/client.go b/http/client.go
index c10f038bf..4b9e16f49 100644
--- a/http/client.go
+++ b/http/client.go
@@ -199,7 +199,6 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error
 	}
 	return rsp.Indexes, nil
 }
-
 func (c *InternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *pilosa.Schema, remote bool) error {
 	u := uri.Path(fmt.Sprintf("/schema?remote=%v", remote))
 	buf, err := json.Marshal(s)

From 69245ee20925732f7265d412fcca0c55ef342670 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Fri, 14 May 2021 11:05:53 -0500
Subject: [PATCH 05/15] shard import

---
 ctl/restore.go | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/ctl/restore.go b/ctl/restore.go
index 438369f08..0c26866fd 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -23,6 +23,7 @@ import (
 	"io"
 	gohttp "net/http"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/pilosa/pilosa/v2"
@@ -171,8 +172,18 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 		indexName := record[1]
 		switch record[2] {
 		case "shards":
-			shard := record[3]
+			sshard := record[3]
+			shard, err := strconv.Atoi(sshard)
+			if err != nil {
+				return err
+			}
 			vprint.VV("shard %v %v", shard, indexName)
+			url := primary.URI.Path(fmt.Sprintf("/internal/restore/%v/%v", indexName, shard))
+			vprint.VV("%v", url)
+			_, err = c.Post(url, "application/octet-stream", tarReader)
+			if err != nil {
+				return err
+			}
 		case "translate":
 			vprint.VV("column keys %v", indexName)
 		case "attributes":

From 329f86033d4eae2514d2b0c1fdfa207a5e0a4daa Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Fri, 14 May 2021 11:28:23 -0500
Subject: [PATCH 06/15] restore field translate keys

---
 client.go      | 10 ++++++++++
 ctl/restore.go |  5 +++++
 2 files changed, 15 insertions(+)

diff --git a/client.go b/client.go
index cbce63d39..b1ddaab30 100644
--- a/client.go
+++ b/client.go
@@ -91,6 +91,9 @@ type InternalClient interface {
 
 	GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error)
 	GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error)
+
+	ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, rddbdata io.Reader) error
+	ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error
 }
 
 //===============
@@ -274,3 +277,10 @@ func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map
 func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) {
 	return nil, nil
 }
+func (c nopInternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, rddbdata io.Reader) error {
+	return nil
+}
+
+func (c nopInternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error {
+	return nil
+}
diff --git a/ctl/restore.go b/ctl/restore.go
index 0c26866fd..6c67ad8ef 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -180,6 +180,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 			vprint.VV("shard %v %v", shard, indexName)
 			url := primary.URI.Path(fmt.Sprintf("/internal/restore/%v/%v", indexName, shard))
 			vprint.VV("%v", url)
+			//TODO (twg) cluster aware client
 			_, err = c.Post(url, "application/octet-stream", tarReader)
 			if err != nil {
 				return err
@@ -193,6 +194,10 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 			switch action := record[4]; action {
 			case "translate":
 				vprint.VV("field keys %v %v", indexName, fieldName)
+				err := cmd.client.ImportFieldKeys(ctx, &primary.URI, indexName, fieldName, false, tarReader)
+				if err != nil {
+					return err
+				}
 			case "attributes":
 				vprint.VV("field attributes %v %v", indexName, fieldName)
 			default:

From a1a9103f62ffc7af9dce5599b5938df698343ea1 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Fri, 14 May 2021 11:37:54 -0500
Subject: [PATCH 07/15] restore column translate keys

---
 ctl/restore.go | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/ctl/restore.go b/ctl/restore.go
index 6c67ad8ef..f1f98432f 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -172,8 +172,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 		indexName := record[1]
 		switch record[2] {
 		case "shards":
-			sshard := record[3]
-			shard, err := strconv.Atoi(sshard)
+			shard, err := strconv.Atoi(record[3])
 			if err != nil {
 				return err
 			}
@@ -186,7 +185,16 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 				return err
 			}
 		case "translate":
-			vprint.VV("column keys %v", indexName)
+			partitionID, err := strconv.Atoi(record[3])
+			vprint.VV("column keys %v (%v)", indexName, partitionID)
+			if err != nil {
+				return err
+			}
+
+			err = cmd.client.ImportIndexKeys(ctx, &primary.URI, indexName, partitionID, false, tarReader)
+			if err != nil {
+				return err
+			}
 		case "attributes":
 			vprint.VV("column attributes %v", indexName)
 		case "fields":

From cc0c1b829a1da431d2df2ab2c850d25ece2ba980 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Tue, 18 May 2021 11:03:44 -0500
Subject: [PATCH 08/15] restore idalloc

---
 api.go          | 10 ++++------
 ctl/restore.go  | 11 ++++++++---
 http/handler.go | 10 ++++++++++
 idalloc.go      | 32 ++++++++++++++++++++++++++++++++
 4 files changed, 54 insertions(+), 9 deletions(-)

diff --git a/api.go b/api.go
index a452bf8d5..9bf5c7555 100644
--- a/api.go
+++ b/api.go
@@ -40,7 +40,6 @@ import (
 	"github.com/pilosa/pilosa/v2/stats"
 	"github.com/pilosa/pilosa/v2/topology"
 	"github.com/pilosa/pilosa/v2/tracing"
-	"github.com/pilosa/pilosa/v2/vprint"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 )
@@ -2203,6 +2202,9 @@ func (api *API) WriteIDAllocDataTo(w io.Writer) error {
 	_, err := api.holder.ida.WriteTo(w)
 	return err
 }
+func (api *API) RestoreIDAlloc(r io.Reader) error {
+	return api.holder.ida.Replace(r)
+}
 
 // TranslateIndexDB is an internal function to load the index keys database
 // rd is a boltdb file.
@@ -2240,16 +2242,13 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
 	db := dbs.W[0]
 	finalPath := db.Path() + "/data"
 	tempPath := finalPath + ".tmp"
-	vprint.VV("restore to %v", tempPath)
 	o, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
 	if err != nil {
 		return err
 	}
 	w := bufio.NewWriter(o)
 	//close db if open
-	vprint.VV("restore index:%v shard:%v", indexName, shard)
-	n, err := io.Copy(w, rd)
-	vprint.VV("Written:%v", n)
+	_, err = io.Copy(w, rd)
 	w.Flush()
 	o.Close()
 	if err != nil {
@@ -2260,7 +2259,6 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
 	if err != nil {
 		return err
 	}
-	vprint.VV("Rename %v to %v", tempPath, finalPath)
 	err = os.Rename(tempPath, finalPath)
 	if err != nil {
 		_ = os.Remove(tempPath)
diff --git a/ctl/restore.go b/ctl/restore.go
index f1f98432f..be706cccc 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -155,7 +155,6 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 				vprint.VV("Load Schema")
 				url := primary.URI.Path("/schema")
 				vprint.VV("SCHEMA %v", url)
-				//schemaBytes, err := ioutil.ReadAll(tarReader)
 				_, err = c.Post(url, "application/json", tarReader)
 				if err != nil {
 					return err
@@ -163,6 +162,11 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 
 			case "idalloc":
 				vprint.VV("Load ids")
+				url := primary.URI.Path("/internal/idalloc/restore")
+				_, err = c.Post(url, "application/octet-stream", tarReader)
+				if err != nil {
+					return err
+				}
 			default:
 				panic("UNKNOWN " + record[0])
 
@@ -196,7 +200,8 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 				return err
 			}
 		case "attributes":
-			vprint.VV("column attributes %v", indexName)
+			//skip
+			//vprint.VV("column attributes %v", indexName)
 		case "fields":
 			fieldName := record[3]
 			switch action := record[4]; action {
@@ -207,7 +212,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 					return err
 				}
 			case "attributes":
-				vprint.VV("field attributes %v %v", indexName, fieldName)
+			//	vprint.VV("field attributes %v %v", indexName, fieldName)
 			default:
 				panic("unknown:" + action)
 			}
diff --git a/http/handler.go b/http/handler.go
index 6f60b18df..b87198a5d 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -438,6 +438,7 @@ func newRouter(handler *Handler) http.Handler {
 
 	router.HandleFunc("/internal/idalloc/reserve", handler.handleReserveIDs).Methods("POST").Name("ReserveIDs")
 	router.HandleFunc("/internal/idalloc/commit", handler.handleCommitIDs).Methods("POST").Name("CommitIDs")
+	router.HandleFunc("/internal/idalloc/restore", handler.handleRestoreIDAlloc).Methods("POST").Name("RestoreIDAllocData")
 	router.HandleFunc("/internal/idalloc/reset/{index}", handler.handleResetIDAlloc).Methods("POST").Name("ResetIDAlloc")
 	router.HandleFunc("/internal/idalloc/data", handler.handleIDAllocData).Methods("GET").Name("IDAllocData")
 
@@ -2913,6 +2914,15 @@ func (h *Handler) handleIDAllocData(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func (h *Handler) handleRestoreIDAlloc(w http.ResponseWriter, r *http.Request) {
+	if err := h.api.RestoreIDAlloc(r.Body); err != nil {
+		http.Error(w, fmt.Sprintf("restoring id allocation: %v", err.Error()), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Add("Content-Type", "text/plain")
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte("OK")) //nolint:errcheck
+}
 func (h *Handler) handlePostRestore(w http.ResponseWriter, r *http.Request) {
 	/*
 		if !validHeaderAcceptType(r.Header, "text", "plain") {
diff --git a/idalloc.go b/idalloc.go
index c9c954d76..4c6ea423f 100644
--- a/idalloc.go
+++ b/idalloc.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"math/bits"
+	"os"
 	"sort"
 	"time"
 
@@ -64,6 +65,37 @@ func OpenIDAllocator(path string) (*idAllocator, error) {
 	}
 	return &idAllocator{db}, nil
 }
+func (ida *idAllocator) Replace(reader io.Reader) error {
+	newFile := ida.db.Path() + ".bak"
+	liveFile := ida.db.Path()
+	file, err := os.OpenFile(newFile, os.O_RDWR|os.O_CREATE, 0600)
+	if err != nil {
+		return nil
+	}
+	_, err = io.Copy(file, reader)
+	if err != nil {
+		return err
+	}
+	file.Close()
+	err = ida.db.Close()
+	if err != nil {
+		return err
+	}
+	err = os.Rename(liveFile, liveFile+".sav")
+	if err != nil {
+		return err
+	}
+	err = os.Rename(newFile, liveFile)
+	if err != nil {
+		err = os.Rename(liveFile+".sav", liveFile)
+		return err
+	} else {
+		_ = os.Remove(liveFile + ".sav")
+	}
+	db, err := bolt.Open(liveFile, 0666, &bolt.Options{Timeout: 1 * time.Second})
+	ida.db = db
+	return err
+}
 
 func (ida *idAllocator) Close() error {
 	if ida == nil || ida.db == nil {

From b25ad81e678b7beeec97df898cbf65e2700ff436 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Thu, 20 May 2021 10:44:36 -0500
Subject: [PATCH 09/15] restore without restart

---
 api.go         | 26 ++++++++++++++++++++++++--
 ctl/restore.go |  2 --
 executor.go    |  1 -
 field.go       |  1 -
 index.go       |  4 ++++
 server.go      |  1 -
 6 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/api.go b/api.go
index 9bf5c7555..64931a0a9 100644
--- a/api.go
+++ b/api.go
@@ -2265,9 +2265,31 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
 		return err
 	}
 	api.holder.recalculateCaches()
+	err = db.OpenDB()
+	if err != nil {
+		return err
+	}
+	tx, err := db.NewTx(false, idx.name, Txo{})
+	defer tx.Rollback()
+	//arguments idx,shard do not matter for rbf they
+	//are ignored
+	flvs, err := tx.GetSortedFieldViewList(idx, shard)
+	for _, flv := range flvs {
+		fld := idx.field(flv.Field)
+		view, ok := fld.viewMap[flv.View]
+		if !ok {
+			view, err = fld.createViewIfNotExists(flv.View)
+			if err != nil {
+				return err
+			}
+		}
+		_, err = view.CreateFragmentIfNotExists(shard)
+		if err != nil {
+			return err
+		}
+	}
 
-	return db.OpenDB()
-
+	return nil
 }
 
 type serverInfo struct {
diff --git a/ctl/restore.go b/ctl/restore.go
index be706cccc..2df29d657 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -159,7 +159,6 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 				if err != nil {
 					return err
 				}
-
 			case "idalloc":
 				vprint.VV("Load ids")
 				url := primary.URI.Path("/internal/idalloc/restore")
@@ -182,7 +181,6 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 			}
 			vprint.VV("shard %v %v", shard, indexName)
 			url := primary.URI.Path(fmt.Sprintf("/internal/restore/%v/%v", indexName, shard))
-			vprint.VV("%v", url)
 			//TODO (twg) cluster aware client
 			_, err = c.Post(url, "application/octet-stream", tarReader)
 			if err != nil {
diff --git a/executor.go b/executor.go
index 0f3ef6148..6c18bd7cd 100644
--- a/executor.go
+++ b/executor.go
@@ -3688,7 +3688,6 @@ func (e *executor) executeRowsShard(ctx context.Context, qcx *Qcx, index string,
 		return nil, err
 	}
 	defer finisher(&err0)
-
 	for _, view := range views {
 		if err := ctx.Err(); err != nil {
 			return nil, err
diff --git a/field.go b/field.go
index 3f2c4c534..bbfe55227 100644
--- a/field.go
+++ b/field.go
@@ -1029,7 +1029,6 @@ func (f *Field) createViewIfNotExistsBase(cvm *CreateViewMessage) (*view, bool,
 			return nil, false, errors.Wrap(err, "persisting view")
 		}
 	}
-
 	view := f.newView(f.viewPath(cvm.View), cvm.View)
 
 	if err := view.openEmpty(); err != nil {
diff --git a/index.go b/index.go
index 18fa1d49e..5f8ca9f25 100644
--- a/index.go
+++ b/index.go
@@ -814,6 +814,10 @@ func (i *Index) DeleteField(name string) error {
 	return i.translationSyncer.Reset()
 }
 
+func (i *Index) UpdateAvailbleShards(field, view string, shard uint64) {
+
+}
+
 type indexSlice []*Index
 
 func (p indexSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
diff --git a/server.go b/server.go
index e940b092c..bb7f85c94 100644
--- a/server.go
+++ b/server.go
@@ -493,7 +493,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 	s.holder.schemator = s.schemator
 	s.holder.sharder = s.sharder
 	s.holder.serializer = s.serializer
-
 	return s, nil
 }
 

From 7520e0ef287daedf4a150c7482a0f5d44f71b932 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Thu, 20 May 2021 11:24:55 -0500
Subject: [PATCH 10/15] rebuild rank caches on restore

---
 api.go      |  6 +++++-
 fragment.go | 25 +++++++++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/api.go b/api.go
index 64931a0a9..5b8b09181 100644
--- a/api.go
+++ b/api.go
@@ -2283,7 +2283,11 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
 				return err
 			}
 		}
-		_, err = view.CreateFragmentIfNotExists(shard)
+		frag, err := view.CreateFragmentIfNotExists(shard)
+		if err != nil {
+			return err
+		}
+		err = frag.RebuildRankCache(ctx)
 		if err != nil {
 			return err
 		}
diff --git a/fragment.go b/fragment.go
index 915a29556..f7304f7ea 100644
--- a/fragment.go
+++ b/fragment.go
@@ -2890,6 +2890,31 @@ func (f *fragment) FlushCache() error {
 	defer f.mu.Unlock()
 	return f.flushCache()
 }
+func (f *fragment) RebuildRankCache(ctx context.Context) error {
+	if f.CacheType != CacheTypeRanked {
+		return nil //only rebuild ranked caches
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	tx, err := f.holder.BeginTx(false, f.idx, f.shard)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	rows, err := f.unprotectedRows(ctx, tx, uint64(0))
+	if err != nil {
+		return err
+	}
+	for _, id := range rows {
+		n, err := tx.CountRange(f.index(), f.field(), f.view(), f.shard, id*ShardWidth, (id+1)*ShardWidth)
+		if err != nil {
+			return errors.Wrap(err, "CountRange")
+		}
+		f.cache.BulkAdd(id, n)
+	}
+	f.cache.Invalidate()
+	return nil
+}
 
 func (f *fragment) flushCache() error {
 	if f.cache == nil {

From 35f4c33de9d953cc6dcb94bd7eb910e998628525 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Thu, 20 May 2021 11:36:05 -0500
Subject: [PATCH 11/15] first cache rebuild was ineffective

---
 api.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/api.go b/api.go
index 5b8b09181..fa371155b 100644
--- a/api.go
+++ b/api.go
@@ -2264,7 +2264,6 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
 		_ = os.Remove(tempPath)
 		return err
 	}
-	api.holder.recalculateCaches()
 	err = db.OpenDB()
 	if err != nil {
 		return err

From 3d577d762a82a5f3634e9d28b1deea13bbd11657 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Thu, 20 May 2021 11:44:28 -0500
Subject: [PATCH 12/15] cleanup

---
 ctl/restore.go | 65 +++++++-------------------------------------------
 1 file changed, 9 insertions(+), 56 deletions(-)

diff --git a/ctl/restore.go b/ctl/restore.go
index 2df29d657..d4ff058bb 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -47,44 +47,11 @@ type RestoreCommand struct {
 
 // NewRestoreCommand returns a new instance of RestoreCommand.
 func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand {
-	/*
-		h := &gohttp.Client{}
-		host := "SOMETHING"
-		c, err := http.NewInternalClient(host, h)
-		if err != nil {
-			panic(err)
-		}
-	*/
-
 	return &RestoreCommand{
 		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
-		//		client: c,
 	}
 }
 
-/* helper to allow for both gz and just plan tar
-	f, err := os.Open(cmd.Path)
-	if err != nil {
-		return (err)
-	}
-	defer f.Close()
-	var tarReader *tar.Reader
-	if strings.HasSuffix(cmd.Path, "gz") {
-		gzf, err := gzip.NewReader(f)
-		if err != nil {
-			return err
-		}
-		tarReader = tar.NewReader(gzf)
-	} else {
-		tarReader = tar.NewReader(f)
-	}
-return
-}
-*/
-func readSchema(path string) string {
-	return "{}"
-}
-
 // Run executes the restore.
 func (cmd *RestoreCommand) Run(ctx context.Context) error {
 	// Create a client to the server.
@@ -93,10 +60,15 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 		return fmt.Errorf("creating client: %w", err)
 	}
 	cmd.client = client
-
-	f, err := os.Open(cmd.Path)
-	if err != nil {
-		return (err)
+	var f *os.File
+	// read from Stdin if path specified as -
+	if cmd.Path == "-" {
+		f = os.Stdin
+	} else {
+		f, err = os.Open(cmd.Path)
+		if err != nil {
+			return (err)
+		}
 	}
 	defer f.Close()
 	var tarReader *tar.Reader
@@ -109,24 +81,6 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 	} else {
 		tarReader = tar.NewReader(f)
 	}
-	//maybe begin transaction?
-	/*
-		schemaJson := readSchema(cmd.Path)
-		//Push the schema from the archive into the cluster belonging to the host.
-		client := &gohttp.Client{}
-		url := "FIXME"
-		_, err = client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson))
-		if err != nil {
-			return err
-		}
-	*/
-	//TODO (twg) load schema
-	//TODO (twg) load rbf shard
-	//TODO (twg) load row keys
-	//TODO (twg) load column keys
-	//TODO (twg) load row attributes keys
-	//TODO (twg) load col attributes keys
-	//TODO (twg) load idalloc
 	nodes, err := cmd.client.Nodes(ctx)
 	if err != nil {
 		return err
@@ -154,7 +108,6 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 			case "schema":
 				vprint.VV("Load Schema")
 				url := primary.URI.Path("/schema")
-				vprint.VV("SCHEMA %v", url)
 				_, err = c.Post(url, "application/json", tarReader)
 				if err != nil {
 					return err

From 05fea53f7308b12802678d290f825e3d4b98f520 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Fri, 21 May 2021 09:27:53 -0500
Subject: [PATCH 13/15] linter

---
 .gitmodules | 3 ---
 api.go      | 7 +++++++
 lattice     | 1 -
 3 files changed, 7 insertions(+), 4 deletions(-)
 delete mode 100644 .gitmodules
 delete mode 160000 lattice

diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index afb2db2d6..000000000
--- a/.gitmodules
+++ /dev/null
@@ -1,3 +0,0 @@
-[submodule "lattice"]
-	path = lattice
-	url = git@github.com:molecula/lattice.git
diff --git a/api.go b/api.go
index fa371155b..1f3bbdfcd 100644
--- a/api.go
+++ b/api.go
@@ -2269,10 +2269,17 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
 		return err
 	}
 	tx, err := db.NewTx(false, idx.name, Txo{})
+	if err != nil {
+		return err
+	}
 	defer tx.Rollback()
 	//arguments idx,shard do not matter for rbf they
 	//are ignored
 	flvs, err := tx.GetSortedFieldViewList(idx, shard)
+	if err != nil {
+		return nil
+	}
+
 	for _, flv := range flvs {
 		fld := idx.field(flv.Field)
 		view, ok := fld.viewMap[flv.View]
diff --git a/lattice b/lattice
deleted file mode 160000
index d4bada428..000000000
--- a/lattice
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit d4bada428e45823a70432321843c38fcf2a384a0

From ff86f82ef278c3a9de24dca46c5d116465a8d13c Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Sun, 23 May 2021 10:27:50 -0500
Subject: [PATCH 14/15] applied ben's suggestions

---
 api.go          | 18 +++++++++++++-----
 ctl/restore.go  |  2 +-
 http/handler.go |  5 -----
 index.go        |  4 ----
 4 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/api.go b/api.go
index 1f3bbdfcd..35ff82b20 100644
--- a/api.go
+++ b/api.go
@@ -2246,11 +2246,19 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
 	if err != nil {
 		return err
 	}
-	w := bufio.NewWriter(o)
-	//close db if open
-	_, err = io.Copy(w, rd)
-	w.Flush()
-	o.Close()
+	defer o.Close()
+
+	bw := bufio.NewWriter(o)
+	if _, err = io.Copy(bw, rd); err != nil {
+		return err
+	} else if err := bw.Flush(); err != nil {
+		return err
+	} else if err := o.Sync(); err != nil {
+		return err
+	} else if err := o.Close(); err != nil {
+		return err
+	}
+
 	if err != nil {
 		_ = os.Remove(tempPath)
 		return err
diff --git a/ctl/restore.go b/ctl/restore.go
index d4ff058bb..7793f39e6 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -120,7 +120,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 					return err
 				}
 			default:
-				panic("UNKNOWN " + record[0])
+				return err
 
 			}
 			continue
diff --git a/http/handler.go b/http/handler.go
index b87198a5d..f43a0f000 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -2924,11 +2924,6 @@ func (h *Handler) handleRestoreIDAlloc(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("OK")) //nolint:errcheck
 }
 func (h *Handler) handlePostRestore(w http.ResponseWriter, r *http.Request) {
-	/*
-		if !validHeaderAcceptType(r.Header, "text", "plain") {
-			http.Error(w, "text/plain is not an acceptable response type", http.StatusNotAcceptable)
-		}
-	*/
 	indexName, ok := mux.Vars(r)["index"]
 	if !ok {
 		http.Error(w, "index name is required", http.StatusBadRequest)
diff --git a/index.go b/index.go
index 5f8ca9f25..18fa1d49e 100644
--- a/index.go
+++ b/index.go
@@ -814,10 +814,6 @@ func (i *Index) DeleteField(name string) error {
 	return i.translationSyncer.Reset()
 }
 
-func (i *Index) UpdateAvailbleShards(field, view string, shard uint64) {
-
-}
-
 type indexSlice []*Index
 
 func (p indexSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

From 69fda13ff41d691cd1ebf116f934b0d1e22f1f1f Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Mon, 24 May 2021 08:48:32 -0500
Subject: [PATCH 15/15] cleanup arg processing

---
 ctl/restore.go | 51 ++++++++++++++++++++++++++++----------------------
 1 file changed, 29 insertions(+), 22 deletions(-)

diff --git a/ctl/restore.go b/ctl/restore.go
index 7793f39e6..50c43ca56 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -29,20 +29,20 @@ import (
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/server"
 	"github.com/pilosa/pilosa/v2/topology"
-	"github.com/pilosa/pilosa/v2/vprint"
 )
 
 // RestoreCommand represents a command for restoring a backup to
 type RestoreCommand struct {
+	TLS  server.TLSConfig
+	Host string
+
 	// Filepath to the backup file.
 	Path string
-	Host string
 	// Reusable client.
 	client pilosa.InternalClient
 
 	// Standard input/output
 	*pilosa.CmdIO
-	TLS server.TLSConfig
 }
 
 // NewRestoreCommand returns a new instance of RestoreCommand.
@@ -54,23 +54,31 @@ func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreComman
 
 // Run executes the restore.
 func (cmd *RestoreCommand) Run(ctx context.Context) error {
+	logger := cmd.Logger()
+
+	// Validate arguments.
+	if cmd.Path == "" {
+		return fmt.Errorf("-s flag required")
+	}
+	useStdin := cmd.Path == "-"
+
+	var f *os.File
+	// read from Stdin if path specified as -
+	if useStdin {
+		f = os.Stdin
+	} else {
+		f, err := os.Open(cmd.Path)
+		if err != nil {
+			return (err)
+		}
+		defer f.Close()
+	}
 	// Create a client to the server.
 	client, err := commandClient(cmd)
 	if err != nil {
 		return fmt.Errorf("creating client: %w", err)
 	}
 	cmd.client = client
-	var f *os.File
-	// read from Stdin if path specified as -
-	if cmd.Path == "-" {
-		f = os.Stdin
-	} else {
-		f, err = os.Open(cmd.Path)
-		if err != nil {
-			return (err)
-		}
-	}
-	defer f.Close()
 	var tarReader *tar.Reader
 	if strings.HasSuffix(cmd.Path, "gz") {
 		gzf, err := gzip.NewReader(f)
@@ -106,14 +114,14 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 		if len(record) == 1 {
 			switch record[0] {
 			case "schema":
-				vprint.VV("Load Schema")
+				logger.Printf("Load Schema")
 				url := primary.URI.Path("/schema")
 				_, err = c.Post(url, "application/json", tarReader)
 				if err != nil {
 					return err
 				}
 			case "idalloc":
-				vprint.VV("Load ids")
+				logger.Printf("Load ids")
 				url := primary.URI.Path("/internal/idalloc/restore")
 				_, err = c.Post(url, "application/octet-stream", tarReader)
 				if err != nil {
@@ -132,7 +140,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 			if err != nil {
 				return err
 			}
-			vprint.VV("shard %v %v", shard, indexName)
+			logger.Printf("shard %v %v", shard, indexName)
 			url := primary.URI.Path(fmt.Sprintf("/internal/restore/%v/%v", indexName, shard))
 			//TODO (twg) cluster aware client
 			_, err = c.Post(url, "application/octet-stream", tarReader)
@@ -141,7 +149,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 			}
 		case "translate":
 			partitionID, err := strconv.Atoi(record[3])
-			vprint.VV("column keys %v (%v)", indexName, partitionID)
+			logger.Printf("column keys %v (%v)", indexName, partitionID)
 			if err != nil {
 				return err
 			}
@@ -152,20 +160,19 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
 			}
 		case "attributes":
 			//skip
-			//vprint.VV("column attributes %v", indexName)
 		case "fields":
 			fieldName := record[3]
 			switch action := record[4]; action {
 			case "translate":
-				vprint.VV("field keys %v %v", indexName, fieldName)
+				logger.Printf("field keys %v %v", indexName, fieldName)
 				err := cmd.client.ImportFieldKeys(ctx, &primary.URI, indexName, fieldName, false, tarReader)
 				if err != nil {
 					return err
 				}
 			case "attributes":
-			//	vprint.VV("field attributes %v %v", indexName, fieldName)
+			//skip
 			default:
-				panic("unknown:" + action)
+				return fmt.Errorf("unknown restore action: %v", action)
 			}
 
 		}