scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add retryability to restore command

Browse source
This commit is contained in:
Matthew Jaffee 2021-12-20 17:42:22 -06:00
parent 3105a24542
commit f676fbfc51
4 changed files with 23 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cmd/restore.go
View file

@ -25,6 +25,7 @@ The Restore command will take a backup archive and restore it to a new, clean cl
flags.StringVarP(&cmd.Path, "source", "s", "", "backup file; specify '-' to restore from stdin tar stream")
flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.")
flags.IntVar(&cmd.Concurrency, "concurrency", 1, "number of concurrent uploads")
flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.")
ctl.SetTLSConfig(
flags, "",
&cmd.TLS.CertificatePath,

21
ctl/restore.go
View file

@ -13,8 +13,12 @@ import (
"path/filepath"
"strconv"
"strings"
"time"
"github.com/hashicorp/go-retryablehttp"
pilosa "github.com/molecula/featurebase/v2"
fb_http "github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/server"
"github.com/molecula/featurebase/v2/topology"
"golang.org/x/sync/errgroup"
@ -29,6 +33,10 @@ type RestoreCommand struct {
// Filepath to the backup file.
Path string
// Amount of time after first failed request to continue retrying.
RetryPeriod time.Duration `json:"retry-period"`
// Reusable client.
client pilosa.InternalClient
@ -62,7 +70,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) {
return fmt.Errorf("parsing tls config: %w", err)
}
// Create a client to the server.
client, err := commandClient(cmd)
client, err := commandClient(cmd, fb_http.WithClientRetryPeriod(cmd.RetryPeriod))
if err != nil {
return fmt.Errorf("creating client: %w", err)
}
@ -119,7 +127,8 @@ func (cmd *RestoreCommand) restoreSchema(ctx context.Context, primary *topology.
if len(existingSchema) == 0 {
cmd.Logger().Printf("Load Schema")
url := primary.URI.Path("/schema")
var client http.Client
client := retryablehttp.NewClient()
client.RetryWaitMax = cmd.RetryPeriod
_, err = client.Post(url, "application/json", f)
} else {
schema := &pilosa.Schema{}
@ -174,7 +183,8 @@ func (cmd *RestoreCommand) restoreIDAlloc(ctx context.Context, primary *topology
logger.Printf("Load idalloc")
url := primary.URI.Path("/internal/idalloc/restore")
var client http.Client
client := retryablehttp.NewClient()
client.RetryWaitMax = cmd.RetryPeriod
_, err = client.Post(url, "application/octet-stream", f)
return err
}
@ -244,14 +254,15 @@ func (cmd *RestoreCommand) restoreShard(ctx context.Context, filename string) er
defer f.Close()
url := node.URI.Path(fmt.Sprintf("/internal/restore/%v/%v", indexName, shard))
req, err := http.NewRequest("POST", url, f)
req, err := retryablehttp.NewRequest("POST", url, f)
if err != nil {
return err
}
req = req.WithContext(ctx)
req.Header.Set("Content-Type", "application/octet-stream")
var client http.Client
client := retryablehttp.NewClient()
client.RetryWaitMax = cmd.RetryPeriod
resp, err := client.Do(req)
if err != nil {
return err

1
go.mod
View file

@ -25,6 +25,7 @@ require (
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
github.com/gorilla/handlers v1.3.0
github.com/gorilla/mux v1.7.0
github.com/hashicorp/go-retryablehttp v0.7.0
github.com/improbable-eng/grpc-web v0.13.0
github.com/lib/pq v1.8.0
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b

5
go.sum
View file

@ -186,10 +186,15 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4=
github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=