diff --git a/.travis.yml b/.travis.yml index 6d5674662..fbe1e0c9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,6 @@ install: - make install-dep install-statik vendor generate-statik script: - make test -# TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009 -after_success: - - go get github.com/mattn/goveralls - - $HOME/gopath/bin/goveralls -service=travis-ci -ignore "internal/internal.go,internal/public.pb.go,internal/private.pb.go" before_deploy: - pip install awscli --user `whoami` deploy: @@ -33,6 +29,7 @@ deploy: matrix: allow_failures: - go: master + fast_finish: true notifications: slack: secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" diff --git a/api.go b/api.go index a2e2094c3..c8b4b6666 100644 --- a/api.go +++ b/api.go @@ -431,79 +431,6 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName return blocks, nil } -// RestoreFrame reads all the data that this host should have for a given frame -// from replicas in the cluster and restores that data to it. -func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { - if err := api.validate(apiRestoreFrame); err != nil { - return errors.Wrap(err, "validating api method") - } - - // Create a client for the remote cluster. - client := NewInternalHTTPClientFromURI(host, api.RemoteClient) - - // Determine the maximum number of slices. - maxSlices, err := client.MaxSliceByIndex(ctx) - if err != nil { - return errors.Wrap(err, "getting max slice") - } - - // Retrieve frame. - f := api.Holder.Frame(indexName, frameName) - if f == nil { - return ErrFrameNotFound - } - - // Retrieve list of all views. - views, err := client.FrameViews(ctx, indexName, frameName) - if err != nil { - return errors.Wrap(err, "getting views") - } - - // Loop over each slice and import it if this node owns it. - for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { - // Ignore this slice if we don't own it. - if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { - continue - } - - // Loop over view names. - for _, view := range views { - // Create view. - v, err := f.CreateViewIfNotExists(view) - if err != nil { - return errors.Wrap(err, "creating view") - } - - // Otherwise retrieve the local fragment. - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - return errors.Wrap(err, "creating fragment") - } - - // Stream backup from remote node. - rd, err := client.BackupSlice(ctx, indexName, frameName, view, slice) - if err != nil { - return errors.Wrap(err, "getting backup") - } else if rd == nil { - continue // slice doesn't exist - } - - // Restore to local frame and always close reader. - if err := func() error { - defer rd.Close() - if _, err := frag.ReadFrom(rd); err != nil { - return errors.Wrap(err, "reading fragment") - } - return nil - }(); err != nil { - return err - } - } - } - - return nil -} - // Hosts returns a list of the hosts in the cluster including their ID, // URL, and which is the coordinator. func (api *API) Hosts(ctx context.Context) []*Node { diff --git a/client.go b/client.go index 0b22aed81..a5b630e10 100644 --- a/client.go +++ b/client.go @@ -15,20 +15,17 @@ package pilosa import ( - "archive/tar" "bytes" "context" "encoding/json" "fmt" "io" "io/ioutil" - "log" "math/rand" "net/http" "net/url" "sort" "strconv" - "time" "crypto/tls" @@ -581,107 +578,6 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde return nil } -// BackupTo backs up an entire frame from a cluster to w. -func (c *InternalHTTPClient) BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error { - if index == "" { - return ErrIndexRequired - } else if frame == "" { - return ErrFrameRequired - } - - // Create tar writer around writer. - tw := tar.NewWriter(w) - - // Find the maximum number of slices. - var maxSlices map[string]uint64 - var err error - if view == ViewStandard { - maxSlices, err = c.MaxSliceByIndex(ctx) - } else { - return ErrInvalidView - } - - if err != nil { - return fmt.Errorf("slice n: %s", err) - } - - // Backup every slice to the tar file. - for i := uint64(0); i <= maxSlices[index]; i++ { - if err := c.backupSliceTo(ctx, tw, index, frame, view, i); err != nil { - return errors.Wrap(err, "backing up slice") - } - } - - // Close tar file. - if err := tw.Close(); err != nil { - return errors.Wrap(err, "closing") - } - - return nil -} - -// backupSliceTo backs up a single slice to tw. -func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame, view string, slice uint64) error { - // Return error if unable to backup from any slice. - r, err := c.BackupSlice(ctx, index, frame, view, slice) - if err != nil { - return fmt.Errorf("backup slice: slice=%d, err=%s", slice, err) - } else if r == nil { - return nil - } - defer r.Close() - - // Read entire buffer to determine file size. - data, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrap(err, "reading") - } else if err := r.Close(); err != nil { - return errors.Wrap(err, "closing") - } - - // Write slice file header. - if err := tw.WriteHeader(&tar.Header{ - Name: strconv.FormatUint(slice, 10), - Mode: 0666, - Size: int64(len(data)), - ModTime: time.Now(), - }); err != nil { - return errors.Wrap(err, "writing header") - } - - // Write buffer to file. - if _, err := tw.Write(data); err != nil { - return errors.Wrap(err, "writing buffer") - } - - return nil -} - -// BackupSlice retrieves a streaming backup from a single slice. -// This function tries slice owners until one succeeds. -func (c *InternalHTTPClient) BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) { - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return nil, fmt.Errorf("slice nodes: %s", err) - } - - // Try to backup slice from each one until successful. - for _, i := range rand.Perm(len(nodes)) { - r, err := c.backupSliceNode(ctx, index, frame, view, slice, nodes[i]) - if err == nil { - return r, nil // successfully attached - } else if err == ErrFragmentNotFound { - return nil, nil // slice doesn't exist - } else if err != nil { - log.Println(err) - continue - } - } - - return nil, fmt.Errorf("unable to connect to any owner") -} - func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) { node := &Node{ URI: uri, @@ -724,86 +620,6 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, return resp.Body, nil } -// RestoreFrom restores a frame from a backup file to an entire cluster. -func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error { - if index == "" { - return ErrIndexRequired - } else if frame == "" { - return ErrFrameRequired - } - - // Create tar reader around input. - tr := tar.NewReader(r) - - // Process each file. - for { - hdr, err := tr.Next() - if err == io.EOF { - return nil - } else if err != nil { - return errors.Wrap(err, "opening") - } - - // Parse slice from entry name. - slice, err := strconv.ParseUint(hdr.Name, 10, 64) - if err != nil { - return fmt.Errorf("invalid backup entry: %s", hdr.Name) - } - - // Read file into buffer. - var buf bytes.Buffer - if _, err := io.CopyN(&buf, tr, hdr.Size); err != nil { - return errors.Wrap(err, "copying") - } - - // Restore file to all nodes that own it. - if err := c.restoreSliceFrom(ctx, buf.Bytes(), index, frame, view, slice); err != nil { - return errors.Wrap(err, "restoring") - } - } -} - -// restoreSliceFrom restores a single slice to all owning nodes. -func (c *InternalHTTPClient) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, view string, slice uint64) error { - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Restore slice to each owner. - for _, node := range nodes { - u := nodePathToURL(node, "/fragment/data") - u.RawQuery = url.Values{ - "index": {index}, - "frame": {frame}, - "view": {view}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Build request. - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/octet-stream") - req.Header.Set("User-Agent", "pilosa/"+Version) - - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - resp.Body.Close() - - // Return error if response not OK. - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: host=%s, code=%d", node.URI, resp.StatusCode) - } - } - - return nil -} - // CreateFrame creates a new frame on the server. func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error { if index == "" { @@ -1316,9 +1132,6 @@ type InternalClient interface { EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error - BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error - BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) - RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error RestoreFrame(ctx context.Context, host, index, frame string) error FrameViews(ctx context.Context, index, frame string) ([]string, error) diff --git a/client_test.go b/client_test.go index d8f1bf6d7..e4287726b 100644 --- a/client_test.go +++ b/client_test.go @@ -15,7 +15,6 @@ package pilosa_test import ( - "bytes" "context" "fmt" "net/http" @@ -316,76 +315,6 @@ func TestClient_ImportValue(t *testing.T) { } } -// Ensure client backup and restore a frame. -func TestClient_BackupRestore(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000) - - s := test.NewServer() - defer s.Close() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - c := test.MustNewClient(s.Host(), defaultClient) - - // Backup from frame. - var buf bytes.Buffer - if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewStandard); err != nil { - t.Fatal(err) - } - - // Restore to a different frame. - if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewStandard); err != nil { - t.Fatal(err) - } - - // Verify data. - if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { - t.Fatalf("unexpected columns(0): %+v", a) - } - if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { - t.Fatalf("unexpected columns(0): %+v", a) - } - if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { - t.Fatalf("unexpected columns(0): %+v", a) - } - if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Columns(); !reflect.DeepEqual(a, []uint64{20000}) { - t.Fatalf("unexpected columns: %+v", a) - } -} - -// backup returns error with invalid view -func TestClient_BackupInvalidView(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) - - s := test.NewServer() - defer s.Close() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - c := test.MustNewClient(s.Host(), defaultClient) - - // Backup from frame. - var buf bytes.Buffer - err := c.BackupTo(context.Background(), &buf, "i", "f", "invalid_view") - if err != pilosa.ErrInvalidView { - t.Fatal(err) - } -} - // Ensure client can retrieve a list of all checksums for blocks in a fragment. func TestClient_FragmentBlocks(t *testing.T) { hldr := test.MustOpenHolder() diff --git a/cmd/backup.go b/cmd/backup.go deleted file mode 100644 index 4bbbf70c2..000000000 --- a/cmd/backup.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "context" - "io" - "os" - - "github.com/pilosa/pilosa/ctl" - "github.com/spf13/cobra" -) - -var Backuper *ctl.BackupCommand - -func NewBackupCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Backuper = ctl.NewBackupCommand(os.Stdin, os.Stdout, os.Stderr) - backupCmd := &cobra.Command{ - Use: "backup", - Short: "Backup data from pilosa.", - Long: ` -Backs up the view from across the cluster into a single file. -`, - RunE: func(cmd *cobra.Command, args []string) error { - if err := Backuper.Run(context.Background()); err != nil { - return err - } - return nil - }, - } - flags := backupCmd.Flags() - flags.StringVarP(&Backuper.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Backuper.Index, "index", "i", "", "Pilosa index to backup.") - flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup.") - flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup.") - flags.StringVarP(&Backuper.Path, "output-file", "o", "", "File to write backup to - default stdout") - ctl.SetTLSConfig(flags, &Backuper.TLS.CertificatePath, &Backuper.TLS.CertificateKeyPath, &Backuper.TLS.SkipVerify) - - return backupCmd -} - -func init() { - subcommandFns["backup"] = NewBackupCmd -} diff --git a/cmd/backup_test.go b/cmd/backup_test.go deleted file mode 100644 index 58d9bf21c..000000000 --- a/cmd/backup_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd_test - -import ( - "strings" - "testing" - - "github.com/pilosa/pilosa/cmd" -) - -func TestBackupHelp(t *testing.T) { - output, err := ExecNewRootCommand(t, "backup", "--help") - if !strings.Contains(output, "Usage:") || - !strings.Contains(output, "Flags:") || - !strings.Contains(output, "pilosa backup") || err != nil { - t.Fatalf("Command 'backup --help' not working, err: '%v', output: '%s'", err, output) - } -} - -func TestBackupConfig(t *testing.T) { - tests := []commandTest{ - { - args: []string{"backup", "--output-file", "/somefile"}, - env: map[string]string{"PILOSA_HOST": "localhost:12345"}, - cfgFileContent: ` -index = "myindex" -frame = "f1" -`, - validation: func() error { - v := validator{} - v.Check(cmd.Backuper.Host, "localhost:12345") - v.Check(cmd.Backuper.Index, "myindex") - v.Check(cmd.Backuper.Frame, "f1") - v.Check(cmd.Backuper.Path, "/somefile") - return v.Error() - }, - }, - } - executeDry(t, tests) -} diff --git a/cmd/restore.go b/cmd/restore.go deleted file mode 100644 index e5169be12..000000000 --- a/cmd/restore.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "context" - "io" - "os" - - "github.com/spf13/cobra" - - "github.com/pilosa/pilosa/ctl" -) - -var Restorer *ctl.RestoreCommand - -func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Restorer = ctl.NewRestoreCommand(os.Stdin, os.Stdout, os.Stderr) - - restoreCmd := &cobra.Command{ - Use: "restore", - Short: "Restore data to pilosa from a backup file.", - Long: ` -Restores a view to the cluster from a backup file. -`, - RunE: func(cmd *cobra.Command, args []string) error { - if err := Restorer.Run(context.Background()); err != nil { - return err - } - return nil - }, - } - flags := restoreCmd.Flags() - flags.StringVarP(&Restorer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Restorer.Index, "index", "i", "", "Pilosa index to restore into.") - flags.StringVarP(&Restorer.Frame, "frame", "f", "", "Frame to restore into.") - flags.StringVarP(&Restorer.View, "view", "v", "", "View to restore into.") - flags.StringVarP(&Restorer.Path, "input-file", "d", "", "File to restore data from.") - ctl.SetTLSConfig(flags, &Restorer.TLS.CertificatePath, &Restorer.TLS.CertificateKeyPath, &Restorer.TLS.SkipVerify) - - return restoreCmd -} - -func init() { - subcommandFns["restore"] = NewRestoreCommand -} diff --git a/cmd/restore_test.go b/cmd/restore_test.go deleted file mode 100644 index cdad8ca92..000000000 --- a/cmd/restore_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd_test - -import ( - "strings" - "testing" - - "github.com/pilosa/pilosa/cmd" -) - -func TestRestoreHelp(t *testing.T) { - output, err := ExecNewRootCommand(t, "restore", "--help") - if !strings.Contains(output, "Usage:") || - !strings.Contains(output, "Flags:") || - !strings.Contains(output, "pilosa restore") || err != nil { - t.Fatalf("Command 'restore --help' not working, err: '%v', output: '%s'", err, output) - } -} - -func TestRestoreConfig(t *testing.T) { - tests := []commandTest{ - { - args: []string{"restore", "--input-file", "/somefile"}, - env: map[string]string{"PILOSA_HOST": "localhost:12345"}, - cfgFileContent: ` -index = "myindex" -frame = "f1" -`, - validation: func() error { - v := validator{} - v.Check(cmd.Restorer.Host, "localhost:12345") - v.Check(cmd.Restorer.Index, "myindex") - v.Check(cmd.Restorer.Frame, "f1") - v.Check(cmd.Restorer.Path, "/somefile") - return v.Error() - }, - }, - } - executeDry(t, tests) -} diff --git a/ctl/backup.go b/ctl/backup.go deleted file mode 100644 index 54a760695..000000000 --- a/ctl/backup.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "context" - "io" - "os" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/server" - "github.com/pkg/errors" -) - -// BackupCommand represents a command for backing up a view. -type BackupCommand struct { - // Destination host and port. - Host string - - // Name of the index, frame, view to backup. - Index string - Frame string - View string - - // Output file to write to. - Path string - - // Standard input/output - *pilosa.CmdIO - - TLS server.TLSConfig -} - -// NewBackupCommand returns a new instance of BackupCommand. -func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand { - return &BackupCommand{ - CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), - } -} - -// Run executes the backup. -func (cmd *BackupCommand) Run(ctx context.Context) error { - // Validate arguments. - if cmd.Path == "" { - return errors.New("output file required") - } - - // Create a client to the server. - client, err := CommandClient(cmd) - if err != nil { - return errors.Wrap(err, "creating client") - } - - // Open output file. - f, err := os.Create(cmd.Path) - if err != nil { - return errors.Wrap(err, "creating file") - } - defer f.Close() - - // Begin streaming backup. - if err := client.BackupTo(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil { - return errors.Wrap(err, "backing up") - } - - // Sync & close file to ensure durability. - if err := f.Sync(); err != nil { - return errors.Wrap(err, "syncing") - } else if err = f.Close(); err != nil { - return errors.Wrap(err, "closing file") - } - - return nil -} - -func (cmd *BackupCommand) TLSHost() string { - return cmd.Host -} - -func (cmd *BackupCommand) TLSConfiguration() server.TLSConfig { - return cmd.TLS -} diff --git a/ctl/backup_test.go b/ctl/backup_test.go deleted file mode 100644 index a073bc8a8..000000000 --- a/ctl/backup_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "bytes" - "context" - "io/ioutil" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/test" -) - -func TestBackupCommand_FileRequired(t *testing.T) { - buf := bytes.Buffer{} - stdin, stdout, stderr := GetIO(buf) - - cm := NewBackupCommand(stdin, stdout, stderr) - err := cm.Run(context.Background()) - if err.Error() != "output file required" { - t.Fatalf("expect error: output file required, actual: %s", err) - } - -} - -func TestBackupCommand_Run(t *testing.T) { - - buf := bytes.Buffer{} - stdin, stdout, stderr := GetIO(buf) - - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - defer s.Close() - - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - cm := NewBackupCommand(stdin, stdout, stderr) - file, err := ioutil.TempFile("", "import.csv") - - cm.Index = "i" - cm.Host = s.Host() - cm.Frame = "f" - cm.View = pilosa.ViewStandard - cm.Path = file.Name() - err = cm.Run(context.Background()) - if err != nil { - t.Fatalf("Command not working, error: '%s'", err) - } -} diff --git a/ctl/bench_test.go b/ctl/bench_test.go index 4790ccb44..766ec8a1a 100644 --- a/ctl/bench_test.go +++ b/ctl/bench_test.go @@ -15,6 +15,7 @@ package ctl import ( + "bufio" "bytes" "context" "fmt" @@ -86,3 +87,12 @@ func TestBenchCommand_Run(t *testing.T) { fmt.Println(buf.String()) } } + +// declare stdin, stdout, stderr +func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { + rder := []byte{} + stdin := bytes.NewReader(rder) + stdout := bufio.NewWriter(&buf) + stderr := bufio.NewWriter(&buf) + return stdin, stdout, stderr +} diff --git a/ctl/restore.go b/ctl/restore.go deleted file mode 100644 index ef863096e..000000000 --- a/ctl/restore.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "context" - "io" - "os" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/server" - "github.com/pkg/errors" -) - -// RestoreCommand represents a command for restoring a frame from a backup. -type RestoreCommand struct { - // Destination host and port. - Host string - - // Name of the index & frame to backup. - Index string - Frame string - View string - - // Import file to read from. - Path string - - // Standard input/output - *pilosa.CmdIO - - TLS server.TLSConfig -} - -// NewRestoreCommand returns a new instance of RestoreCommand. -func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand { - return &RestoreCommand{ - CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), - } -} - -// Run executes the restore command. -func (cmd *RestoreCommand) Run(ctx context.Context) error { - // Validate arguments. - if cmd.Path == "" { - return errors.New("backup file required") - } - - // Create a client to the server. - client, err := CommandClient(cmd) - if err != nil { - return errors.Wrap(err, "creating client") - } - - // Open backup file. - f, err := os.Open(cmd.Path) - if err != nil { - return errors.Wrap(err, "opening file") - } - defer f.Close() - - // Restore backup file to the cluster. - if err := client.RestoreFrom(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil { - return errors.Wrap(err, "restoring") - } - - return nil -} - -func (cmd *RestoreCommand) TLSHost() string { - return cmd.Host -} - -func (cmd *RestoreCommand) TLSConfiguration() server.TLSConfig { - return cmd.TLS -} diff --git a/ctl/restore_test.go b/ctl/restore_test.go deleted file mode 100644 index 896db57c5..000000000 --- a/ctl/restore_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "bufio" - "bytes" - "context" - "io" - "io/ioutil" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/test" -) - -func TestRestoreCommand_FileRequired(t *testing.T) { - cm := RestoreCommand{} - ctx := context.Background() - err := cm.Run(ctx) - if err.Error() != "backup file required" { - t.Fatalf("expect error: output file required, actual: '%s'", err) - } -} - -func TestRestoreCommand_Run(t *testing.T) { - file, err := ioutil.TempFile("", "restore.csv") - if err != nil { - t.Fatal(err) - } - buf := bytes.Buffer{} - stdin, stdout, stderr := GetIO(buf) - - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - defer s.Close() - - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - cm := NewRestoreCommand(stdin, stdout, stderr) - cm.Path = file.Name() - cm.Index = "i" - cm.Frame = "f" - cm.View = pilosa.ViewStandard - cm.Host = s.Host() - cm.Run(context.Background()) - if err != nil { - t.Fatalf("Backup Run doesn't work: %s", err) - } -} - -// declare stdin, stdout, stderr -func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { - rder := []byte{} - stdin := bytes.NewReader(rder) - stdout := bufio.NewWriter(&buf) - stderr := bufio.NewWriter(&buf) - return stdin, stdout, stderr -} diff --git a/docs/installation.md b/docs/installation.md index c4f30ed8d..039ea370d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -49,7 +49,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - backup Backup data from pilosa. bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. @@ -58,7 +57,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) help Help about any command import Bulk load data into pilosa. inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. server Run Pilosa. Flags: @@ -110,7 +108,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - backup Backup data from pilosa. bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. @@ -119,7 +116,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) help Help about any command import Bulk load data into pilosa. inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. server Run Pilosa. Flags: @@ -177,7 +173,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - backup Backup data from pilosa. bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. @@ -186,7 +181,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) help Help about any command import Bulk load data into pilosa. inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. server Run Pilosa. Flags: @@ -268,7 +262,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende pilosa [command] Available Commands: - backup Backup data from pilosa. bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. @@ -277,7 +270,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende help Help about any command import Bulk load data into pilosa. inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. server Run Pilosa. Flags: @@ -335,7 +327,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende pilosa [command] Available Commands: - backup Backup data from pilosa. bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. @@ -344,7 +335,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende help Help about any command import Bulk load data into pilosa. inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. server Run Pilosa. Flags: diff --git a/handler.go b/handler.go index f82db66c4..1848c71d5 100644 --- a/handler.go +++ b/handler.go @@ -144,7 +144,6 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.Handle("/debug/vars", expvar.Handler()).Methods("GET") - router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups) router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") @@ -159,7 +158,6 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") - router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST").Name("PostFragmentData") router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") router.HandleFunc("/import", handler.handlePostImport).Methods("POST") router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST") @@ -172,7 +170,6 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST").Name("PostFrameRestore") router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET") router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE") @@ -1040,48 +1037,6 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) } } -// handleGetFragmentData handles GET /fragment/data requests. -func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) { - // Read slice parameter. - q := r.URL.Query() - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "slice required", http.StatusBadRequest) - return - } - - // Retrieve fragment from holder. - f, err := h.API.MarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice) - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - // Stream fragment to response body. - if _, err := f.WriteTo(w); err != nil { - h.Logger.Printf("fragment backup error: %s", err) - } -} - -// handlePostFragmentData handles POST /fragment/data requests. -func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) { - // Read slice parameter. - q := r.URL.Query() - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "slice required", http.StatusBadRequest) - return - } - - if err = h.API.UnmarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice, r.Body); err != nil { - if errors.Cause(err) == ErrFrameNotFound { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - } -} - // handleGetFragmentBlockData handles GET /fragment/block/data requests. func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { buf, err := h.API.FragmentBlockData(r.Context(), r.Body) @@ -1134,38 +1089,6 @@ type getFragmentBlocksResponse struct { Blocks []FragmentBlock `json:"blocks"` } -// handlePostFrameRestore handles POST /frame/restore requests. -func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - q := r.URL.Query() - hostStr := q.Get("host") - - // Validate query parameters. - if hostStr == "" { - http.Error(w, "host required", http.StatusBadRequest) - return - } - - host, err := NewURIFromAddress(hostStr) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - } - - err = h.API.RestoreFrame(r.Context(), indexName, frameName, host) - switch errors.Cause(err) { - case nil: - break - case ErrFrameNotFound: - fallthrough - case ErrFragmentNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } -} - // handleGetVersion handles /version requests. func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { err := json.NewEncoder(w).Encode(struct { diff --git a/handler_test.go b/handler_test.go index 2b27516c2..a7ba8fa85 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1056,55 +1056,6 @@ type FrameFields struct { Fields []pilosa.Field } -// Ensure the handler can backup a fragment and then restore it. -func TestHandler_Fragment_BackupRestore(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - // Set bits in the index. - f0 := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - f0.MustSetBits(100, 1, 2, 3) - - // Begin backing up from slice i/f/0. - resp, err := http.Get(s.URL + "/fragment/data?index=i&frame=f&view=standard&slice=0") - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - // Ensure response came back OK. - if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected backup status code: %d", resp.StatusCode) - } - - // Create frame. - if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - // Restore backup to slice x/y/0. - if resp, err := http.Post(s.URL+"/fragment/data?index=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - resp.Body.Close() - t.Fatalf("unexpected restore status code: %d", resp.StatusCode) - } else { - resp.Body.Close() - } - - // Verify data is correctly restored. - f1 := hldr.Fragment("x", "y", pilosa.ViewStandard, 0) - if f1 == nil { - t.Fatal("fragment x/y/standard/0 not created") - } else if columns := f1.Row(100).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 3}) { - t.Fatalf("unexpected restored columns: %+v", columns) - } -} - // Ensure the handler can retrieve the version. func TestHandler_Version(t *testing.T) { hldr := test.MustOpenHolder() diff --git a/server/server_test.go b/server/server_test.go index b70b8a046..9092212b2 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -235,80 +235,6 @@ func TestMain_SetColumnAttrs(t *testing.T) { } } -// Ensure program can set columns on one cluster and then restore to a second cluster. -func TestMain_FrameRestore(t *testing.T) { - mains1 := test.MustRunMainWithCluster(t, 2) - m10 := mains1[0] - m11 := mains1[1] - - // Create frames. - client := m10.Client() - if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { - t.Fatal("create index:", err) - } - if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { - t.Fatal("create frame:", err) - } - - // Write data on first cluster. - if _, err := m10.Query("i", "", ` - SetBit(row=1, frame="f", col=100) - SetBit(row=1, frame="f", col=1000) - SetBit(row=1, frame="f", col=100000) - SetBit(row=1, frame="f", col=200000) - SetBit(row=1, frame="f", col=400000) - SetBit(row=1, frame="f", col=600000) - SetBit(row=1, frame="f", col=800000) - `); err != nil { - t.Fatal("setting columns:", err) - } - - // Query row on first cluster. - if res, err := m10.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil { - t.Fatal("bitmap query:", err) - } else if res != `{"results":[{"attrs":{},"columns":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { - t.Fatalf("unexpected result: %s", res) - } - - // Start second cluster. - mains2 := test.MustRunMainWithCluster(t, 2) - m20 := mains2[0] - defer m20.Close() - m21 := mains2[1] - defer m21.Close() - - // Import from first cluster. - client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), server.GetHTTPClient(nil)) - if err != nil { - t.Fatal("new client:", err) - } - client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), server.GetHTTPClient(nil)) - if err != nil { - t.Fatal("new client:", err) - } - - if err := m20.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { - t.Fatal("create new index:", err) - } - if err := m20.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { - t.Fatal("create new frame:", err) - } - - if err := client20.RestoreFrame(context.Background(), m10.Server.URI.HostPort(), "i", "f"); err != nil { - t.Fatal("restore frame:", err) - } - if err := client21.RestoreFrame(context.Background(), m11.Server.URI.HostPort(), "i", "f"); err != nil { - t.Fatal("restore frame:", err) - } - - // Query row on second cluster. - if res, err := m20.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil { - t.Fatal("another bitmap query:", err) - } else if res != `{"results":[{"attrs":{},"columns":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { - t.Fatalf("2unexpected result: %s", res) - } -} - // Ensure the host can be parsed. func TestConfig_Parse_Host(t *testing.T) { if c, err := ParseConfig(`bind = "local"`); err != nil {