mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 02:44:59 +00:00
Merge branch 'master' into cors-support
This commit is contained in:
Cody Soyland
GitHub
commit
31c4e18d8d
17 changed files with 11 additions and 1086 deletions
|
|
@ -16,10 +16,6 @@ install:
|
|||
- make install-dep install-statik vendor generate-statik
|
||||
script:
|
||||
- make test
|
||||
# TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009
|
||||
after_success:
|
||||
- go get github.com/mattn/goveralls
|
||||
- $HOME/gopath/bin/goveralls -service=travis-ci -ignore "internal/internal.go,internal/public.pb.go,internal/private.pb.go"
|
||||
before_deploy:
|
||||
- pip install awscli --user `whoami`
|
||||
deploy:
|
||||
|
|
@ -33,6 +29,7 @@ deploy:
|
|||
matrix:
|
||||
allow_failures:
|
||||
- go: master
|
||||
fast_finish: true
|
||||
notifications:
|
||||
slack:
|
||||
secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI="
|
||||
|
|
|
|||
73
api.go
73
api.go
|
|
@ -431,79 +431,6 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName
|
|||
return blocks, nil
|
||||
}
|
||||
|
||||
// RestoreFrame reads all the data that this host should have for a given frame
|
||||
// from replicas in the cluster and restores that data to it.
|
||||
func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error {
|
||||
if err := api.validate(apiRestoreFrame); err != nil {
|
||||
return errors.Wrap(err, "validating api method")
|
||||
}
|
||||
|
||||
// Create a client for the remote cluster.
|
||||
client := NewInternalHTTPClientFromURI(host, api.RemoteClient)
|
||||
|
||||
// Determine the maximum number of slices.
|
||||
maxSlices, err := client.MaxSliceByIndex(ctx)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "getting max slice")
|
||||
}
|
||||
|
||||
// Retrieve frame.
|
||||
f := api.Holder.Frame(indexName, frameName)
|
||||
if f == nil {
|
||||
return ErrFrameNotFound
|
||||
}
|
||||
|
||||
// Retrieve list of all views.
|
||||
views, err := client.FrameViews(ctx, indexName, frameName)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "getting views")
|
||||
}
|
||||
|
||||
// Loop over each slice and import it if this node owns it.
|
||||
for slice := uint64(0); slice <= maxSlices[indexName]; slice++ {
|
||||
// Ignore this slice if we don't own it.
|
||||
if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Loop over view names.
|
||||
for _, view := range views {
|
||||
// Create view.
|
||||
v, err := f.CreateViewIfNotExists(view)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "creating view")
|
||||
}
|
||||
|
||||
// Otherwise retrieve the local fragment.
|
||||
frag, err := v.CreateFragmentIfNotExists(slice)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "creating fragment")
|
||||
}
|
||||
|
||||
// Stream backup from remote node.
|
||||
rd, err := client.BackupSlice(ctx, indexName, frameName, view, slice)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "getting backup")
|
||||
} else if rd == nil {
|
||||
continue // slice doesn't exist
|
||||
}
|
||||
|
||||
// Restore to local frame and always close reader.
|
||||
if err := func() error {
|
||||
defer rd.Close()
|
||||
if _, err := frag.ReadFrom(rd); err != nil {
|
||||
return errors.Wrap(err, "reading fragment")
|
||||
}
|
||||
return nil
|
||||
}(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Hosts returns a list of the hosts in the cluster including their ID,
|
||||
// URL, and which is the coordinator.
|
||||
func (api *API) Hosts(ctx context.Context) []*Node {
|
||||
|
|
|
|||
187
client.go
187
client.go
|
|
@ -15,20 +15,17 @@
|
|||
package pilosa
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"crypto/tls"
|
||||
|
||||
|
|
@ -581,107 +578,6 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde
|
|||
return nil
|
||||
}
|
||||
|
||||
// BackupTo backs up an entire frame from a cluster to w.
|
||||
func (c *InternalHTTPClient) BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error {
|
||||
if index == "" {
|
||||
return ErrIndexRequired
|
||||
} else if frame == "" {
|
||||
return ErrFrameRequired
|
||||
}
|
||||
|
||||
// Create tar writer around writer.
|
||||
tw := tar.NewWriter(w)
|
||||
|
||||
// Find the maximum number of slices.
|
||||
var maxSlices map[string]uint64
|
||||
var err error
|
||||
if view == ViewStandard {
|
||||
maxSlices, err = c.MaxSliceByIndex(ctx)
|
||||
} else {
|
||||
return ErrInvalidView
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("slice n: %s", err)
|
||||
}
|
||||
|
||||
// Backup every slice to the tar file.
|
||||
for i := uint64(0); i <= maxSlices[index]; i++ {
|
||||
if err := c.backupSliceTo(ctx, tw, index, frame, view, i); err != nil {
|
||||
return errors.Wrap(err, "backing up slice")
|
||||
}
|
||||
}
|
||||
|
||||
// Close tar file.
|
||||
if err := tw.Close(); err != nil {
|
||||
return errors.Wrap(err, "closing")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// backupSliceTo backs up a single slice to tw.
|
||||
func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame, view string, slice uint64) error {
|
||||
// Return error if unable to backup from any slice.
|
||||
r, err := c.BackupSlice(ctx, index, frame, view, slice)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup slice: slice=%d, err=%s", slice, err)
|
||||
} else if r == nil {
|
||||
return nil
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
// Read entire buffer to determine file size.
|
||||
data, err := ioutil.ReadAll(r)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "reading")
|
||||
} else if err := r.Close(); err != nil {
|
||||
return errors.Wrap(err, "closing")
|
||||
}
|
||||
|
||||
// Write slice file header.
|
||||
if err := tw.WriteHeader(&tar.Header{
|
||||
Name: strconv.FormatUint(slice, 10),
|
||||
Mode: 0666,
|
||||
Size: int64(len(data)),
|
||||
ModTime: time.Now(),
|
||||
}); err != nil {
|
||||
return errors.Wrap(err, "writing header")
|
||||
}
|
||||
|
||||
// Write buffer to file.
|
||||
if _, err := tw.Write(data); err != nil {
|
||||
return errors.Wrap(err, "writing buffer")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// BackupSlice retrieves a streaming backup from a single slice.
|
||||
// This function tries slice owners until one succeeds.
|
||||
func (c *InternalHTTPClient) BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) {
|
||||
// Retrieve a list of nodes that own the slice.
|
||||
nodes, err := c.FragmentNodes(ctx, index, slice)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("slice nodes: %s", err)
|
||||
}
|
||||
|
||||
// Try to backup slice from each one until successful.
|
||||
for _, i := range rand.Perm(len(nodes)) {
|
||||
r, err := c.backupSliceNode(ctx, index, frame, view, slice, nodes[i])
|
||||
if err == nil {
|
||||
return r, nil // successfully attached
|
||||
} else if err == ErrFragmentNotFound {
|
||||
return nil, nil // slice doesn't exist
|
||||
} else if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unable to connect to any owner")
|
||||
}
|
||||
|
||||
func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) {
|
||||
node := &Node{
|
||||
URI: uri,
|
||||
|
|
@ -724,86 +620,6 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame,
|
|||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// RestoreFrom restores a frame from a backup file to an entire cluster.
|
||||
func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error {
|
||||
if index == "" {
|
||||
return ErrIndexRequired
|
||||
} else if frame == "" {
|
||||
return ErrFrameRequired
|
||||
}
|
||||
|
||||
// Create tar reader around input.
|
||||
tr := tar.NewReader(r)
|
||||
|
||||
// Process each file.
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return errors.Wrap(err, "opening")
|
||||
}
|
||||
|
||||
// Parse slice from entry name.
|
||||
slice, err := strconv.ParseUint(hdr.Name, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid backup entry: %s", hdr.Name)
|
||||
}
|
||||
|
||||
// Read file into buffer.
|
||||
var buf bytes.Buffer
|
||||
if _, err := io.CopyN(&buf, tr, hdr.Size); err != nil {
|
||||
return errors.Wrap(err, "copying")
|
||||
}
|
||||
|
||||
// Restore file to all nodes that own it.
|
||||
if err := c.restoreSliceFrom(ctx, buf.Bytes(), index, frame, view, slice); err != nil {
|
||||
return errors.Wrap(err, "restoring")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// restoreSliceFrom restores a single slice to all owning nodes.
|
||||
func (c *InternalHTTPClient) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, view string, slice uint64) error {
|
||||
// Retrieve a list of nodes that own the slice.
|
||||
nodes, err := c.FragmentNodes(ctx, index, slice)
|
||||
if err != nil {
|
||||
return fmt.Errorf("slice nodes: %s", err)
|
||||
}
|
||||
|
||||
// Restore slice to each owner.
|
||||
for _, node := range nodes {
|
||||
u := nodePathToURL(node, "/fragment/data")
|
||||
u.RawQuery = url.Values{
|
||||
"index": {index},
|
||||
"frame": {frame},
|
||||
"view": {view},
|
||||
"slice": {strconv.FormatUint(slice, 10)},
|
||||
}.Encode()
|
||||
|
||||
// Build request.
|
||||
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "creating request")
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
req.Header.Set("User-Agent", "pilosa/"+Version)
|
||||
|
||||
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "executing request")
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
// Return error if response not OK.
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("unexpected status code: host=%s, code=%d", node.URI, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateFrame creates a new frame on the server.
|
||||
func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error {
|
||||
if index == "" {
|
||||
|
|
@ -1316,9 +1132,6 @@ type InternalClient interface {
|
|||
EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error
|
||||
ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error
|
||||
ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error
|
||||
BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error
|
||||
BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error)
|
||||
RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error
|
||||
CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error
|
||||
RestoreFrame(ctx context.Context, host, index, frame string) error
|
||||
FrameViews(ctx context.Context, index, frame string) ([]string, error)
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
package pilosa_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
|
@ -316,76 +315,6 @@ func TestClient_ImportValue(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure client backup and restore a frame.
|
||||
func TestClient_BackupRestore(t *testing.T) {
|
||||
hldr := test.MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000)
|
||||
|
||||
s := test.NewServer()
|
||||
defer s.Close()
|
||||
s.Handler.API.Cluster = test.NewCluster(1)
|
||||
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
|
||||
s.Handler.API.Holder = hldr.Holder
|
||||
|
||||
c := test.MustNewClient(s.Host(), defaultClient)
|
||||
|
||||
// Backup from frame.
|
||||
var buf bytes.Buffer
|
||||
if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewStandard); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Restore to a different frame.
|
||||
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewStandard); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify data.
|
||||
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
|
||||
t.Fatalf("unexpected columns(0): %+v", a)
|
||||
}
|
||||
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) {
|
||||
t.Fatalf("unexpected columns(0): %+v", a)
|
||||
}
|
||||
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) {
|
||||
t.Fatalf("unexpected columns(0): %+v", a)
|
||||
}
|
||||
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Columns(); !reflect.DeepEqual(a, []uint64{20000}) {
|
||||
t.Fatalf("unexpected columns: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// backup returns error with invalid view
|
||||
func TestClient_BackupInvalidView(t *testing.T) {
|
||||
hldr := test.MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
|
||||
|
||||
s := test.NewServer()
|
||||
defer s.Close()
|
||||
s.Handler.API.Cluster = test.NewCluster(1)
|
||||
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
|
||||
s.Handler.API.Holder = hldr.Holder
|
||||
|
||||
c := test.MustNewClient(s.Host(), defaultClient)
|
||||
|
||||
// Backup from frame.
|
||||
var buf bytes.Buffer
|
||||
err := c.BackupTo(context.Background(), &buf, "i", "f", "invalid_view")
|
||||
if err != pilosa.ErrInvalidView {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure client can retrieve a list of all checksums for blocks in a fragment.
|
||||
func TestClient_FragmentBlocks(t *testing.T) {
|
||||
hldr := test.MustOpenHolder()
|
||||
|
|
|
|||
|
|
@ -1,56 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/pilosa/pilosa/ctl"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var Backuper *ctl.BackupCommand
|
||||
|
||||
func NewBackupCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
|
||||
Backuper = ctl.NewBackupCommand(os.Stdin, os.Stdout, os.Stderr)
|
||||
backupCmd := &cobra.Command{
|
||||
Use: "backup",
|
||||
Short: "Backup data from pilosa.",
|
||||
Long: `
|
||||
Backs up the view from across the cluster into a single file.
|
||||
`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if err := Backuper.Run(context.Background()); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
flags := backupCmd.Flags()
|
||||
flags.StringVarP(&Backuper.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
|
||||
flags.StringVarP(&Backuper.Index, "index", "i", "", "Pilosa index to backup.")
|
||||
flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup.")
|
||||
flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup.")
|
||||
flags.StringVarP(&Backuper.Path, "output-file", "o", "", "File to write backup to - default stdout")
|
||||
ctl.SetTLSConfig(flags, &Backuper.TLS.CertificatePath, &Backuper.TLS.CertificateKeyPath, &Backuper.TLS.SkipVerify)
|
||||
|
||||
return backupCmd
|
||||
}
|
||||
|
||||
func init() {
|
||||
subcommandFns["backup"] = NewBackupCmd
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cmd_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/pilosa/pilosa/cmd"
|
||||
)
|
||||
|
||||
func TestBackupHelp(t *testing.T) {
|
||||
output, err := ExecNewRootCommand(t, "backup", "--help")
|
||||
if !strings.Contains(output, "Usage:") ||
|
||||
!strings.Contains(output, "Flags:") ||
|
||||
!strings.Contains(output, "pilosa backup") || err != nil {
|
||||
t.Fatalf("Command 'backup --help' not working, err: '%v', output: '%s'", err, output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupConfig(t *testing.T) {
|
||||
tests := []commandTest{
|
||||
{
|
||||
args: []string{"backup", "--output-file", "/somefile"},
|
||||
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
|
||||
cfgFileContent: `
|
||||
index = "myindex"
|
||||
frame = "f1"
|
||||
`,
|
||||
validation: func() error {
|
||||
v := validator{}
|
||||
v.Check(cmd.Backuper.Host, "localhost:12345")
|
||||
v.Check(cmd.Backuper.Index, "myindex")
|
||||
v.Check(cmd.Backuper.Frame, "f1")
|
||||
v.Check(cmd.Backuper.Path, "/somefile")
|
||||
return v.Error()
|
||||
},
|
||||
},
|
||||
}
|
||||
executeDry(t, tests)
|
||||
}
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/pilosa/pilosa/ctl"
|
||||
)
|
||||
|
||||
var Restorer *ctl.RestoreCommand
|
||||
|
||||
func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
|
||||
Restorer = ctl.NewRestoreCommand(os.Stdin, os.Stdout, os.Stderr)
|
||||
|
||||
restoreCmd := &cobra.Command{
|
||||
Use: "restore",
|
||||
Short: "Restore data to pilosa from a backup file.",
|
||||
Long: `
|
||||
Restores a view to the cluster from a backup file.
|
||||
`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if err := Restorer.Run(context.Background()); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
flags := restoreCmd.Flags()
|
||||
flags.StringVarP(&Restorer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
|
||||
flags.StringVarP(&Restorer.Index, "index", "i", "", "Pilosa index to restore into.")
|
||||
flags.StringVarP(&Restorer.Frame, "frame", "f", "", "Frame to restore into.")
|
||||
flags.StringVarP(&Restorer.View, "view", "v", "", "View to restore into.")
|
||||
flags.StringVarP(&Restorer.Path, "input-file", "d", "", "File to restore data from.")
|
||||
ctl.SetTLSConfig(flags, &Restorer.TLS.CertificatePath, &Restorer.TLS.CertificateKeyPath, &Restorer.TLS.SkipVerify)
|
||||
|
||||
return restoreCmd
|
||||
}
|
||||
|
||||
func init() {
|
||||
subcommandFns["restore"] = NewRestoreCommand
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cmd_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/pilosa/pilosa/cmd"
|
||||
)
|
||||
|
||||
func TestRestoreHelp(t *testing.T) {
|
||||
output, err := ExecNewRootCommand(t, "restore", "--help")
|
||||
if !strings.Contains(output, "Usage:") ||
|
||||
!strings.Contains(output, "Flags:") ||
|
||||
!strings.Contains(output, "pilosa restore") || err != nil {
|
||||
t.Fatalf("Command 'restore --help' not working, err: '%v', output: '%s'", err, output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreConfig(t *testing.T) {
|
||||
tests := []commandTest{
|
||||
{
|
||||
args: []string{"restore", "--input-file", "/somefile"},
|
||||
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
|
||||
cfgFileContent: `
|
||||
index = "myindex"
|
||||
frame = "f1"
|
||||
`,
|
||||
validation: func() error {
|
||||
v := validator{}
|
||||
v.Check(cmd.Restorer.Host, "localhost:12345")
|
||||
v.Check(cmd.Restorer.Index, "myindex")
|
||||
v.Check(cmd.Restorer.Frame, "f1")
|
||||
v.Check(cmd.Restorer.Path, "/somefile")
|
||||
return v.Error()
|
||||
},
|
||||
},
|
||||
}
|
||||
executeDry(t, tests)
|
||||
}
|
||||
|
|
@ -1,94 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ctl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/pilosa/pilosa"
|
||||
"github.com/pilosa/pilosa/server"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// BackupCommand represents a command for backing up a view.
|
||||
type BackupCommand struct {
|
||||
// Destination host and port.
|
||||
Host string
|
||||
|
||||
// Name of the index, frame, view to backup.
|
||||
Index string
|
||||
Frame string
|
||||
View string
|
||||
|
||||
// Output file to write to.
|
||||
Path string
|
||||
|
||||
// Standard input/output
|
||||
*pilosa.CmdIO
|
||||
|
||||
TLS server.TLSConfig
|
||||
}
|
||||
|
||||
// NewBackupCommand returns a new instance of BackupCommand.
|
||||
func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand {
|
||||
return &BackupCommand{
|
||||
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
|
||||
}
|
||||
}
|
||||
|
||||
// Run executes the backup.
|
||||
func (cmd *BackupCommand) Run(ctx context.Context) error {
|
||||
// Validate arguments.
|
||||
if cmd.Path == "" {
|
||||
return errors.New("output file required")
|
||||
}
|
||||
|
||||
// Create a client to the server.
|
||||
client, err := CommandClient(cmd)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "creating client")
|
||||
}
|
||||
|
||||
// Open output file.
|
||||
f, err := os.Create(cmd.Path)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "creating file")
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// Begin streaming backup.
|
||||
if err := client.BackupTo(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
|
||||
return errors.Wrap(err, "backing up")
|
||||
}
|
||||
|
||||
// Sync & close file to ensure durability.
|
||||
if err := f.Sync(); err != nil {
|
||||
return errors.Wrap(err, "syncing")
|
||||
} else if err = f.Close(); err != nil {
|
||||
return errors.Wrap(err, "closing file")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cmd *BackupCommand) TLSHost() string {
|
||||
return cmd.Host
|
||||
}
|
||||
|
||||
func (cmd *BackupCommand) TLSConfiguration() server.TLSConfig {
|
||||
return cmd.TLS
|
||||
}
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ctl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
"github.com/pilosa/pilosa"
|
||||
"github.com/pilosa/pilosa/test"
|
||||
)
|
||||
|
||||
func TestBackupCommand_FileRequired(t *testing.T) {
|
||||
buf := bytes.Buffer{}
|
||||
stdin, stdout, stderr := GetIO(buf)
|
||||
|
||||
cm := NewBackupCommand(stdin, stdout, stderr)
|
||||
err := cm.Run(context.Background())
|
||||
if err.Error() != "output file required" {
|
||||
t.Fatalf("expect error: output file required, actual: %s", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestBackupCommand_Run(t *testing.T) {
|
||||
|
||||
buf := bytes.Buffer{}
|
||||
stdin, stdout, stderr := GetIO(buf)
|
||||
|
||||
hldr := test.MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
s := test.NewServer()
|
||||
defer s.Close()
|
||||
|
||||
s.Handler.API.Cluster = test.NewCluster(1)
|
||||
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
|
||||
s.Handler.API.Holder = hldr.Holder
|
||||
cm := NewBackupCommand(stdin, stdout, stderr)
|
||||
file, err := ioutil.TempFile("", "import.csv")
|
||||
|
||||
cm.Index = "i"
|
||||
cm.Host = s.Host()
|
||||
cm.Frame = "f"
|
||||
cm.View = pilosa.ViewStandard
|
||||
cm.Path = file.Name()
|
||||
err = cm.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Command not working, error: '%s'", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -15,6 +15,7 @@
|
|||
package ctl
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
|
|
@ -86,3 +87,12 @@ func TestBenchCommand_Run(t *testing.T) {
|
|||
fmt.Println(buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// declare stdin, stdout, stderr
|
||||
func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
|
||||
rder := []byte{}
|
||||
stdin := bytes.NewReader(rder)
|
||||
stdout := bufio.NewWriter(&buf)
|
||||
stderr := bufio.NewWriter(&buf)
|
||||
return stdin, stdout, stderr
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,87 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ctl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/pilosa/pilosa"
|
||||
"github.com/pilosa/pilosa/server"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// RestoreCommand represents a command for restoring a frame from a backup.
|
||||
type RestoreCommand struct {
|
||||
// Destination host and port.
|
||||
Host string
|
||||
|
||||
// Name of the index & frame to backup.
|
||||
Index string
|
||||
Frame string
|
||||
View string
|
||||
|
||||
// Import file to read from.
|
||||
Path string
|
||||
|
||||
// Standard input/output
|
||||
*pilosa.CmdIO
|
||||
|
||||
TLS server.TLSConfig
|
||||
}
|
||||
|
||||
// NewRestoreCommand returns a new instance of RestoreCommand.
|
||||
func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand {
|
||||
return &RestoreCommand{
|
||||
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
|
||||
}
|
||||
}
|
||||
|
||||
// Run executes the restore command.
|
||||
func (cmd *RestoreCommand) Run(ctx context.Context) error {
|
||||
// Validate arguments.
|
||||
if cmd.Path == "" {
|
||||
return errors.New("backup file required")
|
||||
}
|
||||
|
||||
// Create a client to the server.
|
||||
client, err := CommandClient(cmd)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "creating client")
|
||||
}
|
||||
|
||||
// Open backup file.
|
||||
f, err := os.Open(cmd.Path)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "opening file")
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// Restore backup file to the cluster.
|
||||
if err := client.RestoreFrom(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
|
||||
return errors.Wrap(err, "restoring")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cmd *RestoreCommand) TLSHost() string {
|
||||
return cmd.Host
|
||||
}
|
||||
|
||||
func (cmd *RestoreCommand) TLSConfiguration() server.TLSConfig {
|
||||
return cmd.TLS
|
||||
}
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ctl
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
"github.com/pilosa/pilosa"
|
||||
"github.com/pilosa/pilosa/test"
|
||||
)
|
||||
|
||||
func TestRestoreCommand_FileRequired(t *testing.T) {
|
||||
cm := RestoreCommand{}
|
||||
ctx := context.Background()
|
||||
err := cm.Run(ctx)
|
||||
if err.Error() != "backup file required" {
|
||||
t.Fatalf("expect error: output file required, actual: '%s'", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreCommand_Run(t *testing.T) {
|
||||
file, err := ioutil.TempFile("", "restore.csv")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buf := bytes.Buffer{}
|
||||
stdin, stdout, stderr := GetIO(buf)
|
||||
|
||||
hldr := test.MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
s := test.NewServer()
|
||||
defer s.Close()
|
||||
|
||||
s.Handler.API.Cluster = test.NewCluster(1)
|
||||
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
|
||||
s.Handler.API.Holder = hldr.Holder
|
||||
|
||||
cm := NewRestoreCommand(stdin, stdout, stderr)
|
||||
cm.Path = file.Name()
|
||||
cm.Index = "i"
|
||||
cm.Frame = "f"
|
||||
cm.View = pilosa.ViewStandard
|
||||
cm.Host = s.Host()
|
||||
cm.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Backup Run doesn't work: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// declare stdin, stdout, stderr
|
||||
func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
|
||||
rder := []byte{}
|
||||
stdin := bytes.NewReader(rder)
|
||||
stdout := bufio.NewWriter(&buf)
|
||||
stderr := bufio.NewWriter(&buf)
|
||||
return stdin, stdout, stderr
|
||||
}
|
||||
|
|
@ -49,7 +49,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
|
|||
pilosa [command]
|
||||
|
||||
Available Commands:
|
||||
backup Backup data from pilosa.
|
||||
bench Benchmark operations.
|
||||
check Do a consistency check on a pilosa data file.
|
||||
config Print the current configuration.
|
||||
|
|
@ -58,7 +57,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
|
|||
help Help about any command
|
||||
import Bulk load data into pilosa.
|
||||
inspect Get stats on a pilosa data file.
|
||||
restore Restore data to pilosa from a backup file.
|
||||
server Run Pilosa.
|
||||
|
||||
Flags:
|
||||
|
|
@ -110,7 +108,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
|
|||
pilosa [command]
|
||||
|
||||
Available Commands:
|
||||
backup Backup data from pilosa.
|
||||
bench Benchmark operations.
|
||||
check Do a consistency check on a pilosa data file.
|
||||
config Print the current configuration.
|
||||
|
|
@ -119,7 +116,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
|
|||
help Help about any command
|
||||
import Bulk load data into pilosa.
|
||||
inspect Get stats on a pilosa data file.
|
||||
restore Restore data to pilosa from a backup file.
|
||||
server Run Pilosa.
|
||||
|
||||
Flags:
|
||||
|
|
@ -177,7 +173,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
|
|||
pilosa [command]
|
||||
|
||||
Available Commands:
|
||||
backup Backup data from pilosa.
|
||||
bench Benchmark operations.
|
||||
check Do a consistency check on a pilosa data file.
|
||||
config Print the current configuration.
|
||||
|
|
@ -186,7 +181,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
|
|||
help Help about any command
|
||||
import Bulk load data into pilosa.
|
||||
inspect Get stats on a pilosa data file.
|
||||
restore Restore data to pilosa from a backup file.
|
||||
server Run Pilosa.
|
||||
|
||||
Flags:
|
||||
|
|
@ -268,7 +262,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
|
|||
pilosa [command]
|
||||
|
||||
Available Commands:
|
||||
backup Backup data from pilosa.
|
||||
bench Benchmark operations.
|
||||
check Do a consistency check on a pilosa data file.
|
||||
config Print the current configuration.
|
||||
|
|
@ -277,7 +270,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
|
|||
help Help about any command
|
||||
import Bulk load data into pilosa.
|
||||
inspect Get stats on a pilosa data file.
|
||||
restore Restore data to pilosa from a backup file.
|
||||
server Run Pilosa.
|
||||
|
||||
Flags:
|
||||
|
|
@ -335,7 +327,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
|
|||
pilosa [command]
|
||||
|
||||
Available Commands:
|
||||
backup Backup data from pilosa.
|
||||
bench Benchmark operations.
|
||||
check Do a consistency check on a pilosa data file.
|
||||
config Print the current configuration.
|
||||
|
|
@ -344,7 +335,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
|
|||
help Help about any command
|
||||
import Bulk load data into pilosa.
|
||||
inspect Get stats on a pilosa data file.
|
||||
restore Restore data to pilosa from a backup file.
|
||||
server Run Pilosa.
|
||||
|
||||
Flags:
|
||||
|
|
|
|||
77
handler.go
77
handler.go
|
|
@ -144,7 +144,6 @@ func NewRouter(handler *Handler) *mux.Router {
|
|||
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST")
|
||||
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
|
||||
router.Handle("/debug/vars", expvar.Handler()).Methods("GET")
|
||||
router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData")
|
||||
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
|
||||
router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups)
|
||||
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
|
||||
|
|
@ -159,7 +158,6 @@ func NewRouter(handler *Handler) *mux.Router {
|
|||
router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport")
|
||||
router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET")
|
||||
router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks")
|
||||
router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST").Name("PostFragmentData")
|
||||
router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
|
||||
router.HandleFunc("/import", handler.handlePostImport).Methods("POST")
|
||||
router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST")
|
||||
|
|
@ -172,7 +170,6 @@ func NewRouter(handler *Handler) *mux.Router {
|
|||
router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST")
|
||||
router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE")
|
||||
router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST")
|
||||
router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST").Name("PostFrameRestore")
|
||||
router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST")
|
||||
router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET")
|
||||
router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE")
|
||||
|
|
@ -1040,48 +1037,6 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
|
|||
}
|
||||
}
|
||||
|
||||
// handleGetFragmentData handles GET /fragment/data requests.
|
||||
func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) {
|
||||
// Read slice parameter.
|
||||
q := r.URL.Query()
|
||||
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "slice required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Retrieve fragment from holder.
|
||||
f, err := h.API.MarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// Stream fragment to response body.
|
||||
if _, err := f.WriteTo(w); err != nil {
|
||||
h.Logger.Printf("fragment backup error: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// handlePostFragmentData handles POST /fragment/data requests.
|
||||
func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) {
|
||||
// Read slice parameter.
|
||||
q := r.URL.Query()
|
||||
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "slice required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err = h.API.UnmarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice, r.Body); err != nil {
|
||||
if errors.Cause(err) == ErrFrameNotFound {
|
||||
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
|
||||
} else {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleGetFragmentBlockData handles GET /fragment/block/data requests.
|
||||
func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) {
|
||||
buf, err := h.API.FragmentBlockData(r.Context(), r.Body)
|
||||
|
|
@ -1134,38 +1089,6 @@ type getFragmentBlocksResponse struct {
|
|||
Blocks []FragmentBlock `json:"blocks"`
|
||||
}
|
||||
|
||||
// handlePostFrameRestore handles POST /frame/restore requests.
|
||||
func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) {
|
||||
indexName := mux.Vars(r)["index"]
|
||||
frameName := mux.Vars(r)["frame"]
|
||||
|
||||
q := r.URL.Query()
|
||||
hostStr := q.Get("host")
|
||||
|
||||
// Validate query parameters.
|
||||
if hostStr == "" {
|
||||
http.Error(w, "host required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
host, err := NewURIFromAddress(hostStr)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
}
|
||||
|
||||
err = h.API.RestoreFrame(r.Context(), indexName, frameName, host)
|
||||
switch errors.Cause(err) {
|
||||
case nil:
|
||||
break
|
||||
case ErrFrameNotFound:
|
||||
fallthrough
|
||||
case ErrFragmentNotFound:
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
default:
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
// handleGetVersion handles /version requests.
|
||||
func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) {
|
||||
err := json.NewEncoder(w).Encode(struct {
|
||||
|
|
|
|||
|
|
@ -1056,55 +1056,6 @@ type FrameFields struct {
|
|||
Fields []pilosa.Field
|
||||
}
|
||||
|
||||
// Ensure the handler can backup a fragment and then restore it.
|
||||
func TestHandler_Fragment_BackupRestore(t *testing.T) {
|
||||
hldr := test.MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
s := test.NewServer()
|
||||
s.Handler.API.Holder = hldr.Holder
|
||||
defer s.Close()
|
||||
|
||||
// Set bits in the index.
|
||||
f0 := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
|
||||
f0.MustSetBits(100, 1, 2, 3)
|
||||
|
||||
// Begin backing up from slice i/f/0.
|
||||
resp, err := http.Get(s.URL + "/fragment/data?index=i&frame=f&view=standard&slice=0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Ensure response came back OK.
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("unexpected backup status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Create frame.
|
||||
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Restore backup to slice x/y/0.
|
||||
if resp, err := http.Post(s.URL+"/fragment/data?index=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
t.Fatalf("unexpected restore status code: %d", resp.StatusCode)
|
||||
} else {
|
||||
resp.Body.Close()
|
||||
}
|
||||
|
||||
// Verify data is correctly restored.
|
||||
f1 := hldr.Fragment("x", "y", pilosa.ViewStandard, 0)
|
||||
if f1 == nil {
|
||||
t.Fatal("fragment x/y/standard/0 not created")
|
||||
} else if columns := f1.Row(100).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 3}) {
|
||||
t.Fatalf("unexpected restored columns: %+v", columns)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the handler can retrieve the version.
|
||||
func TestHandler_Version(t *testing.T) {
|
||||
hldr := test.MustOpenHolder()
|
||||
|
|
|
|||
|
|
@ -235,80 +235,6 @@ func TestMain_SetColumnAttrs(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure program can set columns on one cluster and then restore to a second cluster.
|
||||
func TestMain_FrameRestore(t *testing.T) {
|
||||
mains1 := test.MustRunMainWithCluster(t, 2)
|
||||
m10 := mains1[0]
|
||||
m11 := mains1[1]
|
||||
|
||||
// Create frames.
|
||||
client := m10.Client()
|
||||
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatal("create index:", err)
|
||||
}
|
||||
if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal("create frame:", err)
|
||||
}
|
||||
|
||||
// Write data on first cluster.
|
||||
if _, err := m10.Query("i", "", `
|
||||
SetBit(row=1, frame="f", col=100)
|
||||
SetBit(row=1, frame="f", col=1000)
|
||||
SetBit(row=1, frame="f", col=100000)
|
||||
SetBit(row=1, frame="f", col=200000)
|
||||
SetBit(row=1, frame="f", col=400000)
|
||||
SetBit(row=1, frame="f", col=600000)
|
||||
SetBit(row=1, frame="f", col=800000)
|
||||
`); err != nil {
|
||||
t.Fatal("setting columns:", err)
|
||||
}
|
||||
|
||||
// Query row on first cluster.
|
||||
if res, err := m10.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil {
|
||||
t.Fatal("bitmap query:", err)
|
||||
} else if res != `{"results":[{"attrs":{},"columns":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
}
|
||||
|
||||
// Start second cluster.
|
||||
mains2 := test.MustRunMainWithCluster(t, 2)
|
||||
m20 := mains2[0]
|
||||
defer m20.Close()
|
||||
m21 := mains2[1]
|
||||
defer m21.Close()
|
||||
|
||||
// Import from first cluster.
|
||||
client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), server.GetHTTPClient(nil))
|
||||
if err != nil {
|
||||
t.Fatal("new client:", err)
|
||||
}
|
||||
client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), server.GetHTTPClient(nil))
|
||||
if err != nil {
|
||||
t.Fatal("new client:", err)
|
||||
}
|
||||
|
||||
if err := m20.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatal("create new index:", err)
|
||||
}
|
||||
if err := m20.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal("create new frame:", err)
|
||||
}
|
||||
|
||||
if err := client20.RestoreFrame(context.Background(), m10.Server.URI.HostPort(), "i", "f"); err != nil {
|
||||
t.Fatal("restore frame:", err)
|
||||
}
|
||||
if err := client21.RestoreFrame(context.Background(), m11.Server.URI.HostPort(), "i", "f"); err != nil {
|
||||
t.Fatal("restore frame:", err)
|
||||
}
|
||||
|
||||
// Query row on second cluster.
|
||||
if res, err := m20.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil {
|
||||
t.Fatal("another bitmap query:", err)
|
||||
} else if res != `{"results":[{"attrs":{},"columns":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
|
||||
t.Fatalf("2unexpected result: %s", res)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the host can be parsed.
|
||||
func TestConfig_Parse_Host(t *testing.T) {
|
||||
if c, err := ParseConfig(`bind = "local"`); err != nil {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue