scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-06 00:25:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #251 from jaffee/sshenaningans

Browse source 
Sshenanigans
This commit is contained in:
Matthew Jaffee 2017-01-13 11:45:14 -06:00 • committed by GitHub
commit c3cdf94207
4 changed files with 144 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pilosactl/build.go → build/build.go
View file

@ -1,4 +1,4 @@
package pilosactl
package build
import (
"fmt"
@ -8,7 +8,7 @@ import (
"os/exec"
)
func BuildBinary(pkg, goos, goarch string) (io.Reader, error) {
func Binary(pkg, goos, goarch string) (io.Reader, error) {
binFile, err := ioutil.TempFile("", "pilosactl")
if err != nil {
return nil, fmt.Errorf("build binary: %v", err)

98
cmd/pilosactl/main.go
View file

@ -27,9 +27,11 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/bench"
"github.com/pilosa/pilosa/build"
"github.com/pilosa/pilosa/creator"
"github.com/pilosa/pilosa/pilosactl"
"github.com/pilosa/pilosa/roaring"
pssh "github.com/pilosa/pilosa/ssh"
"github.com/satori/go.uuid"
"golang.org/x/crypto/ssh"
@ -1001,6 +1003,10 @@ type CreateCommand struct {
SSHUser string
CopyBinary bool
GOOS string
GOARCH string
// Standard input/output
Stdin io.Reader
Stdout io.Writer
@ -1028,6 +1034,9 @@ func (cmd *CreateCommand) ParseFlags(args []string) error {
fs.IntVar(&cmd.GoMaxProcs, "gomaxprocs", 0, "")
fs.StringVar(&hosts, "hosts", "", "")
fs.StringVar(&cmd.SSHUser, "ssh-user", "", "")
fs.BoolVar(&cmd.CopyBinary, "copy-binary", false, "")
fs.StringVar(&cmd.GOOS, "goos", "linux", "")
fs.StringVar(&cmd.GOARCH, "goarch", "amd64", "")
if err := fs.Parse(args); err != nil {
return err
@ -1071,6 +1080,16 @@ The following flags are allowed:
when starting a cluster on remote hosts, this
will set the value of GOMAXPROCS.
-copy-binary
controls whether or not to build and copy pilosa to agents
-goos
when using copy-binary, GOOS to use while building binary
-goarch
when using copy-binary, GOARCH to use while building binary
`)
}
@ -1097,6 +1116,9 @@ func (cmd *CreateCommand) Run(ctx context.Context) error {
SSHUser: cmd.SSHUser,
Stderr: cmd.Stderr,
GoMaxProcs: cmd.GoMaxProcs,
CopyBinary: cmd.CopyBinary,
GOOS: cmd.GOOS,
GOARCH: cmd.GOARCH,
}
default:
return fmt.Errorf("Unknown cluster type %v", cmd.Type)
@ -1303,7 +1325,6 @@ func (cmd *BagentCommand) Run(ctx context.Context) error {
if err != nil {
fmt.Fprintln(cmd.Stderr, err)
}
// fmt.Fprintln(cmd.Stdout, res)
return nil
}
@ -1326,15 +1347,17 @@ type BspawnCommand struct {
// locally.
AgentHosts []string
// If this is true, build and copy pilosactl binary to agent hosts.
CopyBinary bool
// Makes output human readable
Human bool
HumanReadable bool
// Result destination, ["stdout", "s3"]
Output string
// If this is true, build and copy pilosactl binary to agent hosts.
CopyBinary bool
GOOS string
GOARCH string
// Benchmarks is a slice of Spawns which specifies all of the bagent
// commands to run. These will all be run in parallel, started on each
// of the agents in a round robin fashion.
@ -1370,12 +1393,15 @@ func (cmd *BspawnCommand) ParseFlags(args []string) error {
fs.SetOutput(ioutil.Discard)
creatorHosts := fs.String("creator.hosts", "", "")
creatorLFP := fs.String("creator.log-file-prefix", "", "")
creatorCopyBinary := fs.Bool("creator.copy-binary", false, "")
pilosaHosts := fs.String("pilosa-hosts", "", "")
agentHosts := fs.String("agent-hosts", "", "")
sshUser := fs.String("ssh-user", "", "")
fs.BoolVar(&cmd.Human, "human", false, "")
fs.BoolVar(&cmd.HumanReadable, "human", false, "")
fs.StringVar(&cmd.Output, "output", "stdout", "")
fs.BoolVar(&cmd.CopyBinary, "copy-binary", false, "")
fs.StringVar(&cmd.GOOS, "goos", "linux", "")
fs.StringVar(&cmd.GOARCH, "goarch", "amd64", "")
err := fs.Parse(args)
if err != nil {
@ -1404,7 +1430,10 @@ func (cmd *BspawnCommand) ParseFlags(args []string) error {
}
// TODO support all creator args - just checking for creatorHosts here won't be sufficient
if *creatorHosts != "" {
cmd.CreatorArgs = []string{"-hosts=" + *creatorHosts, "-log-file-prefix=" + *creatorLFP, "-ssh-user=" + cmd.SSHUser}
cmd.CreatorArgs = []string{"-hosts=" + *creatorHosts, "-log-file-prefix=" + *creatorLFP, "-ssh-user=" + cmd.SSHUser, "-goos=" + cmd.GOOS, "-goarch=" + cmd.GOARCH}
if *creatorCopyBinary {
cmd.CreatorArgs = append(cmd.CreatorArgs, "-copy-binary")
}
}
return nil
@ -1427,6 +1456,9 @@ The following flags are allowed and will override the values in the config file:
-creator.log-file-prefix
log-file-prefix argument for pilosactl create
-creator.copy-binary
pilosactl create should build and copy pilosa binary to cluster
-pilosa-hosts
pilosa hosts to run against (will ignore creator args)
@ -1434,16 +1466,24 @@ The following flags are allowed and will override the values in the config file:
hosts to use for benchmark agents
-ssh-user
pilosa hosts to run against (will ignore creator args)
username to use when contacting remote hosts
-human
toggle human readable output (indented json)
-output
string to select output destination, "stdout" or "s3"
-copy-binary
controls whether or not to build and copy pilosactl to agents
-human
toggle human readable output (indented json with formatted times)
-goos
when using copy-binary, GOOS to use while building binary
-goarch
when using copy-binary, GOARCH to use while building binary
-output
string to select output destination, "stdout" or "s3"
`)
}
@ -1493,42 +1533,30 @@ func (cmd *BspawnCommand) Run(ctx context.Context) error {
} else {
return fmt.Errorf("invalid bspawn output destination")
}
enc := json.NewEncoder(writer)
if cmd.Human {
enc := json.NewEncoder(writer)
if cmd.HumanReadable {
enc.SetIndent("", " ")
output = bench.Prettify(output)
}
return enc.Encode(output)
}
func copyBinary(fleet pilosactl.SSHFleet, pkg, goos, goarch string) error {
bin, err := pilosactl.BuildBinary(pkg, goos, goarch)
if err != nil {
return err
}
wc, err := fleet.OpenFile(path.Base(pkg), "+x")
if err != nil {
return err
}
_, err = io.Copy(wc, bin)
if err != nil {
return err
}
return wc.Close()
}
func (cmd *BspawnCommand) spawnRemote(ctx context.Context) (map[string]interface{}, error) {
agentIndex := 0
agentConnections, err := pilosactl.SSHClients(cmd.AgentHosts, cmd.SSHUser, "", cmd.Stderr)
agentFleet, err := pssh.NewFleet(cmd.AgentHosts, cmd.SSHUser, "", cmd.Stderr)
if err != nil {
return nil, err
}
if cmd.CopyBinary {
err = copyBinary(agentConnections, "github.com/pilosa/pilosa/cmd/pilosactl", "linux", "amd64")
pkg := "github.com/pilosa/pilosa/cmd/pilosactl"
bin, err := build.Binary(pkg, cmd.GOOS, cmd.GOARCH)
if err != nil {
return nil, err
}
err = agentFleet.WriteFile(path.Base(pkg), "+x", bin)
if err != nil {
return nil, err
}
@ -1541,7 +1569,7 @@ func (cmd *BspawnCommand) spawnRemote(ctx context.Context) (map[string]interface
for _, sp := range cmd.Benchmarks {
results[sp.Name] = make(map[int]interface{})
for i := 0; i < sp.Num; i++ {
sess, err := agentConnections[agentIndex].NewSession()
sess, err := agentFleet[agentIndex].NewSession()
if err != nil {
return nil, err
}

37
creator/remote.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"io"
"net"
"path"
"strconv"
"sync"
@ -11,7 +12,8 @@ import (
"github.com/BurntSushi/toml"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pilosactl"
"github.com/pilosa/pilosa/build"
pssh "github.com/pilosa/pilosa/ssh"
"golang.org/x/crypto/ssh"
)
@ -22,6 +24,9 @@ type RemoteCluster struct {
Keyfile string
Key []byte
GoMaxProcs int
CopyBinary bool
GOOS string
GOARCH string
Stderr io.Writer
wg *sync.WaitGroup
logs []io.Reader
@ -38,8 +43,26 @@ func (c *RemoteCluster) Start() error {
if len(c.ClusterHosts) == 0 {
return fmt.Errorf("no type or hosts specified - cannot continue")
}
// TODO: build pilosa
// TODO: copy binary to hosts
if c.CopyBinary {
fleet, err := pssh.NewFleet(c.ClusterHosts, c.SSHUser, c.Keyfile, c.Stderr)
if err != nil {
return fmt.Errorf("copying binary: %v", err)
}
pkg := "github.com/pilosa/pilosa/cmd/pilosa"
bin, err := build.Binary(pkg, c.GOOS, c.GOARCH)
if err != nil {
return fmt.Errorf("building binary: %v", err)
}
err = fleet.WriteFile(path.Base(pkg), "+x", bin)
if err != nil {
return fmt.Errorf("writing binary to fleet: %v", err)
}
}
// build config
conf := pilosa.NewConfigForHosts(c.ClusterHosts)
conf.Cluster.ReplicaN = c.ReplicaN
@ -51,15 +74,15 @@ func (c *RemoteCluster) Start() error {
// Set up config for this host
host, port, err := net.SplitHostPort(hostport)
if err != nil {
return err
return fmt.Errorf("splitting hostport: %v", err)
}
conf.Host = hostport
conf.DataDir = "~/.pilosa" + port
// Connect to remote host
client, err := pilosactl.NewSSH(host, c.SSHUser, "", c.Stderr)
client, err := pssh.NewClient(host, c.SSHUser, "", c.Stderr)
if err != nil {
return err
return fmt.Errorf("connecting to host: %v", err)
}
configname := "pilosa" + port + ".conf"
w, err := client.OpenFile(configname, "")
@ -109,7 +132,7 @@ func (c *RemoteCluster) Start() error {
gomaxprocsString = "GOMAXPROCS=" + strconv.Itoa(c.GoMaxProcs) + " "
}
err = sess.Start(gomaxprocsString + "pilosa -config " + configname)
err = sess.Start("PATH=.:$PATH " + gomaxprocsString + "pilosa -config " + configname)
if err != nil {
return err
}

62
pilosactl/ssh.go → ssh/ssh.go
View file

@ -1,4 +1,4 @@
package pilosactl
package ssh
import (
"fmt"
@ -14,15 +14,15 @@ import (
"golang.org/x/crypto/ssh/agent"
)
type SSH struct {
type Client struct {
client *ssh.Client
Stderr io.Writer
}
// NewSSH wraps up some of the complexity of using the crypto/ssh pacakge
// NewClient wraps up some of the complexity of using the crypto/ssh pacakge
// directly assuming you want to connect using public key auth and you can pass
// a keyfile or your key is accessible through ssh agent.
func NewSSH(host, username, keyfile string, stderr io.Writer) (*SSH, error) {
func NewClient(host, username, keyfile string, stderr io.Writer) (*Client, error) {
if username == "" {
user, err := user.Current()
if err != nil {
@ -53,18 +53,19 @@ func NewSSH(host, username, keyfile string, stderr io.Writer) (*SSH, error) {
client, err := ssh.Dial("tcp", host, config)
if err != nil {
return nil, fmt.Errorf("NewSHH failed Dial - host: %v, config: %v, err: %v ", host, config, err)
return nil, fmt.Errorf("NewSSH failed Dial - host: %v, config: %v, err: %v ", host, config, err)
}
return &SSH{client: client, Stderr: stderr}, nil
return &Client{client: client, Stderr: stderr}, nil
}
type SSHFleet []*SSH
type Fleet []*Client
func SSHClients(hosts []string, username, keyfile string, stderr io.Writer) (SSHFleet, error) {
clients := make([]*SSH, len(hosts))
func NewFleet(hosts []string, username, keyfile string, stderr io.Writer) (Fleet, error) {
hosts = DedupHosts(hosts)
clients := make([]*Client, len(hosts))
for i, host := range hosts {
client, err := NewSSH(host, username, keyfile, stderr)
client, err := NewClient(host, username, keyfile, stderr)
if err != nil {
return nil, err
}
@ -73,7 +74,25 @@ func SSHClients(hosts []string, username, keyfile string, stderr io.Writer) (SSH
return clients, nil
}
func (s *SSH) NewSession() (*ssh.Session, error) {
// DedupHosts takes a slice of hosts, strips off any specified ports, and
// returns a de-duplicated slice of hosts.
func DedupHosts(hosts []string) []string {
seenHosts := make(map[string]bool)
ret := []string{}
for _, h := range hosts {
colonIdx := strings.Index(h, ":")
if colonIdx != -1 {
h = h[:colonIdx]
}
if !seenHosts[h] {
ret = append(ret, h)
}
seenHosts[h] = true
}
return ret
}
func (s *Client) NewSession() (*ssh.Session, error) {
return s.client.NewSession()
}
@ -100,7 +119,7 @@ func (r *remoteFile) Close() error {
// will be passed directly to chmod to set the file permissions. rm, touch,
// chmod, cat and support for semicolons, double ampersand, and output
// redirection (>>) must be available in the remote shell.
func (s *SSH) OpenFile(name, perm string) (io.WriteCloser, error) {
func (s *Client) OpenFile(name, perm string) (io.WriteCloser, error) {
sess, err := s.NewSession()
if err != nil {
return nil, err
@ -156,7 +175,7 @@ func (mwc *multiWriteCloser) Close() error {
return nil
}
func (sf SSHFleet) OpenFile(name, perm string) (io.WriteCloser, error) {
func (sf Fleet) OpenFile(name, perm string) (io.WriteCloser, error) {
writers := newMultiWriteCloser()
for _, cli := range sf {
wc, err := cli.OpenFile(name, "+x")
@ -167,3 +186,20 @@ func (sf SSHFleet) OpenFile(name, perm string) (io.WriteCloser, error) {
}
return writers, nil
}
// WriteFile writes all of data (until EOF) into a file with the given name on
// each of the hosts in the fleet. It sets the permissions on the file to <perm>
// which is any valid input to chmod. If <perm> is the empty string, it defaults
// to 0664
func (sf Fleet) WriteFile(name, perm string, data io.Reader) error {
wc, err := sf.OpenFile(name, perm)
if err != nil {
return err
}
_, err = io.Copy(wc, data)
if err != nil {
return err
}
return wc.Close()
}