From 38933e742bd5cded0e905ff5decd96acfc3903e3 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 11 Jan 2017 08:40:40 -0600 Subject: [PATCH 01/11] hardcode path --- cmd/pilosactl/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 2932f0fbf..e8ab8d7ef 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -1547,7 +1547,7 @@ func (cmd *BspawnCommand) spawnRemote(ctx context.Context) (map[string]interface resLock.Unlock() }(stdout, sp.Name, i) sess.Stderr = cmd.Stderr - err = sess.Start("PATH=.:$PATH pilosactl bagent -agent-num=" + strconv.Itoa(i) + " -hosts=" + strings.Join(cmd.PilosaHosts, ",") + " " + strings.Join(sp.Args, " ")) + err = sess.Start("PATH=.:$PATH /usr/local/bin/pilosactl bagent -agent-num=" + strconv.Itoa(i) + " -hosts=" + strings.Join(cmd.PilosaHosts, ",") + " " + strings.Join(sp.Args, " ")) if err != nil { return nil, err } From bb1157394d2bbd23815cdf946354de44bf7f174e Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 11 Jan 2017 09:18:28 -0600 Subject: [PATCH 02/11] Add output destination flag --- cmd/pilosactl/main.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index e8ab8d7ef..d10a8bb52 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -1171,6 +1171,9 @@ type BagentCommand struct { // Enable pretty printing of results, for human consumption. HumanReadable bool `json:"human-readable"` + // Result destination, ["stdout", "aws"] + Output string `json:"output"` + // Slice of pilosa hosts to run the Benchmarks against. Hosts []string `json:"hosts"` @@ -1186,6 +1189,7 @@ func NewBagentCommand(stdin io.Reader, stdout, stderr io.Writer) *BagentCommand Hosts: []string{}, AgentNum: 0, HumanReadable: false, + Output: "aws", Stdin: stdin, Stdout: stdout, @@ -1207,6 +1211,7 @@ func (cmd *BagentCommand) ParseFlags(args []string) error { fs.StringVar(&pilosaHosts, "hosts", "localhost:15000", "") fs.IntVar(&cmd.AgentNum, "agent-num", 0, "") fs.BoolVar(&cmd.HumanReadable, "human", false, "") + fs.StringVar(&cmd.Output, "output", "aws", "") if err := fs.Parse(args); err != nil { return err @@ -1273,6 +1278,9 @@ The following arguments are available: -human Boolean to enable human-readable format. + -output + String to select output destination, "stdout" or "aws" + subcommands: diagonal-set-bits random-set-bits @@ -1332,6 +1340,9 @@ type BspawnCommand struct { // Makes output human readable Human bool + // Result destination, ["stdout", "aws"] + Output string + // Benchmarks is a slice of Spawns which specifies all of the bagent // commands to run. These will all be run in parallel, started on each // of the agents in a round robin fashion. @@ -1371,6 +1382,7 @@ func (cmd *BspawnCommand) ParseFlags(args []string) error { agentHosts := fs.String("agent-hosts", "", "") sshUser := fs.String("ssh-user", "", "") fs.BoolVar(&cmd.Human, "human", false, "") + fs.StringVar(&cmd.Output, "output", "aws", "") fs.BoolVar(&cmd.CopyBinary, "copy-binary", false, "") err := fs.Parse(args) @@ -1437,6 +1449,9 @@ The following flags are allowed and will override the values in the config file: -human toggle human readable output (indented json with formatted times) + + -output + string to select output destination, "stdout" or "aws" `) } From 5b586fefc905dbc610b98345ff6b61dc71d5afa1 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 11 Jan 2017 09:25:09 -0600 Subject: [PATCH 03/11] Implement S3 writer --- bench/store.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 bench/store.go diff --git a/bench/store.go b/bench/store.go new file mode 100644 index 000000000..36a64cded --- /dev/null +++ b/bench/store.go @@ -0,0 +1,45 @@ +package bench + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" +) + +type Uploader interface { + Write(p []byte) (n int, err error) +} + +func NewS3Uploader(key string) *S3Uploader { + region := "us-east-1" + return &S3Uploader{ + region, + "benchmarks-pilosa", + s3.New(session.New(&aws.Config{Region: aws.String(region)})), + key, + } +} + +type S3Uploader struct { + region string + bucket string + service *s3.S3 + key string +} + +// pilosa-sandbox is "us-east-1" +// benchmarks go to benchmarks-pilosa/run-uuid/??.txt +// first return value of PutObject contains an ETag (hash) of the uploaded object, returned here but not needed +func (u *S3Uploader) Write(data []byte) (int, error) { + fmt.Println(string(data)) + _, err := u.service.PutObject(&s3.PutObjectInput{ + Body: strings.NewReader(string(data)), + Bucket: &u.bucket, + Key: &u.key, + }) + + return 0, err +} From c1ca299904dfce23e9202368432e0ef84b76e107 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 11 Jan 2017 09:25:37 -0600 Subject: [PATCH 04/11] Send results to S3 --- cmd/pilosactl/main.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index d10a8bb52..c0b110571 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -1492,7 +1492,17 @@ func (cmd *BspawnCommand) Run(ctx context.Context) error { return err } output["results"] = res - enc := json.NewEncoder(cmd.Stdout) + + var writer io.Writer + if cmd.Output == "aws" { + writer = bench.NewS3Uploader(runUUID.String() + ".json") + } else if cmd.Output == "stdout" { + writer = cmd.Stdout + } else { + return fmt.Errorf("invalid bspawn output destination") + } + enc := json.NewEncoder(writer) + if cmd.Human { enc.SetIndent("", " ") output = bench.Prettify(output) From 39c9d1ab250605fcf789870ecce5c861f7d450ff Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 11 Jan 2017 09:38:45 -0600 Subject: [PATCH 05/11] Cleanup and update comments --- bench/store.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/bench/store.go b/bench/store.go index 36a64cded..b838e4bf3 100644 --- a/bench/store.go +++ b/bench/store.go @@ -9,20 +9,20 @@ import ( "github.com/aws/aws-sdk-go/service/s3" ) -type Uploader interface { - Write(p []byte) (n int, err error) -} - +// NewS3Uploader creates an S3Uploader with a hardcoded region and bucket, +// and a specified key. func NewS3Uploader(key string) *S3Uploader { region := "us-east-1" + bucket := "benchmarks-pilosa" return &S3Uploader{ region, - "benchmarks-pilosa", + bucket, s3.New(session.New(&aws.Config{Region: aws.String(region)})), key, } } +// S3Uploader is an io.Writer for sending output to AWS S3 storage. type S3Uploader struct { region string bucket string @@ -30,10 +30,9 @@ type S3Uploader struct { key string } -// pilosa-sandbox is "us-east-1" -// benchmarks go to benchmarks-pilosa/run-uuid/??.txt -// first return value of PutObject contains an ETag (hash) of the uploaded object, returned here but not needed +// Write writes data to the uploader's bucket/key func (u *S3Uploader) Write(data []byte) (int, error) { + // first return value of PutObject contains an ETag (hash) of the uploaded object, not needed here fmt.Println(string(data)) _, err := u.service.PutObject(&s3.PutObjectInput{ Body: strings.NewReader(string(data)), From d1e9ae2924933a47db655aaf1c175b3c07c86fb8 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 11 Jan 2017 10:41:25 -0600 Subject: [PATCH 06/11] Rename 'aws' destination to 's3' --- cmd/pilosactl/main.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index c0b110571..2f0ca92d0 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -1171,7 +1171,7 @@ type BagentCommand struct { // Enable pretty printing of results, for human consumption. HumanReadable bool `json:"human-readable"` - // Result destination, ["stdout", "aws"] + // Result destination, ["stdout", "s3"] Output string `json:"output"` // Slice of pilosa hosts to run the Benchmarks against. @@ -1189,7 +1189,7 @@ func NewBagentCommand(stdin io.Reader, stdout, stderr io.Writer) *BagentCommand Hosts: []string{}, AgentNum: 0, HumanReadable: false, - Output: "aws", + Output: "s3", Stdin: stdin, Stdout: stdout, @@ -1211,7 +1211,7 @@ func (cmd *BagentCommand) ParseFlags(args []string) error { fs.StringVar(&pilosaHosts, "hosts", "localhost:15000", "") fs.IntVar(&cmd.AgentNum, "agent-num", 0, "") fs.BoolVar(&cmd.HumanReadable, "human", false, "") - fs.StringVar(&cmd.Output, "output", "aws", "") + fs.StringVar(&cmd.Output, "output", "s3", "") if err := fs.Parse(args); err != nil { return err @@ -1278,8 +1278,8 @@ The following arguments are available: -human Boolean to enable human-readable format. - -output - String to select output destination, "stdout" or "aws" + -output + String to select output destination, "stdout" or "s3" subcommands: diagonal-set-bits @@ -1340,7 +1340,7 @@ type BspawnCommand struct { // Makes output human readable Human bool - // Result destination, ["stdout", "aws"] + // Result destination, ["stdout", "s3"] Output string // Benchmarks is a slice of Spawns which specifies all of the bagent @@ -1382,7 +1382,7 @@ func (cmd *BspawnCommand) ParseFlags(args []string) error { agentHosts := fs.String("agent-hosts", "", "") sshUser := fs.String("ssh-user", "", "") fs.BoolVar(&cmd.Human, "human", false, "") - fs.StringVar(&cmd.Output, "output", "aws", "") + fs.StringVar(&cmd.Output, "output", "s3", "") fs.BoolVar(&cmd.CopyBinary, "copy-binary", false, "") err := fs.Parse(args) @@ -1450,8 +1450,8 @@ The following flags are allowed and will override the values in the config file: -human toggle human readable output (indented json with formatted times) - -output - string to select output destination, "stdout" or "aws" + -output + string to select output destination, "stdout" or "s3" `) } @@ -1494,7 +1494,7 @@ func (cmd *BspawnCommand) Run(ctx context.Context) error { output["results"] = res var writer io.Writer - if cmd.Output == "aws" { + if cmd.Output == "s3" { writer = bench.NewS3Uploader(runUUID.String() + ".json") } else if cmd.Output == "stdout" { writer = cmd.Stdout From 9fbab68be3422c181a04333b74ab64889b310275 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 11 Jan 2017 13:07:42 -0600 Subject: [PATCH 07/11] Simplify S3Uploader --- bench/store.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/bench/store.go b/bench/store.go index b838e4bf3..6f752cc08 100644 --- a/bench/store.go +++ b/bench/store.go @@ -9,22 +9,17 @@ import ( "github.com/aws/aws-sdk-go/service/s3" ) -// NewS3Uploader creates an S3Uploader with a hardcoded region and bucket, -// and a specified key. -func NewS3Uploader(key string) *S3Uploader { - region := "us-east-1" - bucket := "benchmarks-pilosa" +// NewS3Uploader creates an S3Uploader with specified bucket and key +func NewS3Uploader(bucket string, key string) *S3Uploader { return &S3Uploader{ - region, bucket, - s3.New(session.New(&aws.Config{Region: aws.String(region)})), + s3.New(session.New(&aws.Config{})), key, } } // S3Uploader is an io.Writer for sending output to AWS S3 storage. type S3Uploader struct { - region string bucket string service *s3.S3 key string From bfd405a8ffac92c710bed6da23734e4b7ae88a4b Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 11 Jan 2017 13:10:52 -0600 Subject: [PATCH 08/11] Update NewS3Uploader call --- cmd/pilosactl/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 2f0ca92d0..20ef2fcab 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -1495,7 +1495,7 @@ func (cmd *BspawnCommand) Run(ctx context.Context) error { var writer io.Writer if cmd.Output == "s3" { - writer = bench.NewS3Uploader(runUUID.String() + ".json") + writer = bench.NewS3Uploader("benchmarks-pilosa", runUUID.String()+".json") } else if cmd.Output == "stdout" { writer = cmd.Stdout } else { From 7f25abeb2e21c0aefede5217372b7439a58e1eb8 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 11 Jan 2017 13:11:55 -0600 Subject: [PATCH 09/11] Switch default from s3 to stdout --- cmd/pilosactl/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 20ef2fcab..3b9138ad1 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -1382,7 +1382,7 @@ func (cmd *BspawnCommand) ParseFlags(args []string) error { agentHosts := fs.String("agent-hosts", "", "") sshUser := fs.String("ssh-user", "", "") fs.BoolVar(&cmd.Human, "human", false, "") - fs.StringVar(&cmd.Output, "output", "s3", "") + fs.StringVar(&cmd.Output, "output", "stdout", "") fs.BoolVar(&cmd.CopyBinary, "copy-binary", false, "") err := fs.Parse(args) From a05b7723731769c0294f17c1d3c1c46c3de06d55 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 11 Jan 2017 13:12:19 -0600 Subject: [PATCH 10/11] Remove -output flag from bagent --- cmd/pilosactl/main.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 3b9138ad1..c2ffe4ef6 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -1171,9 +1171,6 @@ type BagentCommand struct { // Enable pretty printing of results, for human consumption. HumanReadable bool `json:"human-readable"` - // Result destination, ["stdout", "s3"] - Output string `json:"output"` - // Slice of pilosa hosts to run the Benchmarks against. Hosts []string `json:"hosts"` @@ -1189,7 +1186,6 @@ func NewBagentCommand(stdin io.Reader, stdout, stderr io.Writer) *BagentCommand Hosts: []string{}, AgentNum: 0, HumanReadable: false, - Output: "s3", Stdin: stdin, Stdout: stdout, @@ -1211,7 +1207,6 @@ func (cmd *BagentCommand) ParseFlags(args []string) error { fs.StringVar(&pilosaHosts, "hosts", "localhost:15000", "") fs.IntVar(&cmd.AgentNum, "agent-num", 0, "") fs.BoolVar(&cmd.HumanReadable, "human", false, "") - fs.StringVar(&cmd.Output, "output", "s3", "") if err := fs.Parse(args); err != nil { return err @@ -1278,9 +1273,6 @@ The following arguments are available: -human Boolean to enable human-readable format. - -output - String to select output destination, "stdout" or "s3" - subcommands: diagonal-set-bits random-set-bits From 53c15af4c4aef01a9d4bcaf025e616e78540838d Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 11 Jan 2017 13:46:32 -0600 Subject: [PATCH 11/11] Remove absolute path to pilosactl --- cmd/pilosactl/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index c2ffe4ef6..e0f2a82d0 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -1564,7 +1564,7 @@ func (cmd *BspawnCommand) spawnRemote(ctx context.Context) (map[string]interface resLock.Unlock() }(stdout, sp.Name, i) sess.Stderr = cmd.Stderr - err = sess.Start("PATH=.:$PATH /usr/local/bin/pilosactl bagent -agent-num=" + strconv.Itoa(i) + " -hosts=" + strings.Join(cmd.PilosaHosts, ",") + " " + strings.Join(sp.Args, " ")) + err = sess.Start("PATH=.:$PATH pilosactl bagent -agent-num=" + strconv.Itoa(i) + " -hosts=" + strings.Join(cmd.PilosaHosts, ",") + " " + strings.Join(sp.Args, " ")) if err != nil { return nil, err }