merge master, resolve conflict

2026-09-07 17:15:56 +00:00 · 2017-01-27 13:59:47 -06:00 · 2017-01-27 13:59:47 -06:00 · da6c5d238c
commit da6c5d238c
parent 1228e177be a927c68f67
8 changed files with 72 additions and 39 deletions
--- a/README.md
+++ b/README.md
@ -235,24 +235,24 @@ Examples:

 Create a 5 node cluster locally (using 5 different ports), with a replication factor of 2.
 ```
-pilosactl create 
-  -serverN 5 
+pilosactl create \
+  -serverN 5 \
  -replicaN 2
 ```

 Create a cluster on 3 remote hosts - all logs will come to local stderr, pilosa binary must be available on remote hosts. The ssh user on the remote hosts needs to be the same as your local user. Otherwise use the `ssh-user` option.
 ```
-pilosactl create 
+pilosactl create \
  -hosts="node1.example.com:15000,node2.example.com:15000,node3.example.com:15000"
 ```

 Create a cluster on 3 remote hosts running OSX, but build the binary locally and copy it up. Stream the stderr of each node to a separate local log file.
 ```
-pilosactl create 
-  -hosts="mac1.example.com:15000,mac2.example.com:15000,mac3.example.com:15000" 
-  -copy-binary 
-  -goos=darwin 
-  -goarch=amd64 
+pilosactl create \
+  -hosts="mac1.example.com:15000,mac2.example.com:15000,mac3.example.com:15000" \
+  -copy-binary \
+  -goos=darwin \
+  -goarch=amd64 \
  -log-file-prefix=clusterlogs
 ```

@ -262,7 +262,9 @@ pilosactl create

 E.G.
 ```
-pilosactl bagent import -h
+pilosactl bagent \
+  -hosts="localhost:15000,localhost:15001" \
+  import -h
 ```

 Multiple subcommands and their arguments may be concatenated at the command line and they will be run serially. This is useful (i.e.) for importing a bunch of data, and then executing queries against it.
@ -270,7 +272,10 @@ Multiple subcommands and their arguments may be concatenated at the command line
 This will generate and import a bunch of data, and then execute random queries against it.

 ```
-pilosactl bagent import -max-bits-per-map=10000 random-query -iterations 100
+pilosactl bagent \
+  -hosts="localhost:15000,localhost:15001" \
+  import -max-bits-per-map=10000 \
+  random-query -iterations 100
 ```

 ### Bspawn
@ -290,7 +295,7 @@ The configuration file is a json object with the top level key `benchmarks`. Thi
        {
            "num": 2,
            "name": "rand-plus-zipf",
-            "args": ["random-set-bits", "-iterations", "20000", "zipf-set-bits", "-iterations", "100"]
+            "args": ["random-set-bits", "-iterations", "20000", "zipf", "-iterations", "100"]
        }
    ]
 }
--- a/bench/bench.go
+++ b/bench/bench.go
@ -13,7 +13,9 @@ type Benchmark interface {
 	// it is being run simultaneously on multiple "agents". E.G. the agentNum
 	// might be used to make a random seed different for each agent, or have
 	// each agent set a different set of bits. Init's doc string should document
-	// how the agentNum affects it.
+	// how the agentNum affects it. Every benchmark should have a 'Name' field
+	// set by init, which appears when the benchmark is marshalled to json as
+	// "name".
 	Init(hosts []string, agentNum int) error

 	// Run runs the benchmark. The return value of Run is kept generic so that
--- a/bench/zipf.go
+++ b/bench/zipf.go
@ -12,10 +12,10 @@ import (
 	"time"
 )

-// ZipfSetBits sets random bits according to the Zipf-Mandelbrot distribution.
+// Zipf sets random bits according to the Zipf-Mandelbrot distribution.
 // This distribution accepts two parameters, Exponent and Ratio, for both bitmaps and profiles.
 // It also uses PermutationGenerator to permute IDs randomly.
-type ZipfSetBits struct {
+type Zipf struct {
 	HasClient
 	Name            string  `json:"name"`
 	BaseBitmapID    int64   `json:"base-bitmap-id"`
@ -29,6 +29,7 @@ type ZipfSetBits struct {
 	BitmapRatio     float64 `json:"bitmap-ratio"`
 	ProfileExponent float64 `json:"profile-exponent"`
 	ProfileRatio    float64 `json:"profile-ratio"`
+	Operation       string  `json:"operation"`
 	bitmapRng       *rand.Zipf
 	profileRng      *rand.Zipf
 	bitmapPerm      *PermutationGenerator
@ -36,9 +37,9 @@ type ZipfSetBits struct {
 }

 // Usage returns the usage message to be printed.
-func (b *ZipfSetBits) Usage() string {
+func (b *Zipf) Usage() string {
 	return `
-zipf-set-bits sets random bits according to the Zipf distribution.
+zipf sets random bits according to the Zipf distribution.
 This is a power-law distribution controlled by two parameters.
 Exponent, in the range (1, inf), with a default value of 1.001, controls
 the "sharpness" of the distribution, with higher exponent being sharper.
@ -47,7 +48,7 @@ maximum variation of the distribution, with higher ratio being more uniform.

 Agent number modifies random seed.

-Usage: zipf-set-bits [arguments]
+Usage: zipf [arguments]

 The following arguments are available:

@ -86,6 +87,9 @@ The following arguments are available:

 	-client-type string
 		Can be 'single' (all agents hitting one host) or 'round_robin'
+
+	-operation string
+		Can be 'set' or 'clear'
 	-content-type string
 		protobuf or pql
 `[1:]
@ -94,8 +98,8 @@ The following arguments are available:
 // ConsumeFlags parses all flags up to the next non flag argument (argument does
 // not start with "-" and isn't the value of a flag). It returns the remaining
 // args.
-func (b *ZipfSetBits) ConsumeFlags(args []string) ([]string, error) {
-	fs := flag.NewFlagSet("ZipfSetBits", flag.ContinueOnError)
+func (b *Zipf) ConsumeFlags(args []string) ([]string, error) {
+	fs := flag.NewFlagSet("Zipf", flag.ContinueOnError)
 	fs.SetOutput(ioutil.Discard)
 	fs.Int64Var(&b.BaseBitmapID, "base-bitmap-id", 0, "")
 	fs.Int64Var(&b.BitmapIDRange, "bitmap-id-range", 100000, "")
@ -109,6 +113,7 @@ func (b *ZipfSetBits) ConsumeFlags(args []string) ([]string, error) {
 	fs.Float64Var(&b.ProfileExponent, "profile-exponent", 1.01, "")
 	fs.Float64Var(&b.ProfileRatio, "profile-ratio", 0.25, "")
 	fs.StringVar(&b.ClientType, "client-type", "single", "")
+	fs.StringVar(&b.Operation, "operation", "set", "")
 	fs.StringVar(&b.ContentType, "content-type", "protobuf", "")

 	if err := fs.Parse(args); err != nil {
@ -131,8 +136,8 @@ func getZipfOffset(N int64, exp, ratio float64) float64 {

 // Init sets up the benchmark based on the agent number and initializes the
 // client.
-func (b *ZipfSetBits) Init(hosts []string, agentNum int) error {
-	b.Name = "zipf-set-bits"
+func (b *Zipf) Init(hosts []string, agentNum int) error {
+	b.Name = "zipf"
 	b.Seed = b.Seed + int64(agentNum)
 	rnd := rand.New(rand.NewSource(b.Seed))
 	bitmapOffset := getZipfOffset(b.BitmapIDRange, b.BitmapExponent, b.BitmapRatio)
@ -143,16 +148,24 @@ func (b *ZipfSetBits) Init(hosts []string, agentNum int) error {
 	b.bitmapPerm = NewPermutationGenerator(b.BitmapIDRange, b.Seed)
 	b.profilePerm = NewPermutationGenerator(b.ProfileIDRange, b.Seed+1)

+	if b.Operation != "set" && b.Operation != "clear" {
+		return fmt.Errorf("Unsupported operation: \"%s\" (must be \"set\" or \"clear\")", b.Operation)
+	}
+
 	return b.HasClient.Init(hosts, agentNum)
 }

-// Run runs the ZipfSetBits benchmark
-func (b *ZipfSetBits) Run(ctx context.Context) map[string]interface{} {
+// Run runs the Zipf benchmark
+func (b *Zipf) Run(ctx context.Context) map[string]interface{} {
 	results := make(map[string]interface{})
 	if b.client == nil {
-		results["error"] = fmt.Errorf("No client set for ZipfSetBits")
+		results["error"] = fmt.Errorf("No client set for Zipf")
 		return results
 	}
+	operation := "SetBit"
+	if b.Operation == "clear" {
+		operation = "ClearBit"
+	}
 	s := NewStats()
 	var start time.Time
 	for n := 0; n < b.Iterations; n++ {
@ -163,7 +176,7 @@ func (b *ZipfSetBits) Run(ctx context.Context) map[string]interface{} {
 		bitmapID := b.bitmapPerm.Next(int64(bitmapIDOriginal))
 		profID := b.profilePerm.Next(int64(profIDOriginal))

-		query := fmt.Sprintf("SetBit(%d, 'frame.n', %d)", b.BaseBitmapID+int64(bitmapID), b.BaseProfileID+int64(profID))
+		query := fmt.Sprintf("%s(%d, 'frame.n', %d)", operation, b.BaseBitmapID+int64(bitmapID), b.BaseProfileID+int64(profID))
 		start = time.Now()
 		_, err := b.client.ExecuteQuery(ctx, b.DB, query, true)
 		if err != nil {
--- a/cmd/pilosactl/main.go
+++ b/cmd/pilosactl/main.go
@ -1098,11 +1098,15 @@ The following flags are allowed:
 func (cmd *CreateCommand) Run(ctx context.Context) error {
 	var clus creator.Cluster
 	if len(cmd.Hosts) == 0 {
+		fmt.Fprintf(cmd.Stderr, "create: no hosts specified - creating cluster in-process\n")
 		clus = &creator.LocalCluster{
 			ReplicaN: cmd.ReplicaN,
 			ServerN:  cmd.ServerN,
 		}
 	} else {
+		if cmd.ServerN != 0 {
+			fmt.Fprintf(cmd.Stderr, "create: hosts were specified, so ignoring serverN\n")
+		}
 		clus = &creator.RemoteCluster{
 			ClusterHosts: cmd.Hosts,
 			ReplicaN:     cmd.ReplicaN,
@ -1125,7 +1129,7 @@ func (cmd *CreateCommand) Run(ctx context.Context) error {
 	signal.Notify(c, os.Interrupt)
 	go func() {
 		for range c {
-			fmt.Fprintf(cmd.Stderr, "\ncaught signal - shutting down\n")
+			fmt.Fprintf(cmd.Stderr, "\ncreate: caught signal - shutting down\n")
 			err := clus.Shutdown()
 			code := 0
 			if err != nil {
@ -1156,7 +1160,7 @@ func (cmd *CreateCommand) Run(ctx context.Context) error {
 		go func(i int, f io.Writer) {
 			_, err := io.Copy(f, logReaders[i])
 			if err != nil {
-				fmt.Fprintf(cmd.Stderr, "Error copying cluster logs: '%v'", err)
+				fmt.Fprintf(cmd.Stderr, "create: error copying cluster logs: '%v'\n", err)
 			}
 		}(i, f)
 	}
@ -1167,6 +1171,7 @@ func (cmd *CreateCommand) Run(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	fmt.Fprintf(cmd.Stderr, "create: cluster started.\n")
 	select {}

 }
@ -1238,8 +1243,8 @@ func (cmd *BagentCommand) ParseFlags(args []string) error {
 			bm = &bench.DiagonalSetBits{}
 		case "random-set-bits":
 			bm = &bench.RandomSetBits{}
-		case "zipf-set-bits":
-			bm = &bench.ZipfSetBits{}
+		case "zipf":
+			bm = &bench.Zipf{}
 		case "multi-db-set-bits":
 			bm = &bench.MultiDBSetBits{}
 		case "random-query":
@ -1287,7 +1292,7 @@ The following flags are allowed:
 	subcommands:
 		diagonal-set-bits
 		random-set-bits
-		zipf-set-bits
+		zipf
 		multi-db-set-bits
 		random-query
 		import
@ -1480,7 +1485,7 @@ func (cmd *BspawnCommand) Run(ctx context.Context) error {
 	output := make(map[string]interface{})
 	output["run-uuid"] = runUUID.String()
 	if len(cmd.PilosaHosts) == 0 {
-		// must create cluster
+		fmt.Fprintln(cmd.Stderr, "bspawn: pilosa-hosts not specified - using create command to build cluster")
 		r, w := io.Pipe()
 		createCmd := NewCreateCommand(cmd.Stdin, w, cmd.Stderr)
 		err := createCmd.ParseFlags(cmd.CreatorArgs)
@ -1490,7 +1495,7 @@ func (cmd *BspawnCommand) Run(ctx context.Context) error {
 		go func() {
 			err := createCmd.Run(ctx)
 			if err != nil {
-				fmt.Fprintf(cmd.Stderr, "Cluster creation error while spawning: %v", err)
+				fmt.Fprintf(cmd.Stderr, "bspawn: cluster creation error while spawning: %v\n", err)
 			}
 		}()
 		clus := &CreateCommand{}
@ -1503,6 +1508,7 @@ func (cmd *BspawnCommand) Run(ctx context.Context) error {
 		output["cluster"] = clus
 	}
 	if len(cmd.AgentHosts) == 0 {
+		fmt.Fprintln(cmd.Stderr, "bpspawn: no agent-hosts specified; all agents will be spawned on localhost")
 		cmd.AgentHosts = []string{"localhost"}
 	}
 	output["spawn"] = cmd
@ -1537,6 +1543,7 @@ func (cmd *BspawnCommand) spawnRemote(ctx context.Context) (map[string]interface
 	}

 	if cmd.CopyBinary {
+		fmt.Fprintf(cmd.Stderr, "bspawn: building pilosactl binary with GOOS=%v and GOARCH=%v to copy to agent hosts\n", cmd.GOOS, cmd.GOARCH)
 		pkg := "github.com/pilosa/pilosa/cmd/pilosactl"
 		bin, err := build.Binary(pkg, cmd.GOOS, cmd.GOARCH)
 		if err != nil {
@ -1553,6 +1560,7 @@ func (cmd *BspawnCommand) spawnRemote(ctx context.Context) (map[string]interface
 	results := make(map[string]interface{})
 	resLock := sync.Mutex{}
 	wg := sync.WaitGroup{}
+	fmt.Fprintln(cmd.Stderr, "bspawn: running benchmarks")
 	for _, sp := range cmd.Benchmarks {
 		results[sp.Name] = make(map[int]interface{})
 		for i := 0; i < sp.Num; i++ {
@ -1574,7 +1582,7 @@ func (cmd *BspawnCommand) spawnRemote(ctx context.Context) (map[string]interface
 				var v interface{}
 				err := dec.Decode(&v)
 				if err != nil {
-					fmt.Fprintf(cmd.Stderr, "error decoding json: %v, spawn: %v", err, name)
+					fmt.Fprintf(cmd.Stderr, "error decoding json: %v, spawn: %v\n", err, name)
 				}
 				resLock.Lock()
 				results[name].(map[int]interface{})[num] = v
--- a/cmd/pilosactl/spawn.json
+++ b/cmd/pilosactl/spawn.json
@ -8,7 +8,7 @@
        {
            "num": 2,
            "name": "rand-plus-zipf",
-            "args": ["random-set-bits", "-iterations", "20000", "zipf-set-bits", "-iterations", "100"]
+            "args": ["random-set-bits", "-iterations", "20000", "zipf", "-iterations", "100"]
        }
    ]
 }
--- a/cmd/pilosactl/zipfspawn.json
+++ b/cmd/pilosactl/zipfspawn.json
@ -2,7 +2,11 @@
    "benchmarks": [
        {
            "num": 1,
-            "args": ["zipf-set-bits", "-iterations", "10000", "-profile-id-range", "100", "-bitmap-id-range", "100", "-seed", "2345", "-client-type", "round_robin", "-bitmap-exponent", "1.001", "-bitmap-ratio", ".9", "-profile-exponent", "1.001", "-profile-ratio", ".3"]
+            "args": ["zipf", "-iterations", "10000", "-profile-id-range", "100", "-bitmap-id-range", "100", "-seed", "2345", "-client-type", "round_robin", "-bitmap-exponent", "1.001", "-bitmap-ratio", ".9", "-profile-exponent", "1.001", "-profile-ratio", ".3"]
+        },
+        {
+            "num": 1,
+            "args": ["zipf", "-iterations", "10000", "-profile-id-range", "100", "-bitmap-id-range", "100", "-seed", "2345", "-client-type", "round_robin", "-bitmap-exponent", "1.001", "-bitmap-ratio", ".9", "-profile-exponent", "1.001", "-profile-ratio", ".3", "-operation", "clear"]
        }
    ]
 }
--- a/creator/remote.go
+++ b/creator/remote.go
@ -49,6 +49,7 @@ func (c *RemoteCluster) Start() error {
 		return fmt.Errorf("connecting to cluster hosts: %v", err)
 	}
 	if c.CopyBinary {
+		fmt.Fprintf(c.Stderr, "create: building pilosa binary with GOOS=%v and GOARCH=%v to copy to hosts", c.GOOS, c.GOARCH)

 		pkg := "github.com/pilosa/pilosa/cmd/pilosa"
 		bin, err := build.Binary(pkg, c.GOOS, c.GOARCH)
@ -96,7 +97,7 @@ func (c *RemoteCluster) Start() error {
 		}
 		err = w.Close()
 		if err != nil {
-			return err
+			return fmt.Errorf("closing config writer: %v", err)
 		}

 		// Start pilosa on remote host
@ -141,7 +142,7 @@ func (c *RemoteCluster) Start() error {
 			defer c.wg.Done()
 			err = sess.Wait()
 			if err != nil {
-				fmt.Fprintf(c.Stderr, "problem with remote pilosa process: %v", err)
+				fmt.Fprintf(c.Stderr, "problem with remote pilosa process: %v\n", err)
 			}
 		}()
 	}
--- a/ssh/ssh.go
+++ b/ssh/ssh.go
@ -203,12 +203,12 @@ func (sf Fleet) OpenFile(name, perm string) (io.WriteCloser, error) {
 func (sf Fleet) WriteFile(name, perm string, data io.Reader) error {
 	wc, err := sf.OpenFile(name, perm)
 	if err != nil {
-		return err
+		return fmt.Errorf("opening: %v", err)
 	}

 	_, err = io.Copy(wc, data)
 	if err != nil {
-		return err
+		return fmt.Errorf("copying: %v", err)
 	}
 	return wc.Close()
 }