From f3a21ad57b5e522402197fc7ea18fa5cc08b6516 Mon Sep 17 00:00:00 2001 From: jaffee Date: Mon, 16 Jan 2017 16:57:38 -0600 Subject: [PATCH 1/6] remove agentNum from Benchmark.Run Benchmarks should modify their parameters in Init based on the agentNum --- bench/bench.go | 26 +++++++++++++++----------- bench/diagonal.go | 9 +++++---- bench/import.go | 16 ++++++++-------- bench/multidb.go | 8 +++++--- bench/random.go | 7 ++++--- bench/randquery.go | 8 ++++---- bench/sliceheight.go | 6 +++--- bench/zipf.go | 7 ++++--- cmd/pilosactl/main.go | 6 +++--- 9 files changed, 51 insertions(+), 42 deletions(-) diff --git a/bench/bench.go b/bench/bench.go index 0ffcc0a40..bbb9e0764 100644 --- a/bench/bench.go +++ b/bench/bench.go @@ -7,27 +7,31 @@ import "context" // methods so that benchmark running code can time only the running of the // benchmark, and not any setup. type Benchmark interface { - // Init takes a list of hosts and is generally expected to set up a - // connection to pilosa using whatever client it chooses. + // Init takes a list of hosts and an agent number. It is generally expected + // to set up a connection to pilosa using whatever client it chooses. These + // agentNum should be used to parameterize the benchmark's configuration if + // it is being run simultaneously on multiple "agents". E.G. the agentNum + // might be used to make a random seed different for each agent, or have + // each agent set a different set of bits. A Benchmark should document how + // the agentNum affects it. Init(hosts []string, agentNum int) error - // Run runs the benchmark. It takes an agentNum which should be used to - // parameterize the benchmark if it is being run simultaneously on multiple - // "agents". E.G. the agentNum might be used to make a random seed different - // for each agent, or have each agent set a different set of bits. The return - // value of Run is kept generic so that any relevant statistics or metrics - // that may be specific to the benchmark in question can be reported. - Run(ctx context.Context, agentNum int) map[string]interface{} + // Run runs the benchmark. The return value of Run is kept generic so that + // any relevant statistics or metrics that may be specific to the benchmark + // in question can be reported. TODO guidelines for what gets included in + // results and what will get added by other stuff. + Run(ctx context.Context) map[string]interface{} } // Command extends Benchmark by adding methods for configuring via command line flags and returning usage information. type Command interface { Benchmark - // ConsumeFlags sets and parses flags, and then returns flagSet.Args() + // ConsumeFlags sets and parses flags, and then returns flagSet.Args(). This + // is so that multiple benchmarks can be specified at the command line. ConsumeFlags(args []string) ([]string, error) - // Usage returns information on how to use this benchmark + // Usage returns information on how to use this benchmark. Usage() string } diff --git a/bench/diagonal.go b/bench/diagonal.go index 9b3d92b6b..c512c3eda 100644 --- a/bench/diagonal.go +++ b/bench/diagonal.go @@ -22,6 +22,8 @@ type DiagonalSetBits struct { func (b *DiagonalSetBits) Init(hosts []string, agentNum int) error { b.Name = "diagonal-set-bits" + b.BaseBitmapID = b.BaseBitmapID + (agentNum * b.Iterations) + b.BaseProfileID = b.BaseProfileID + (agentNum * b.Iterations) return b.HasClient.Init(hosts, agentNum) } @@ -67,17 +69,16 @@ func (b *DiagonalSetBits) ConsumeFlags(args []string) ([]string, error) { } // Run runs the DiagonalSetBits benchmark -func (b *DiagonalSetBits) Run(ctx context.Context, agentNum int) map[string]interface{} { +func (b *DiagonalSetBits) Run(ctx context.Context) map[string]interface{} { results := make(map[string]interface{}) if b.client == nil { - results["error"] = fmt.Errorf("No client set for DiagonalSetBits agent: %v", agentNum) + results["error"] = fmt.Errorf("No client set for DiagonalSetBits") return results } s := NewStats() var start time.Time for n := 0; n < b.Iterations; n++ { - iterID := agentizeNum(n, b.Iterations, agentNum) - query := fmt.Sprintf("SetBit(%d, 'frame.n', %d)", b.BaseBitmapID+iterID, b.BaseProfileID+iterID) + query := fmt.Sprintf("SetBit(%d, 'frame.n', %d)", b.BaseBitmapID+n, b.BaseProfileID+n) start = time.Now() _, err := b.client.ExecuteQuery(ctx, b.DB, query, true) if err != nil { diff --git a/bench/import.go b/bench/import.go index 2854ea458..bfddd4c5c 100644 --- a/bench/import.go +++ b/bench/import.go @@ -110,16 +110,16 @@ func (b *Import) Init(hosts []string, agentNum int) error { b.Name = "import" b.Host = hosts[0] // generate csv data - baseBitmapID, maxBitmapID, baseProfileID, maxProfileID := b.BaseBitmapID, b.MaxBitmapID, b.BaseProfileID, b.MaxProfileID + b.Seed = b.Seed + int64(agentNum) switch b.AgentControls { case "height": numBitmapIDs := (b.MaxBitmapID - b.BaseBitmapID) - baseBitmapID = b.BaseBitmapID + (numBitmapIDs * int64(agentNum)) - maxBitmapID = baseBitmapID + numBitmapIDs + b.BaseBitmapID = b.BaseBitmapID + (numBitmapIDs * int64(agentNum)) + b.MaxBitmapID = b.BaseBitmapID + numBitmapIDs case "width": numProfileIDs := (b.MaxProfileID - b.BaseProfileID) - baseProfileID = b.BaseProfileID + (numProfileIDs * int64(agentNum)) - maxProfileID = baseProfileID + numProfileIDs + b.BaseProfileID = b.BaseProfileID + (numProfileIDs * int64(agentNum)) + b.MaxProfileID = b.BaseProfileID + numProfileIDs case "": break default: @@ -130,8 +130,8 @@ func (b *Import) Init(hosts []string, agentNum int) error { return err } // set b.Paths) - num := GenerateImportCSV(f, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, - b.MinBitsPerMap, b.MaxBitsPerMap, b.Seed+int64(agentNum), b.RandomBitmapOrder) + num := GenerateImportCSV(f, b.BaseBitmapID, b.MaxBitmapID, b.BaseProfileID, b.MaxProfileID, + b.MinBitsPerMap, b.MaxBitsPerMap, b.Seed, b.RandomBitmapOrder) b.numbits = num // set b.Paths b.Paths = []string{f.Name()} @@ -139,7 +139,7 @@ func (b *Import) Init(hosts []string, agentNum int) error { } // Run runs the Import benchmark -func (b *Import) Run(ctx context.Context, agentNum int) map[string]interface{} { +func (b *Import) Run(ctx context.Context) map[string]interface{} { results := make(map[string]interface{}) results["numbits"] = b.numbits results["db"] = b.Database diff --git a/bench/multidb.go b/bench/multidb.go index 0c7a69c0d..6e00a2bfe 100644 --- a/bench/multidb.go +++ b/bench/multidb.go @@ -16,10 +16,12 @@ type MultiDBSetBits struct { BaseBitmapID int `json:"base-bitmap-id"` BaseProfileID int `json:"base-profile-id"` Iterations int `json:"iterations"` + Database string `json:"database"` } func (b *MultiDBSetBits) Init(hosts []string, agentNum int) error { b.Name = "multi-db-set-bits" + b.Database = b.Database + strconv.Itoa(agentNum) return b.HasClient.Init(hosts, agentNum) } @@ -61,10 +63,10 @@ func (b *MultiDBSetBits) ConsumeFlags(args []string) ([]string, error) { } // Run runs the MultiDBSetBits benchmark -func (b *MultiDBSetBits) Run(ctx context.Context, agentNum int) map[string]interface{} { +func (b *MultiDBSetBits) Run(ctx context.Context) map[string]interface{} { results := make(map[string]interface{}) if b.client == nil { - results["error"] = fmt.Errorf("No client set for MultiDBSetBits agent: %v", agentNum) + results["error"] = fmt.Errorf("No client set for MultiDBSetBits") return results } s := NewStats() @@ -72,7 +74,7 @@ func (b *MultiDBSetBits) Run(ctx context.Context, agentNum int) map[string]inter for n := 0; n < b.Iterations; n++ { query := fmt.Sprintf("SetBit(%d, 'frame.n', %d)", b.BaseBitmapID+n, b.BaseProfileID+n) start = time.Now() - _, err := b.client.ExecuteQuery(ctx, "multidb"+strconv.Itoa(agentNum), query, true) + _, err := b.client.ExecuteQuery(ctx, b.Database, query, true) if err != nil { results["error"] = err return results diff --git a/bench/random.go b/bench/random.go index 250748cfa..394a03b55 100644 --- a/bench/random.go +++ b/bench/random.go @@ -26,6 +26,7 @@ type RandomSetBits struct { func (b *RandomSetBits) Init(hosts []string, agentNum int) error { b.Name = "random-set-bits" + b.Seed = b.Seed + int64(agentNum) return b.HasClient.Init(hosts, agentNum) } @@ -82,12 +83,12 @@ func (b *RandomSetBits) ConsumeFlags(args []string) ([]string, error) { } // Run runs the RandomSetBits benchmark -func (b *RandomSetBits) Run(ctx context.Context, agentNum int) map[string]interface{} { - src := rand.NewSource(b.Seed + int64(agentNum)) +func (b *RandomSetBits) Run(ctx context.Context) map[string]interface{} { + src := rand.NewSource(b.Seed) rng := rand.New(src) results := make(map[string]interface{}) if b.client == nil { - results["error"] = fmt.Errorf("No client set for RandomSetBits agent: %v", agentNum) + results["error"] = fmt.Errorf("No client set for RandomSetBits") return results } s := NewStats() diff --git a/bench/randquery.go b/bench/randquery.go index a56f29445..09cf8dddb 100644 --- a/bench/randquery.go +++ b/bench/randquery.go @@ -25,6 +25,7 @@ type RandomQuery struct { func (b *RandomQuery) Init(hosts []string, agentNum int) error { b.Name = "random-query" + b.Seed = b.Seed + int64(agentNum) return b.HasClient.Init(hosts, agentNum) } @@ -87,14 +88,13 @@ func (b *RandomQuery) ConsumeFlags(args []string) ([]string, error) { } // Run runs the RandomQuery benchmark -func (b *RandomQuery) Run(ctx context.Context, agentNum int) map[string]interface{} { - seed := b.Seed + int64(agentNum) +func (b *RandomQuery) Run(ctx context.Context) map[string]interface{} { results := make(map[string]interface{}) if b.client == nil { - results["error"] = fmt.Errorf("No client set for RandomQuery agent: %v", agentNum) + results["error"] = fmt.Errorf("No client set for RandomQuery") return results } - qm := NewQueryGenerator(seed) + qm := NewQueryGenerator(b.Seed) s := NewStats() var start time.Time for n := 0; n < b.Iterations; n++ { diff --git a/bench/sliceheight.go b/bench/sliceheight.go index bb9131859..abed6675b 100644 --- a/bench/sliceheight.go +++ b/bench/sliceheight.go @@ -91,7 +91,7 @@ func (b *SliceHeight) Init(hosts []string, agentNum int) error { } // Run runs the SliceHeight benchmark -func (b *SliceHeight) Run(ctx context.Context, agentNum int) map[string]interface{} { +func (b *SliceHeight) Run(ctx context.Context) map[string]interface{} { results := make(map[string]interface{}) imp := NewImport(b.Stdin, b.Stdout, b.Stderr) @@ -109,11 +109,11 @@ func (b *SliceHeight) Run(ctx context.Context, agentNum int) map[string]interfac results["iteration"+strconv.Itoa(i)] = iresults genstart := time.Now() - imp.Init(b.hosts, agentNum) + imp.Init(b.hosts, 0) gendur := time.Now().Sub(genstart) iresults["csvgen"] = gendur - iresults["import"] = imp.Run(ctx, agentNum) + iresults["import"] = imp.Run(ctx) qstart := time.Now() q := &pql.TopN{Frame: b.Frame, N: 50} diff --git a/bench/zipf.go b/bench/zipf.go index dfb317d61..967c20331 100644 --- a/bench/zipf.go +++ b/bench/zipf.go @@ -122,7 +122,8 @@ func getZipfOffset(N int64, exp, ratio float64) float64 { func (b *ZipfSetBits) Init(hosts []string, agentNum int) error { b.Name = "zipf-set-bits" - rnd := rand.New(rand.NewSource(b.Seed + int64(agentNum))) + b.Seed = b.Seed + int64(agentNum) + rnd := rand.New(rand.NewSource(b.Seed)) bitmapOffset := getZipfOffset(b.BitmapIDRange, b.BitmapExponent, b.BitmapRatio) b.bitmapRng = rand.NewZipf(rnd, b.BitmapExponent, bitmapOffset, uint64(b.BitmapIDRange-1)) profileOffset := getZipfOffset(b.ProfileIDRange, b.ProfileExponent, b.ProfileRatio) @@ -135,10 +136,10 @@ func (b *ZipfSetBits) Init(hosts []string, agentNum int) error { } // Run runs the ZipfSetBits benchmark -func (b *ZipfSetBits) Run(ctx context.Context, agentNum int) map[string]interface{} { +func (b *ZipfSetBits) Run(ctx context.Context) map[string]interface{} { results := make(map[string]interface{}) if b.client == nil { - results["error"] = fmt.Errorf("No client set for ZipfSetBits agent: %v", agentNum) + results["error"] = fmt.Errorf("No client set for ZipfSetBits") return results } s := NewStats() diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 965728956..ec8651ab5 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -1303,7 +1303,7 @@ func (cmd *BagentCommand) Run(ctx context.Context) error { return fmt.Errorf("in cmd.Run initialization: %v", err) } - res := sbm.Run(ctx, cmd.AgentNum) + res := sbm.Run(ctx) res["agent-num"] = cmd.AgentNum enc := json.NewEncoder(cmd.Stdout) if cmd.HumanReadable { @@ -1622,14 +1622,14 @@ func (sb *serialBenchmark) Init(hosts []string, agentNum int) error { // Run runs the serial benchmark and returns it's results in a nested map - the // top level keys are the indices of each benchmark in the list of benchmarks, // and the values are the results of each benchmark's Run method. -func (sb *serialBenchmark) Run(ctx context.Context, agentNum int) map[string]interface{} { +func (sb *serialBenchmark) Run(ctx context.Context) map[string]interface{} { benchmarks := make([]map[string]interface{}, len(sb.benchmarkers)) results := map[string]interface{}{"benchmarks": benchmarks} total_start := time.Now() for i, b := range sb.benchmarkers { start := time.Now() - output := b.Run(ctx, agentNum) + output := b.Run(ctx) if _, ok := output["runtime"]; ok { panic(fmt.Sprintf("Benchmark %v added 'runtime' to its results", b)) } From d2d62dbdf2f4d27b4180a8cf11135935f8df0071 Mon Sep 17 00:00:00 2001 From: jaffee Date: Wed, 18 Jan 2017 12:50:40 -0600 Subject: [PATCH 2/6] update benchmark docs --- README.md | 88 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 3617c8678..129e2939a 100644 --- a/README.md +++ b/README.md @@ -19,12 +19,14 @@ Now you can install the `pilosa` binary: $ go install github.com/pilosa/pilosa/cmd/... ``` -Now run `pilosa` with the default configuration: +Now run a single pilosa node with the default configuration: ```sh pilosa ``` +If you would like to quickly create a multi-node pilosa cluster, see the `pilosactl create` documentation. + ## Configuration You can specify a configuration by setting the `-config` flag when running `pilosa`. @@ -221,49 +223,75 @@ $ go install --ldflags="-X main.Version=1.0.0" [Glide]: http://glide.sh/ -## Benchmarks +## Pilosactl -The usual interface for running benchmarks is: +### Create + +`pilosactl create` is used to create pilosa clusters. It has a number of options for controlling how the cluster is configured, what hosts it is on, and even the ability to build the pilosa binary locally and copy it to each cluster node automatically. To start pilosa on remote hosts, you only need `ssh` access to those hosts. See `pilosactl -h` for a full list of options. + +Examples: + +Create a 5 node cluster locally (using 5 different ports), with a replication factor of 2. ``` -pilosactl bspawn benchmark-file.json +pilosactl create + -serverN 5 + -replicaN 2 ``` -There are several example json config files in `cmd/pilosactl` -The `bspawn` command calls other `pilosactl` subcommands such as `create` and `bagent` to perform the benchmarks. These commands can also be used directly if one wishes e.g. to just create a cluster, or locally run a benchmarks against an existing cluster. Pass the `-help` flag to either to get more information about its usage. +Create a cluster on 3 remote hosts - all logs will come to local stderr, pilosa binary must be available on remote hosts. The ssh user on the remote hosts needs to be the same as your local user. Otherwise use the `ssh-user` option. +``` +pilosactl create + -hosts="node1.example.com:15000,node2.example.com:15000,node3.example.com:15000" +``` -### Configuration Format +Create a cluster on 3 remote hosts running OSX, but build the binary locally and copy it up. Stream the stderr of each node to a separate local log file. +``` +pilosactl create + -hosts="mac1.example.com:15000,mac2.example.com:15000,mac3.example.com:15000" + -copy-binary + -goos=darwin + -goarch=amd64 + -log-file-prefix=clusterlogs +``` -bspawn uses a json config format that has 5 top level items - an example is below. +### Bagent +`pilosactl bagent` is what you want if you just want to run a simple benchmark against an existing cluster. Running it with no arguments will print some help, including the set of subcommands that it may be passed. Calling a subcommand with `-h'` will print the options for that subcommand. The `agent-num` flag can be passed an integer which can change the behavior the benchmarks that are run. This is useful when multiple invocations of the same benchmark are made by the `bspawn` command - they can each (for example) set different bits even though they all have the same arguments. + +E.G. +``` +pilosactl bagent import -h +``` + +Multiple subcommands and their arguments may be concatenated at the command line and they will be run serially. This is useful (i.e.) for importing a bunch of data, and then executing queries against it. + +This will generate and import a bunch of data, and then execute random queries against it. + +``` +pilosactl bagent import -max-bits-per-map=10000 random-query -iterations 100 +``` + +### Bspawn +`pilosactl bspawn` allows you to automate the creation of clusters and the running of complex benchmarks which span multiple benchmark agents against them. It has a number of options which are described by `pilosactl bspawn` with no arguments, and also takes a config file which describes the Benchmark itself - this file is described below. + +#### Configuration Format + +The configuration file is a json object with the top level key `benchmarks`. This contains a list of objects each of which represents a `bagent` command (the `args` key) that will be run some number of times concurrently (the `num` key), and a `name` which should describe the overall effect that command. An example is below. ```json { - "CreatorArgs": ["-type", "local", "-serverN", "1", "-replicaN", "1"], - "PilosaHosts": ["localhost:19327"], - "AgentHosts": ["agent.example.com"], - "Benchmarks": [ + "benchmarks": [ { - "Num": 1, - "Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width"] + "num": 3, + "name": "set-diags", + "args": ["diagonal-set-bits", "-iterations", "30000", "-client-type", "round_robin"] }, { - "Num": 1, - "Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "-random-bitmap-order", "-db", "randoload"] + "num": 2, + "name": "rand-plus-zipf", + "args": ["random-set-bits", "-iterations", "20000", "zipf-set-bits", "-iterations", "100"] } ] } - ``` -#### CreatorArgs -Specifies the pilosa cluster that should be created to run benchmarks against. For more information about the configuration for this option, see the `pilosactl create -help` - -#### PilosaHosts -If PilosaHosts is set, CreatorArgs will be ignored, and an existing pilosa cluster specified by the list of hosts will be used. - -#### AgentHosts -If AgentHosts is not empty, the agents specified here are used; if it is empty, agents will be run locally. - -#### Benchmarks -Benchmarks is where the actual benchmarks to run are specified - each contains a `Num` which is the number of agents that should run that benchmark, and Args which specifies the benchmark. The benchmarks in the `Benchmarks` list will be run concurrently. For more information about Args, see the `pilosactl bagent -help`. - -For documentation on a specific `bagent` subcommand do `pilosactl bagent -help` +All of the benchmarks, and agents are run concurrently. Each agent will be passed an `agent-num` which can modify the behavior in a way that is benchmark specific. See the documentation for each benchmark to see how `agent-num` changes its behavior. From ea2860265c5498fd550a585562cee218ded0f05a Mon Sep 17 00:00:00 2001 From: jaffee Date: Wed, 18 Jan 2017 19:35:49 -0600 Subject: [PATCH 3/6] docs updates for benchmarking --- bench/bench.go | 20 ++++++++------------ bench/diagonal.go | 10 ++++++++++ bench/doc.go | 37 +++++++++++++++++++++++++++++++++++++ bench/import.go | 10 ++++++++++ bench/multidb.go | 7 +++++++ bench/prettify.go | 2 ++ bench/query.go | 6 ++++++ bench/random.go | 7 +++++++ bench/randquery.go | 7 +++++++ bench/sliceheight.go | 9 +++++++++ bench/stats.go | 6 ++++++ bench/zipf.go | 8 ++++++++ 12 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 bench/doc.go diff --git a/bench/bench.go b/bench/bench.go index bbb9e0764..73cc625a8 100644 --- a/bench/bench.go +++ b/bench/bench.go @@ -8,18 +8,20 @@ import "context" // benchmark, and not any setup. type Benchmark interface { // Init takes a list of hosts and an agent number. It is generally expected - // to set up a connection to pilosa using whatever client it chooses. These + // to set up a connection to pilosa using whatever client it chooses. The // agentNum should be used to parameterize the benchmark's configuration if // it is being run simultaneously on multiple "agents". E.G. the agentNum // might be used to make a random seed different for each agent, or have - // each agent set a different set of bits. A Benchmark should document how - // the agentNum affects it. + // each agent set a different set of bits. Init's doc string should document + // how the agentNum affects it. Init(hosts []string, agentNum int) error // Run runs the benchmark. The return value of Run is kept generic so that // any relevant statistics or metrics that may be specific to the benchmark // in question can be reported. TODO guidelines for what gets included in - // results and what will get added by other stuff. + // results and what will get added by other stuff. Run does not need to + // report total run time in `results`, as that will be added by calling + // code. Run(ctx context.Context) map[string]interface{} } @@ -31,13 +33,7 @@ type Command interface { // is so that multiple benchmarks can be specified at the command line. ConsumeFlags(args []string) ([]string, error) - // Usage returns information on how to use this benchmark. + // Usage returns information on how to use this benchmark. The usage string + // should explain how the agent num affects the benchmark's operation. Usage() string } - -// agentizeNum is a helper which combines the loop iteration (n) with the total -// number of iterations and the agentNum in order to produce a globally unique -// number across all loop iterations on all agents. -func agentizeNum(n, iterations, agentNum int) int { - return n + (agentNum * iterations) -} diff --git a/bench/diagonal.go b/bench/diagonal.go index c512c3eda..4cb0e354c 100644 --- a/bench/diagonal.go +++ b/bench/diagonal.go @@ -20,6 +20,8 @@ type DiagonalSetBits struct { DB string `json:"db"` } +// Init sets up the pilosa client and modifies the configured values based on +// the agent num. func (b *DiagonalSetBits) Init(hosts []string, agentNum int) error { b.Name = "diagonal-set-bits" b.BaseBitmapID = b.BaseBitmapID + (agentNum * b.Iterations) @@ -27,10 +29,15 @@ func (b *DiagonalSetBits) Init(hosts []string, agentNum int) error { return b.HasClient.Init(hosts, agentNum) } +// Usage returns the usage message to be printed. func (b *DiagonalSetBits) Usage() string { return ` diagonal-set-bits sets bits with increasing profile id and bitmap id. +Agent num offsets both the base profile id and base bitmap id by the number of +iterations, so that only bits on the main diagonal are set, and agents don't +overlap at all. + Usage: diagonal-set-bits [arguments] The following arguments are available: @@ -53,6 +60,9 @@ The following arguments are available: `[1:] } +// ConsumeFlags parses all flags up to the next non flag argument (argument does +// not start with "-" and isn't the value of a flag). It returns the remaining +// args. func (b *DiagonalSetBits) ConsumeFlags(args []string) ([]string, error) { fs := flag.NewFlagSet("DiagonalSetBits", flag.ContinueOnError) fs.SetOutput(ioutil.Discard) diff --git a/bench/doc.go b/bench/doc.go new file mode 100644 index 000000000..1c61a2b20 --- /dev/null +++ b/bench/doc.go @@ -0,0 +1,37 @@ +// bench contains benchmarks and common utilities useful to benchmarks +// +// In order to write new benchmarks, one must satisfy the Benchmark and Command +// interfaces in bench.go. In order to use the benchmark from pilosactl, it +// needs to be wired in in two places. The first is BagentCommand.ParseFlags, +// where a case statement needs to be added, and the second is just adding the +// benchmark to the BagentCommand.Usage usage string. +// +// When writing a new benchmark, there are a few things to keep in mind other +// than just implementing the interface: +// +// The benchmark should modify it's own configuration in its Init method based +// on the agentNum it is given. How it modifies is specific to the benchmark, +// but the idea is that it should make sense to call the benchmark with the same +// configuration, but multiple different agent numbers, and it should do useful +// work each time (i.e. not just setting the same bits, or running the same +// queries). +// +// The Init method should do everything that needs to be done to get the +// benchmark to a runnable state - all code in run should be the stuff that we +// actually want to time. +// +// The Run method does not need to report the total runtime - that is collected +// by calling code. +// +// Usage should follow the format in other benchmarks, and explain how the +// benchmark uses agentNum to modify its behavior +// +// +// Files: +// +// 1. client.go contains pilosa client code which is shared by many benchmarks +// 2. errgroup.go contains the ErrGroup implementation copied from golang.org/x/ +// so as not to pull in a bunch of useless deps. +// 3. stats.go contains useful code for gathering stats about a series of timed +// operations. +package bench diff --git a/bench/import.go b/bench/import.go index bfddd4c5c..2d88b23a3 100644 --- a/bench/import.go +++ b/bench/import.go @@ -13,6 +13,7 @@ import ( "github.com/pilosa/pilosa/pilosactl" ) +// NewImport returns an Import Benchmark which pilosactl importer configured. func NewImport(stdin io.Reader, stdout, stderr io.Writer) *Import { return &Import{ ImportCommand: pilosactl.NewImportCommand(stdin, stdout, stderr), @@ -36,10 +37,13 @@ type Import struct { *pilosactl.ImportCommand } +// Usage returns the usage message to be printed. func (b *Import) Usage() string { return ` import generates an import file and imports using pilosa's bulk import interface +Agent num can have various effects - see -agent-controls flag. + Usage: import [arguments] The following arguments are available: @@ -81,6 +85,9 @@ The following arguments are available: `[1:] } +// ConsumeFlags parses all flags up to the next non flag argument (argument does +// not start with "-" and isn't the value of a flag). It returns the remaining +// args. func (b *Import) ConsumeFlags(args []string) ([]string, error) { fs := flag.NewFlagSet("Import", flag.ContinueOnError) fs.SetOutput(ioutil.Discard) @@ -103,6 +110,7 @@ func (b *Import) ConsumeFlags(args []string) ([]string, error) { return fs.Args(), nil } +// Init generates import data based on the agent num and fields of 'b'. func (b *Import) Init(hosts []string, agentNum int) error { if len(hosts) == 0 { return fmt.Errorf("Need at least one host") @@ -151,12 +159,14 @@ func (b *Import) Run(ctx context.Context) map[string]interface{} { return results } +// Int64Slice is a sortable slice of 64 bit signed ints type Int64Slice []int64 func (s Int64Slice) Len() int { return len(s) } func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] } func (s Int64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +// GenerateImportCSV writes a generated csv to 'w' which is in the form pilosactl expects for imports. func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, minBitsPerMap, maxBitsPerMap, seed int64, randomOrder bool) int { src := rand.NewSource(seed) rng := rand.New(src) diff --git a/bench/multidb.go b/bench/multidb.go index 6e00a2bfe..8ea8cd135 100644 --- a/bench/multidb.go +++ b/bench/multidb.go @@ -19,16 +19,20 @@ type MultiDBSetBits struct { Database string `json:"database"` } +// Init sets up the db name based on the agentNum and sets up the pilosa client. func (b *MultiDBSetBits) Init(hosts []string, agentNum int) error { b.Name = "multi-db-set-bits" b.Database = b.Database + strconv.Itoa(agentNum) return b.HasClient.Init(hosts, agentNum) } +// Usage returns the usage message to be printed. func (b *MultiDBSetBits) Usage() string { return ` multi-db-set-bits sets bits with increasing profile id and bitmap id using a different DB for each agent. +Agent num changes the database being written to. + Usage: multi-db-set-bits [arguments] The following arguments are available: @@ -48,6 +52,9 @@ The following arguments are available: `[1:] } +// ConsumeFlags parses all flags up to the next non flag argument (argument does +// not start with "-" and isn't the value of a flag). It returns the remaining +// args. func (b *MultiDBSetBits) ConsumeFlags(args []string) ([]string, error) { fs := flag.NewFlagSet("MultiDBSetBits", flag.ContinueOnError) fs.SetOutput(ioutil.Discard) diff --git a/bench/prettify.go b/bench/prettify.go index c31f4118c..1902d412b 100644 --- a/bench/prettify.go +++ b/bench/prettify.go @@ -5,6 +5,8 @@ import "time" // wrapper type to force human-readable JSON output type PrettyDuration time.Duration +// MarshalJSON returns a nicely formatted duration, instead of it just being +// treated like an int. func (d PrettyDuration) MarshalJSON() ([]byte, error) { s := time.Duration(d).String() return []byte("\"" + s + "\""), nil diff --git a/bench/query.go b/bench/query.go index fdb65afac..aae645e18 100644 --- a/bench/query.go +++ b/bench/query.go @@ -6,6 +6,7 @@ import ( "github.com/pilosa/pilosa/pql" ) +// NewQueryGenerator initializes a new QueryGenerator func NewQueryGenerator(seed int64) *QueryGenerator { return &QueryGenerator{ IDToFrameFn: func(id uint64) string { return "frame.n" }, @@ -14,12 +15,15 @@ func NewQueryGenerator(seed int64) *QueryGenerator { } } +// QueryGenerator holds the configuration and state for randomly generating +// queries. type QueryGenerator struct { IDToFrameFn func(id uint64) string R *rand.Rand Frames []string } +// Random returns a randomly generated query. func (q *QueryGenerator) Random(maxN, depth, maxargs int, idmin, idmax uint64) pql.Call { // TODO: handle depth==1 or 0 val := q.R.Intn(5) @@ -31,6 +35,7 @@ func (q *QueryGenerator) Random(maxN, depth, maxargs int, idmin, idmax uint64) p } } +// RandomTopN returns a randomly generated TopN query. func (q *QueryGenerator) RandomTopN(maxN, depth, maxargs int, idmin, idmax uint64) *pql.TopN { frameIdx := q.R.Intn(len(q.Frames)) return &pql.TopN{ @@ -40,6 +45,7 @@ func (q *QueryGenerator) RandomTopN(maxN, depth, maxargs int, idmin, idmax uint6 } } +// RandomBitmapCall returns a randomly generate query which is a pql.BitmapCall. func (q *QueryGenerator) RandomBitmapCall(depth, maxargs int, idmin, idmax uint64) pql.BitmapCall { if depth <= 1 { bitmapID := q.R.Int63n(int64(idmax)-int64(idmin)) + int64(idmin) diff --git a/bench/random.go b/bench/random.go index 394a03b55..e3e89400c 100644 --- a/bench/random.go +++ b/bench/random.go @@ -24,16 +24,20 @@ type RandomSetBits struct { DB string `json:"db"` } +// Init adds the agent num to the random seed and initializes the client. func (b *RandomSetBits) Init(hosts []string, agentNum int) error { b.Name = "random-set-bits" b.Seed = b.Seed + int64(agentNum) return b.HasClient.Init(hosts, agentNum) } +// Usage returns the usage message to be printed. func (b *RandomSetBits) Usage() string { return ` random-set-bits sets random bits +Agent number modifies the random seed. + Usage: random-set-bits [arguments] The following arguments are available: @@ -64,6 +68,9 @@ The following arguments are available: `[1:] } +// ConsumeFlags parses all flags up to the next non flag argument (argument does +// not start with "-" and isn't the value of a flag). It returns the remaining +// args. func (b *RandomSetBits) ConsumeFlags(args []string) ([]string, error) { fs := flag.NewFlagSet("RandomSetBits", flag.ContinueOnError) fs.SetOutput(ioutil.Discard) diff --git a/bench/randquery.go b/bench/randquery.go index 09cf8dddb..11fa7016b 100644 --- a/bench/randquery.go +++ b/bench/randquery.go @@ -23,16 +23,20 @@ type RandomQuery struct { DBs []string `json:"dbs"` } +// Init adds the agent num to the random seed and initializes the client. func (b *RandomQuery) Init(hosts []string, agentNum int) error { b.Name = "random-query" b.Seed = b.Seed + int64(agentNum) return b.HasClient.Init(hosts, agentNum) } +// Usage returns the usage message to be printed. func (b *RandomQuery) Usage() string { return ` random-query constructs random queries +Agent number modifies the random seed. + Usage: random-query [arguments] The following arguments are available: @@ -66,6 +70,9 @@ The following arguments are available: `[1:] } +// ConsumeFlags parses all flags up to the next non flag argument (argument does +// not start with "-" and isn't the value of a flag). It returns the remaining +// args. func (b *RandomQuery) ConsumeFlags(args []string) ([]string, error) { fs := flag.NewFlagSet("RandomQuery", flag.ContinueOnError) fs.SetOutput(ioutil.Discard) diff --git a/bench/sliceheight.go b/bench/sliceheight.go index abed6675b..dbfdd144f 100644 --- a/bench/sliceheight.go +++ b/bench/sliceheight.go @@ -12,6 +12,8 @@ import ( "github.com/pilosa/pilosa/pql" ) +// NewSliceHeight creates a new slice height benchmark with stdin/out/err +// initialized. func NewSliceHeight(stdin io.Reader, stdout, stderr io.Writer) *SliceHeight { return &SliceHeight{ Stdin: stdin, @@ -38,10 +40,13 @@ type SliceHeight struct { Stderr io.Writer `json:"-"` } +// Usage returns the usage message to be printed. func (b *SliceHeight) Usage() string { return ` slice-height repeatedly imports more bitmaps into a single slice and tests query times in between. +Agent number has no effect on this benchmark. + Usage: slice-height [arguments] The following arguments are available: @@ -66,6 +71,9 @@ The following arguments are available: `[1:] } +// ConsumeFlags parses all flags up to the next non flag argument (argument does +// not start with "-" and isn't the value of a flag). It returns the remaining +// args. func (b *SliceHeight) ConsumeFlags(args []string) ([]string, error) { fs := flag.NewFlagSet("SliceHeight", flag.ContinueOnError) fs.SetOutput(ioutil.Discard) @@ -84,6 +92,7 @@ func (b *SliceHeight) ConsumeFlags(args []string) ([]string, error) { return fs.Args(), nil } +// Init sets up the slice height benchmark. func (b *SliceHeight) Init(hosts []string, agentNum int) error { b.Name = "slice-height" b.hosts = hosts diff --git a/bench/stats.go b/bench/stats.go index de0bf34a3..7a32987f9 100644 --- a/bench/stats.go +++ b/bench/stats.go @@ -5,6 +5,7 @@ import ( "time" ) +// Stats object helps track timing stats. type Stats struct { Min time.Duration Max time.Duration @@ -16,6 +17,7 @@ type Stats struct { SaveAll bool } +// NewStats gets a Stats object. func NewStats() *Stats { return &Stats{ Min: 1<<63 - 1, @@ -23,6 +25,7 @@ func NewStats() *Stats { } } +// Add adds a new time to the stats object. func (s *Stats) Add(td time.Duration) { if s.SaveAll { s.All = append(s.All, td) @@ -43,10 +46,13 @@ func (s *Stats) Add(td time.Duration) { s.sumSquareDelta += float64(delta * (td - s.Mean)) } +// Avg returns the average of all durations Added to the Stats object. func (s *Stats) Avg() time.Duration { return s.Total / time.Duration(s.Num) } +// AddToResults serializes the summary of Stats and adds them to the results +// map. func AddToResults(s *Stats, results map[string]interface{}) { results["min"] = s.Min results["max"] = s.Max diff --git a/bench/zipf.go b/bench/zipf.go index 967c20331..6343cd4c6 100644 --- a/bench/zipf.go +++ b/bench/zipf.go @@ -35,6 +35,7 @@ type ZipfSetBits struct { profilePerm *PermutationGenerator } +// Usage returns the usage message to be printed. func (b *ZipfSetBits) Usage() string { return ` zipf-set-bits sets random bits according to the Zipf distribution. @@ -44,6 +45,8 @@ the "sharpness" of the distribution, with higher exponent being sharper. Ratio, in the range (0, 1), with a default value of 0.25, controls the maximum variation of the distribution, with higher ratio being more uniform. +Agent number modifies random seed. + Usage: zipf-set-bits [arguments] The following arguments are available: @@ -86,6 +89,9 @@ The following arguments are available: `[1:] } +// ConsumeFlags parses all flags up to the next non flag argument (argument does +// not start with "-" and isn't the value of a flag). It returns the remaining +// args. func (b *ZipfSetBits) ConsumeFlags(args []string) ([]string, error) { fs := flag.NewFlagSet("ZipfSetBits", flag.ContinueOnError) fs.SetOutput(ioutil.Discard) @@ -120,6 +126,8 @@ func getZipfOffset(N int64, exp, ratio float64) float64 { return z * float64(N-1) / (1 - z) } +// Init sets up the benchmark based on the agent number and initializes the +// client. func (b *ZipfSetBits) Init(hosts []string, agentNum int) error { b.Name = "zipf-set-bits" b.Seed = b.Seed + int64(agentNum) From 2c9ce1f4b8e8e5416e2daa006272eac4f350bb03 Mon Sep 17 00:00:00 2001 From: jaffee Date: Wed, 18 Jan 2017 20:56:46 -0600 Subject: [PATCH 4/6] tweaks to bench doc.go --- bench/doc.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bench/doc.go b/bench/doc.go index 1c61a2b20..2e451d69c 100644 --- a/bench/doc.go +++ b/bench/doc.go @@ -9,29 +9,31 @@ // When writing a new benchmark, there are a few things to keep in mind other // than just implementing the interface: // -// The benchmark should modify it's own configuration in its Init method based +// 1. The benchmark should modify it's own configuration in its Init method based // on the agentNum it is given. How it modifies is specific to the benchmark, // but the idea is that it should make sense to call the benchmark with the same // configuration, but multiple different agent numbers, and it should do useful // work each time (i.e. not just setting the same bits, or running the same // queries). // -// The Init method should do everything that needs to be done to get the +// 2. The Init method should do everything that needs to be done to get the // benchmark to a runnable state - all code in run should be the stuff that we // actually want to time. // -// The Run method does not need to report the total runtime - that is collected +// 3. The Run method does not need to report the total runtime - that is collected // by calling code. // -// Usage should follow the format in other benchmarks, and explain how the +// 4. Usage should follow the format in other benchmarks, and explain how the // benchmark uses agentNum to modify its behavior // // // Files: // // 1. client.go contains pilosa client code which is shared by many benchmarks +// // 2. errgroup.go contains the ErrGroup implementation copied from golang.org/x/ // so as not to pull in a bunch of useless deps. +// // 3. stats.go contains useful code for gathering stats about a series of timed // operations. package bench From 1b13191fdc3ff51894daf5a5ae7bd70c63041da3 Mon Sep 17 00:00:00 2001 From: jaffee Date: Wed, 18 Jan 2017 21:18:24 -0600 Subject: [PATCH 5/6] readme tweaks --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 129e2939a..9fd0d4e60 100644 --- a/README.md +++ b/README.md @@ -225,9 +225,11 @@ $ go install --ldflags="-X main.Version=1.0.0" ## Pilosactl +Pilosactl contains a suite of tools for interacting with pilosa. Run `pilosactl` for an overview of commands, and `pilosactl -h` for specific information on that command. + ### Create -`pilosactl create` is used to create pilosa clusters. It has a number of options for controlling how the cluster is configured, what hosts it is on, and even the ability to build the pilosa binary locally and copy it to each cluster node automatically. To start pilosa on remote hosts, you only need `ssh` access to those hosts. See `pilosactl -h` for a full list of options. +`pilosactl create` is used to create pilosa clusters. It has a number of options for controlling how the cluster is configured, what hosts it is on, and even the ability to build the pilosa binary locally and copy it to each cluster node automatically. To start pilosa on remote hosts, you only need `ssh` access to those hosts. See `pilosactl create -h` for a full list of options. Examples: From 2b9284ccef04d366313f397d9d79391b42ad88ef Mon Sep 17 00:00:00 2001 From: jaffee Date: Wed, 18 Jan 2017 23:42:44 -0600 Subject: [PATCH 6/6] it's -> its --- bench/doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/doc.go b/bench/doc.go index 2e451d69c..6554ffe80 100644 --- a/bench/doc.go +++ b/bench/doc.go @@ -9,7 +9,7 @@ // When writing a new benchmark, there are a few things to keep in mind other // than just implementing the interface: // -// 1. The benchmark should modify it's own configuration in its Init method based +// 1. The benchmark should modify its own configuration in its Init method based // on the agentNum it is given. How it modifies is specific to the benchmark, // but the idea is that it should make sense to call the benchmark with the same // configuration, but multiple different agent numbers, and it should do useful