From f3a21ad57b5e522402197fc7ea18fa5cc08b6516 Mon Sep 17 00:00:00 2001 From: jaffee Date: Mon, 16 Jan 2017 16:57:38 -0600 Subject: [PATCH] remove agentNum from Benchmark.Run Benchmarks should modify their parameters in Init based on the agentNum --- bench/bench.go | 26 +++++++++++++++----------- bench/diagonal.go | 9 +++++---- bench/import.go | 16 ++++++++-------- bench/multidb.go | 8 +++++--- bench/random.go | 7 ++++--- bench/randquery.go | 8 ++++---- bench/sliceheight.go | 6 +++--- bench/zipf.go | 7 ++++--- cmd/pilosactl/main.go | 6 +++--- 9 files changed, 51 insertions(+), 42 deletions(-) diff --git a/bench/bench.go b/bench/bench.go index 0ffcc0a40..bbb9e0764 100644 --- a/bench/bench.go +++ b/bench/bench.go @@ -7,27 +7,31 @@ import "context" // methods so that benchmark running code can time only the running of the // benchmark, and not any setup. type Benchmark interface { - // Init takes a list of hosts and is generally expected to set up a - // connection to pilosa using whatever client it chooses. + // Init takes a list of hosts and an agent number. It is generally expected + // to set up a connection to pilosa using whatever client it chooses. These + // agentNum should be used to parameterize the benchmark's configuration if + // it is being run simultaneously on multiple "agents". E.G. the agentNum + // might be used to make a random seed different for each agent, or have + // each agent set a different set of bits. A Benchmark should document how + // the agentNum affects it. Init(hosts []string, agentNum int) error - // Run runs the benchmark. It takes an agentNum which should be used to - // parameterize the benchmark if it is being run simultaneously on multiple - // "agents". E.G. the agentNum might be used to make a random seed different - // for each agent, or have each agent set a different set of bits. The return - // value of Run is kept generic so that any relevant statistics or metrics - // that may be specific to the benchmark in question can be reported. - Run(ctx context.Context, agentNum int) map[string]interface{} + // Run runs the benchmark. The return value of Run is kept generic so that + // any relevant statistics or metrics that may be specific to the benchmark + // in question can be reported. TODO guidelines for what gets included in + // results and what will get added by other stuff. + Run(ctx context.Context) map[string]interface{} } // Command extends Benchmark by adding methods for configuring via command line flags and returning usage information. type Command interface { Benchmark - // ConsumeFlags sets and parses flags, and then returns flagSet.Args() + // ConsumeFlags sets and parses flags, and then returns flagSet.Args(). This + // is so that multiple benchmarks can be specified at the command line. ConsumeFlags(args []string) ([]string, error) - // Usage returns information on how to use this benchmark + // Usage returns information on how to use this benchmark. Usage() string } diff --git a/bench/diagonal.go b/bench/diagonal.go index 9b3d92b6b..c512c3eda 100644 --- a/bench/diagonal.go +++ b/bench/diagonal.go @@ -22,6 +22,8 @@ type DiagonalSetBits struct { func (b *DiagonalSetBits) Init(hosts []string, agentNum int) error { b.Name = "diagonal-set-bits" + b.BaseBitmapID = b.BaseBitmapID + (agentNum * b.Iterations) + b.BaseProfileID = b.BaseProfileID + (agentNum * b.Iterations) return b.HasClient.Init(hosts, agentNum) } @@ -67,17 +69,16 @@ func (b *DiagonalSetBits) ConsumeFlags(args []string) ([]string, error) { } // Run runs the DiagonalSetBits benchmark -func (b *DiagonalSetBits) Run(ctx context.Context, agentNum int) map[string]interface{} { +func (b *DiagonalSetBits) Run(ctx context.Context) map[string]interface{} { results := make(map[string]interface{}) if b.client == nil { - results["error"] = fmt.Errorf("No client set for DiagonalSetBits agent: %v", agentNum) + results["error"] = fmt.Errorf("No client set for DiagonalSetBits") return results } s := NewStats() var start time.Time for n := 0; n < b.Iterations; n++ { - iterID := agentizeNum(n, b.Iterations, agentNum) - query := fmt.Sprintf("SetBit(%d, 'frame.n', %d)", b.BaseBitmapID+iterID, b.BaseProfileID+iterID) + query := fmt.Sprintf("SetBit(%d, 'frame.n', %d)", b.BaseBitmapID+n, b.BaseProfileID+n) start = time.Now() _, err := b.client.ExecuteQuery(ctx, b.DB, query, true) if err != nil { diff --git a/bench/import.go b/bench/import.go index 2854ea458..bfddd4c5c 100644 --- a/bench/import.go +++ b/bench/import.go @@ -110,16 +110,16 @@ func (b *Import) Init(hosts []string, agentNum int) error { b.Name = "import" b.Host = hosts[0] // generate csv data - baseBitmapID, maxBitmapID, baseProfileID, maxProfileID := b.BaseBitmapID, b.MaxBitmapID, b.BaseProfileID, b.MaxProfileID + b.Seed = b.Seed + int64(agentNum) switch b.AgentControls { case "height": numBitmapIDs := (b.MaxBitmapID - b.BaseBitmapID) - baseBitmapID = b.BaseBitmapID + (numBitmapIDs * int64(agentNum)) - maxBitmapID = baseBitmapID + numBitmapIDs + b.BaseBitmapID = b.BaseBitmapID + (numBitmapIDs * int64(agentNum)) + b.MaxBitmapID = b.BaseBitmapID + numBitmapIDs case "width": numProfileIDs := (b.MaxProfileID - b.BaseProfileID) - baseProfileID = b.BaseProfileID + (numProfileIDs * int64(agentNum)) - maxProfileID = baseProfileID + numProfileIDs + b.BaseProfileID = b.BaseProfileID + (numProfileIDs * int64(agentNum)) + b.MaxProfileID = b.BaseProfileID + numProfileIDs case "": break default: @@ -130,8 +130,8 @@ func (b *Import) Init(hosts []string, agentNum int) error { return err } // set b.Paths) - num := GenerateImportCSV(f, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, - b.MinBitsPerMap, b.MaxBitsPerMap, b.Seed+int64(agentNum), b.RandomBitmapOrder) + num := GenerateImportCSV(f, b.BaseBitmapID, b.MaxBitmapID, b.BaseProfileID, b.MaxProfileID, + b.MinBitsPerMap, b.MaxBitsPerMap, b.Seed, b.RandomBitmapOrder) b.numbits = num // set b.Paths b.Paths = []string{f.Name()} @@ -139,7 +139,7 @@ func (b *Import) Init(hosts []string, agentNum int) error { } // Run runs the Import benchmark -func (b *Import) Run(ctx context.Context, agentNum int) map[string]interface{} { +func (b *Import) Run(ctx context.Context) map[string]interface{} { results := make(map[string]interface{}) results["numbits"] = b.numbits results["db"] = b.Database diff --git a/bench/multidb.go b/bench/multidb.go index 0c7a69c0d..6e00a2bfe 100644 --- a/bench/multidb.go +++ b/bench/multidb.go @@ -16,10 +16,12 @@ type MultiDBSetBits struct { BaseBitmapID int `json:"base-bitmap-id"` BaseProfileID int `json:"base-profile-id"` Iterations int `json:"iterations"` + Database string `json:"database"` } func (b *MultiDBSetBits) Init(hosts []string, agentNum int) error { b.Name = "multi-db-set-bits" + b.Database = b.Database + strconv.Itoa(agentNum) return b.HasClient.Init(hosts, agentNum) } @@ -61,10 +63,10 @@ func (b *MultiDBSetBits) ConsumeFlags(args []string) ([]string, error) { } // Run runs the MultiDBSetBits benchmark -func (b *MultiDBSetBits) Run(ctx context.Context, agentNum int) map[string]interface{} { +func (b *MultiDBSetBits) Run(ctx context.Context) map[string]interface{} { results := make(map[string]interface{}) if b.client == nil { - results["error"] = fmt.Errorf("No client set for MultiDBSetBits agent: %v", agentNum) + results["error"] = fmt.Errorf("No client set for MultiDBSetBits") return results } s := NewStats() @@ -72,7 +74,7 @@ func (b *MultiDBSetBits) Run(ctx context.Context, agentNum int) map[string]inter for n := 0; n < b.Iterations; n++ { query := fmt.Sprintf("SetBit(%d, 'frame.n', %d)", b.BaseBitmapID+n, b.BaseProfileID+n) start = time.Now() - _, err := b.client.ExecuteQuery(ctx, "multidb"+strconv.Itoa(agentNum), query, true) + _, err := b.client.ExecuteQuery(ctx, b.Database, query, true) if err != nil { results["error"] = err return results diff --git a/bench/random.go b/bench/random.go index 250748cfa..394a03b55 100644 --- a/bench/random.go +++ b/bench/random.go @@ -26,6 +26,7 @@ type RandomSetBits struct { func (b *RandomSetBits) Init(hosts []string, agentNum int) error { b.Name = "random-set-bits" + b.Seed = b.Seed + int64(agentNum) return b.HasClient.Init(hosts, agentNum) } @@ -82,12 +83,12 @@ func (b *RandomSetBits) ConsumeFlags(args []string) ([]string, error) { } // Run runs the RandomSetBits benchmark -func (b *RandomSetBits) Run(ctx context.Context, agentNum int) map[string]interface{} { - src := rand.NewSource(b.Seed + int64(agentNum)) +func (b *RandomSetBits) Run(ctx context.Context) map[string]interface{} { + src := rand.NewSource(b.Seed) rng := rand.New(src) results := make(map[string]interface{}) if b.client == nil { - results["error"] = fmt.Errorf("No client set for RandomSetBits agent: %v", agentNum) + results["error"] = fmt.Errorf("No client set for RandomSetBits") return results } s := NewStats() diff --git a/bench/randquery.go b/bench/randquery.go index a56f29445..09cf8dddb 100644 --- a/bench/randquery.go +++ b/bench/randquery.go @@ -25,6 +25,7 @@ type RandomQuery struct { func (b *RandomQuery) Init(hosts []string, agentNum int) error { b.Name = "random-query" + b.Seed = b.Seed + int64(agentNum) return b.HasClient.Init(hosts, agentNum) } @@ -87,14 +88,13 @@ func (b *RandomQuery) ConsumeFlags(args []string) ([]string, error) { } // Run runs the RandomQuery benchmark -func (b *RandomQuery) Run(ctx context.Context, agentNum int) map[string]interface{} { - seed := b.Seed + int64(agentNum) +func (b *RandomQuery) Run(ctx context.Context) map[string]interface{} { results := make(map[string]interface{}) if b.client == nil { - results["error"] = fmt.Errorf("No client set for RandomQuery agent: %v", agentNum) + results["error"] = fmt.Errorf("No client set for RandomQuery") return results } - qm := NewQueryGenerator(seed) + qm := NewQueryGenerator(b.Seed) s := NewStats() var start time.Time for n := 0; n < b.Iterations; n++ { diff --git a/bench/sliceheight.go b/bench/sliceheight.go index bb9131859..abed6675b 100644 --- a/bench/sliceheight.go +++ b/bench/sliceheight.go @@ -91,7 +91,7 @@ func (b *SliceHeight) Init(hosts []string, agentNum int) error { } // Run runs the SliceHeight benchmark -func (b *SliceHeight) Run(ctx context.Context, agentNum int) map[string]interface{} { +func (b *SliceHeight) Run(ctx context.Context) map[string]interface{} { results := make(map[string]interface{}) imp := NewImport(b.Stdin, b.Stdout, b.Stderr) @@ -109,11 +109,11 @@ func (b *SliceHeight) Run(ctx context.Context, agentNum int) map[string]interfac results["iteration"+strconv.Itoa(i)] = iresults genstart := time.Now() - imp.Init(b.hosts, agentNum) + imp.Init(b.hosts, 0) gendur := time.Now().Sub(genstart) iresults["csvgen"] = gendur - iresults["import"] = imp.Run(ctx, agentNum) + iresults["import"] = imp.Run(ctx) qstart := time.Now() q := &pql.TopN{Frame: b.Frame, N: 50} diff --git a/bench/zipf.go b/bench/zipf.go index dfb317d61..967c20331 100644 --- a/bench/zipf.go +++ b/bench/zipf.go @@ -122,7 +122,8 @@ func getZipfOffset(N int64, exp, ratio float64) float64 { func (b *ZipfSetBits) Init(hosts []string, agentNum int) error { b.Name = "zipf-set-bits" - rnd := rand.New(rand.NewSource(b.Seed + int64(agentNum))) + b.Seed = b.Seed + int64(agentNum) + rnd := rand.New(rand.NewSource(b.Seed)) bitmapOffset := getZipfOffset(b.BitmapIDRange, b.BitmapExponent, b.BitmapRatio) b.bitmapRng = rand.NewZipf(rnd, b.BitmapExponent, bitmapOffset, uint64(b.BitmapIDRange-1)) profileOffset := getZipfOffset(b.ProfileIDRange, b.ProfileExponent, b.ProfileRatio) @@ -135,10 +136,10 @@ func (b *ZipfSetBits) Init(hosts []string, agentNum int) error { } // Run runs the ZipfSetBits benchmark -func (b *ZipfSetBits) Run(ctx context.Context, agentNum int) map[string]interface{} { +func (b *ZipfSetBits) Run(ctx context.Context) map[string]interface{} { results := make(map[string]interface{}) if b.client == nil { - results["error"] = fmt.Errorf("No client set for ZipfSetBits agent: %v", agentNum) + results["error"] = fmt.Errorf("No client set for ZipfSetBits") return results } s := NewStats() diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 965728956..ec8651ab5 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -1303,7 +1303,7 @@ func (cmd *BagentCommand) Run(ctx context.Context) error { return fmt.Errorf("in cmd.Run initialization: %v", err) } - res := sbm.Run(ctx, cmd.AgentNum) + res := sbm.Run(ctx) res["agent-num"] = cmd.AgentNum enc := json.NewEncoder(cmd.Stdout) if cmd.HumanReadable { @@ -1622,14 +1622,14 @@ func (sb *serialBenchmark) Init(hosts []string, agentNum int) error { // Run runs the serial benchmark and returns it's results in a nested map - the // top level keys are the indices of each benchmark in the list of benchmarks, // and the values are the results of each benchmark's Run method. -func (sb *serialBenchmark) Run(ctx context.Context, agentNum int) map[string]interface{} { +func (sb *serialBenchmark) Run(ctx context.Context) map[string]interface{} { benchmarks := make([]map[string]interface{}, len(sb.benchmarkers)) results := map[string]interface{}{"benchmarks": benchmarks} total_start := time.Now() for i, b := range sb.benchmarkers { start := time.Now() - output := b.Run(ctx, agentNum) + output := b.Run(ctx) if _, ok := output["runtime"]; ok { panic(fmt.Sprintf("Benchmark %v added 'runtime' to its results", b)) }