From af1d35affceba4993e177009c451dc73e3e6ad1b Mon Sep 17 00:00:00 2001 From: jaffee Date: Tue, 6 Dec 2016 16:11:00 -0600 Subject: [PATCH 1/3] import benchmark sort by profile id after bitmap id --- bench/import.go | 22 +++++++++++++++++++--- bench/import_test.go | 33 ++++++++++++++++----------------- cmd/pilosactl/import.json | 7 +------ 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/bench/import.go b/bench/import.go index e06fea145..651cbb49a 100644 --- a/bench/import.go +++ b/bench/import.go @@ -8,6 +8,8 @@ import ( "io/ioutil" "math/rand" + "sort" + "github.com/pilosa/pilosa/pilosactl" ) @@ -147,6 +149,12 @@ func (b *Import) Run(agentNum int) map[string]interface{} { return results } +type Int64Slice []int64 + +func (s Int64Slice) Len() int { return len(s) } +func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] } +func (s Int64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, minBitsPerMap, maxBitsPerMap, seed int64, randomOrder bool) int { src := rand.NewSource(seed) rng := rand.New(src) @@ -156,6 +164,7 @@ func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, ma bitmapIDs = rng.Perm(int(maxBitmapID - baseBitmapID)) } numrows := 0 + profileIDs := make(Int64Slice, maxBitsPerMap) for i := baseBitmapID; i < maxBitmapID; i++ { var bitmapID int64 if randomOrder { @@ -165,11 +174,18 @@ func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, ma } numBitsToSet := rng.Int63n(maxBitsPerMap-minBitsPerMap) + minBitsPerMap + numrows += int(numBitsToSet) for j := int64(0); j < numBitsToSet; j++ { - profileID := rng.Int63n(maxProfileID-baseProfileID) + baseProfileID - fmt.Fprintf(w, "%d,%d\n", bitmapID, profileID) - numrows += 1 + profileIDs[j] = rng.Int63n(maxProfileID-baseProfileID) + baseProfileID } + profIDs := profileIDs[:numBitsToSet] + if !randomOrder { + sort.Sort(profIDs) + } + for j := int64(0); j < numBitsToSet; j++ { + fmt.Fprintf(w, "%d,%d\n", bitmapID, profIDs[j]) + } + } return numrows } diff --git a/bench/import_test.go b/bench/import_test.go index 9bc037a3d..6ef13527b 100644 --- a/bench/import_test.go +++ b/bench/import_test.go @@ -13,17 +13,16 @@ import ( ) func TestImportInit(t *testing.T) { - imp := bench.Import{ - BaseBitmapID: 0, - MaxBitmapID: 10, - BaseProfileID: 0, - MaxProfileID: 10, - RandomBitmapOrder: false, - MinBitsPerMap: 2, - MaxBitsPerMap: 3, - AgentControls: "width", - Seed: 0, - } + imp := bench.NewImport(os.Stdin, os.Stdout, os.Stderr) + imp.BaseBitmapID = 0 + imp.MaxBitmapID = 10 + imp.BaseProfileID = 0 + imp.MaxProfileID = 10 + imp.RandomBitmapOrder = false + imp.MinBitsPerMap = 2 + imp.MaxBitsPerMap = 3 + imp.AgentControls = "width" + imp.Seed = 0 imp.Init([]string{"blah"}, 2) f, err := os.Open(imp.Paths[0]) @@ -38,8 +37,8 @@ func TestImportInit(t *testing.T) { expected := ` 0,21 0,22 -1,22 1,20 +1,22 2,22 2,26 3,21 @@ -52,10 +51,10 @@ func TestImportInit(t *testing.T) { 6,27 7,20 7,20 -8,29 8,23 -9,29 +8,29 9,23 +9,29 `[1:] if string(bytes) != expected { @@ -78,8 +77,8 @@ func TestGenerateImportCSVNonRand(t *testing.T) { expected := ` 0,21 0,22 -1,22 1,20 +1,22 2,22 2,26 3,21 @@ -92,10 +91,10 @@ func TestGenerateImportCSVNonRand(t *testing.T) { 6,27 7,20 7,20 -8,29 8,23 -9,29 +8,29 9,23 +9,29 `[1:] if string(bytes) != expected { diff --git a/cmd/pilosactl/import.json b/cmd/pilosactl/import.json index 23a77593d..0f5ca0d8a 100644 --- a/cmd/pilosactl/import.json +++ b/cmd/pilosactl/import.json @@ -1,15 +1,10 @@ { - "PilosaHosts": ["localhost:19327"], "CreatorArgs": ["-type", "local", "-serverN", "1", "-replicaN", "1"], "Agents": { "Type": "local" }, "Benchmarks": [ { "Num": 1, - "Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width"] - }, - { - "Num": 1, - "Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "-random-bitmap-order", "-db", "randoload"] + "Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "-random-bitmap-order", "-db", "randoload"] } ] } From b4bb9cf70da1cd6f1b878917527649839bc2f6ce Mon Sep 17 00:00:00 2001 From: jaffee Date: Tue, 6 Dec 2016 17:20:38 -0600 Subject: [PATCH 2/3] add slice height benchmark --- bench/query.go | 3 +- bench/randquery.go | 2 +- bench/sliceheight.go | 126 ++++++++++++++++++++++++++++++++ client.go | 2 +- cmd/pilosactl/main.go | 3 + cmd/pilosactl/slice-height.json | 10 +++ 6 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 bench/sliceheight.go create mode 100644 cmd/pilosactl/slice-height.json diff --git a/bench/query.go b/bench/query.go index 4989608c6..fdb65afac 100644 --- a/bench/query.go +++ b/bench/query.go @@ -1,8 +1,9 @@ package bench import ( - "github.com/pilosa/pilosa/pql" "math/rand" + + "github.com/pilosa/pilosa/pql" ) func NewQueryGenerator(seed int64) *QueryGenerator { diff --git a/bench/randquery.go b/bench/randquery.go index a926f48fd..3366f1a6a 100644 --- a/bench/randquery.go +++ b/bench/randquery.go @@ -9,7 +9,7 @@ import ( "time" ) -// RandomQuery sets bits randomly and deterministically based on a seed. +// RandomQuery queries randomly and deterministically based on a seed. type RandomQuery struct { HasClient MaxDepth int diff --git a/bench/sliceheight.go b/bench/sliceheight.go new file mode 100644 index 000000000..0a12cb04f --- /dev/null +++ b/bench/sliceheight.go @@ -0,0 +1,126 @@ +package bench + +import ( + "context" + "flag" + "io" + "io/ioutil" + "strconv" + "time" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/pql" +) + +func NewSliceHeight(stdin io.Reader, stdout, stderr io.Writer) *SliceHeight { + return &SliceHeight{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + } +} + +// SliceHeight benchmark tests the effect of an increasing number of bitmaps in +// a single slice on query time. +type SliceHeight struct { + MaxTime time.Duration + hosts []string + + MinBitsPerMap int64 + MaxBitsPerMap int64 + Seed int64 + Database string + Frame string + + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +func (b *SliceHeight) Usage() string { + return ` +slice-height repeatedly imports more bitmaps into a single slice and tests query times in between. + +Usage: slice-height [arguments] + +The following arguments are available: + + -max-time int + stop benchmark after this many seconds + + -min-bits-per-map int + minimum number of bits set per bitmap + + -max-bits-per-map int + maximum number of bits set per bitmap + + -seed int + seed for RNG + + -db string + pilosa db to use + + -frame string + frame to import into +`[1:] +} + +func (b *SliceHeight) ConsumeFlags(args []string) ([]string, error) { + fs := flag.NewFlagSet("SliceHeight", flag.ContinueOnError) + fs.SetOutput(ioutil.Discard) + + maxTime := fs.Int("max-time", 30, "") + fs.Int64Var(&b.MinBitsPerMap, "min-bits-per-map", 0, "") + fs.Int64Var(&b.MaxBitsPerMap, "max-bits-per-map", 10, "") + fs.Int64Var(&b.Seed, "seed", 0, "") + fs.StringVar(&b.Database, "db", "benchdb", "") + fs.StringVar(&b.Frame, "frame", "testframe", "") + + if err := fs.Parse(args); err != nil { + return nil, err + } + b.MaxTime = time.Duration(*maxTime) * time.Second + return fs.Args(), nil +} + +func (b *SliceHeight) Init(hosts []string, agentNum int) error { + b.hosts = hosts + return nil +} + +// Run runs the SliceHeight benchmark +func (b *SliceHeight) Run(agentNum int) map[string]interface{} { + results := make(map[string]interface{}) + + imp := NewImport(b.Stdin, b.Stdout, b.Stderr) + imp.MaxBitmapID = 100 + imp.MaxProfileID = pilosa.SliceWidth + imp.MinBitsPerMap = b.MinBitsPerMap + imp.MaxBitsPerMap = b.MaxBitsPerMap + imp.Database = b.Database + imp.Frame = b.Frame + + start := time.Now() + + for i := 0; i > -1; i++ { + imp.Init(b.hosts, agentNum) + results["import"+strconv.Itoa(i)] = imp.Run(agentNum) + qt := time.Now() + q := &pql.TopN{Frame: b.Frame, N: 50} + _, err := imp.Client.ExecuteQuery(context.TODO(), b.Database, q.String(), true) + if err != nil { + results["query"+strconv.Itoa(i)+"error"] = err.Error() + } else { + qdur := time.Now().Sub(qt) + results["query"+strconv.Itoa(i)] = qdur + } + imp.BaseBitmapID = imp.MaxBitmapID + imp.MaxBitmapID = imp.MaxBitmapID * 10 + + if time.Now().Sub(start) > b.MaxTime { + break + } + } + + return results +} diff --git a/client.go b/client.go index d9796700d..ae404b374 100644 --- a/client.go +++ b/client.go @@ -191,7 +191,7 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire return nil, errors.New(s) } - return nil, nil + return qresp, nil } // Import bulk imports bits for a single slice to a host. diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 197c14bf6..70ccf826c 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -1167,6 +1167,8 @@ func (cmd *BagentCommand) ParseFlags(args []string) error { bm = &bench.RandomQuery{} case "import": bm = bench.NewImport(cmd.Stdin, cmd.Stdout, cmd.Stderr) + case "slice-height": + bm = bench.NewSliceHeight(cmd.Stdin, cmd.Stdout, cmd.Stderr) default: return fmt.Errorf("Unknown benchmark cmd: %v", remArgs[0]) } @@ -1208,6 +1210,7 @@ The following arguments are available: multi-db-set-bits random-query import + slice-height `) } diff --git a/cmd/pilosactl/slice-height.json b/cmd/pilosactl/slice-height.json new file mode 100644 index 000000000..44be9625b --- /dev/null +++ b/cmd/pilosactl/slice-height.json @@ -0,0 +1,10 @@ +{ + "CreatorArgs": ["-type", "local", "-serverN", "1", "-replicaN", "1"], + "Agents": { "Type": "local" }, + "Benchmarks": [ + { + "Num": 1, + "Args": ["slice-height", "-max-time", "10", "-max-bits-per-map", "100"] + } + ] +} From 4c67cb30c0ec5400a88b04bcc3e8bf1f0999c929 Mon Sep 17 00:00:00 2001 From: jaffee Date: Wed, 7 Dec 2016 12:09:18 -0600 Subject: [PATCH 3/3] make results nicer --- bench/import.go | 7 +++++-- bench/sliceheight.go | 18 +++++++++++++----- cmd/pilosactl/main.go | 8 +++++++- cmd/pilosactl/slice-height.json | 2 +- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/bench/import.go b/bench/import.go index 651cbb49a..537a4f632 100644 --- a/bench/import.go +++ b/bench/import.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "math/rand" + "time" "sort" @@ -140,12 +141,14 @@ func (b *Import) Init(hosts []string, agentNum int) error { // Run runs the Import benchmark func (b *Import) Run(agentNum int) map[string]interface{} { results := make(map[string]interface{}) + results["numbits"] = b.numbits + results["db"] = b.Database + start := time.Now() err := b.ImportCommand.Run(context.TODO()) if err != nil { results["error"] = err.Error() } - results["numbits"] = b.numbits - results["config"] = *b + results["time"] = time.Now().Sub(start) return results } diff --git a/bench/sliceheight.go b/bench/sliceheight.go index 0a12cb04f..eb5e40b84 100644 --- a/bench/sliceheight.go +++ b/bench/sliceheight.go @@ -103,16 +103,24 @@ func (b *SliceHeight) Run(agentNum int) map[string]interface{} { start := time.Now() for i := 0; i > -1; i++ { + iresults := make(map[string]interface{}) + results["iteration"+strconv.Itoa(i)] = iresults + + genstart := time.Now() imp.Init(b.hosts, agentNum) - results["import"+strconv.Itoa(i)] = imp.Run(agentNum) - qt := time.Now() + gendur := time.Now().Sub(genstart) + iresults["csvgen"] = gendur + + iresults["import"] = imp.Run(agentNum) + + qstart := time.Now() q := &pql.TopN{Frame: b.Frame, N: 50} _, err := imp.Client.ExecuteQuery(context.TODO(), b.Database, q.String(), true) if err != nil { - results["query"+strconv.Itoa(i)+"error"] = err.Error() + iresults["query_error"] = err.Error() } else { - qdur := time.Now().Sub(qt) - results["query"+strconv.Itoa(i)] = qdur + qdur := time.Now().Sub(qstart) + iresults["query"] = qdur } imp.BaseBitmapID = imp.MaxBitmapID imp.MaxBitmapID = imp.MaxBitmapID * 10 diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 70ccf826c..7ddb62909 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -1223,7 +1223,13 @@ func (cmd *BagentCommand) Run(ctx context.Context) error { } res := sbm.Run(cmd.AgentNum) - fmt.Fprintln(cmd.Stdout, res) + enc := json.NewEncoder(cmd.Stdout) + enc.SetIndent("", " ") + err = enc.Encode(res) + if err != nil { + fmt.Fprintln(cmd.Stderr, err) + } + // fmt.Fprintln(cmd.Stdout, res) return nil } diff --git a/cmd/pilosactl/slice-height.json b/cmd/pilosactl/slice-height.json index 44be9625b..de3025397 100644 --- a/cmd/pilosactl/slice-height.json +++ b/cmd/pilosactl/slice-height.json @@ -4,7 +4,7 @@ "Benchmarks": [ { "Num": 1, - "Args": ["slice-height", "-max-time", "10", "-max-bits-per-map", "100"] + "Args": ["slice-height", "-max-time", "1", "-max-bits-per-map", "100"] } ] }