diff --git a/bench/import.go b/bench/import.go index b9f5304e2..e06fea145 100644 --- a/bench/import.go +++ b/bench/import.go @@ -11,6 +11,12 @@ import ( "github.com/pilosa/pilosa/pilosactl" ) +func NewImport(stdin io.Reader, stdout, stderr io.Writer) *Import { + return &Import{ + ImportCommand: pilosactl.NewImportCommand(stdin, stdout, stderr), + } +} + // Import sets bits with increasing profile id and bitmap id. type Import struct { BaseBitmapID int64 @@ -22,8 +28,9 @@ type Import struct { MaxBitsPerMap int64 AgentControls string Seed int64 + numbits int - pilosactl.ImportCommand + *pilosactl.ImportCommand } func (b *Import) Usage() string { @@ -94,30 +101,36 @@ func (b *Import) ConsumeFlags(args []string) ([]string, error) { } func (b *Import) Init(hosts []string, agentNum int) error { - var err error - b.Client, err = firstHostClient(hosts) - if err != nil { - return err + if len(hosts) == 0 { + return fmt.Errorf("Need at least one host") } + b.Host = hosts[0] // generate csv data baseBitmapID, maxBitmapID, baseProfileID, maxProfileID := b.BaseBitmapID, b.MaxBitmapID, b.BaseProfileID, b.MaxProfileID - if b.AgentControls == "height" { + switch b.AgentControls { + case "height": numBitmapIDs := (b.MaxBitmapID - b.BaseBitmapID) baseBitmapID = b.BaseBitmapID + (numBitmapIDs * int64(agentNum)) maxBitmapID = baseBitmapID + numBitmapIDs - } - if b.AgentControls == "height" { + case "width": numProfileIDs := (b.MaxProfileID - b.BaseProfileID) baseProfileID = b.BaseProfileID + (numProfileIDs * int64(agentNum)) maxProfileID = baseProfileID + numProfileIDs + case "": + break + default: + return fmt.Errorf("agent-controls: '%v' is not supported", b.AgentControls) } f, err := ioutil.TempFile("", "") if err != nil { return err } - GenerateImportCSV(f, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, + // set b.Paths) + num := GenerateImportCSV(f, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, b.MinBitsPerMap, b.MaxBitsPerMap, b.Seed+int64(agentNum), b.RandomBitmapOrder) + b.numbits = num // set b.Paths + f.Close() b.Paths = []string{f.Name()} return nil } @@ -125,11 +138,16 @@ func (b *Import) Init(hosts []string, agentNum int) error { // Run runs the Import benchmark func (b *Import) Run(agentNum int) map[string]interface{} { results := make(map[string]interface{}) - b.ImportCommand.Run(context.TODO()) + err := b.ImportCommand.Run(context.TODO()) + if err != nil { + results["error"] = err.Error() + } + results["numbits"] = b.numbits + results["config"] = *b return results } -func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, minBitsPerMap, maxBitsPerMap, seed int64, randomOrder bool) { +func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, minBitsPerMap, maxBitsPerMap, seed int64, randomOrder bool) int { src := rand.NewSource(seed) rng := rand.New(src) @@ -137,6 +155,7 @@ func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, ma if randomOrder { bitmapIDs = rng.Perm(int(maxBitmapID - baseBitmapID)) } + numrows := 0 for i := baseBitmapID; i < maxBitmapID; i++ { var bitmapID int64 if randomOrder { @@ -149,6 +168,8 @@ func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, ma for j := int64(0); j < numBitsToSet; j++ { profileID := rng.Int63n(maxProfileID-baseProfileID) + baseProfileID fmt.Fprintf(w, "%d,%d\n", bitmapID, profileID) + numrows += 1 } } + return numrows } diff --git a/bench/import_test.go b/bench/import_test.go index 879842e84..9bc037a3d 100644 --- a/bench/import_test.go +++ b/bench/import_test.go @@ -2,17 +2,73 @@ package bench_test import ( "bytes" + "log" "testing" "io/ioutil" + "os" + "github.com/pilosa/pilosa/bench" ) +func TestImportInit(t *testing.T) { + imp := bench.Import{ + BaseBitmapID: 0, + MaxBitmapID: 10, + BaseProfileID: 0, + MaxProfileID: 10, + RandomBitmapOrder: false, + MinBitsPerMap: 2, + MaxBitsPerMap: 3, + AgentControls: "width", + Seed: 0, + } + + imp.Init([]string{"blah"}, 2) + f, err := os.Open(imp.Paths[0]) + if err != nil { + t.Fatalf("Couldn't open file: %v, err: %v", imp.Paths[0], err) + } + bytes, err := ioutil.ReadAll(f) + if err != nil { + t.Fatalf("error reading file: %v", err) + } + + expected := ` +0,21 +0,22 +1,22 +1,20 +2,22 +2,26 +3,21 +3,23 +4,21 +4,22 +5,20 +5,28 +6,23 +6,27 +7,20 +7,20 +8,29 +8,23 +9,29 +9,23 +`[1:] + + if string(bytes) != expected { + t.Fatalf("unexpected result: %v", string(bytes)) + } + + log.Println(imp) +} + func TestGenerateImportCSVNonRand(t *testing.T) { b := bytes.NewBuffer(make([]byte, 0)) - bench.GenerateImportCSV(b, 0, 10, 21, 29, 2, 3, 0, false) + bench.GenerateImportCSV(b, 0, 10, 20, 30, 2, 3, 2, false) bytes, err := ioutil.ReadAll(b) if err != nil { @@ -21,25 +77,25 @@ func TestGenerateImportCSVNonRand(t *testing.T) { expected := ` 0,21 -0,24 -1,23 -1,28 -2,21 +0,22 +1,22 +1,20 2,22 +2,26 3,21 -3,25 -4,23 +3,23 4,21 +4,22 +5,20 5,28 -5,28 -6,25 6,23 -7,26 -7,23 -8,21 +6,27 +7,20 +7,20 +8,29 8,23 -9,25 -9,24 +9,29 +9,23 `[1:] if string(bytes) != expected { diff --git a/cmd/pilosactl/import.json b/cmd/pilosactl/import.json new file mode 100644 index 000000000..23a77593d --- /dev/null +++ b/cmd/pilosactl/import.json @@ -0,0 +1,15 @@ +{ + "PilosaHosts": ["localhost:19327"], + "CreatorArgs": ["-type", "local", "-serverN", "1", "-replicaN", "1"], + "Agents": { "Type": "local" }, + "Benchmarks": [ + { + "Num": 1, + "Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width"] + }, + { + "Num": 1, + "Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "-random-bitmap-order", "-db", "randoload"] + } + ] +} diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 53028bad1..197c14bf6 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -24,6 +24,7 @@ import ( "unsafe" "encoding/json" + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/bench" "github.com/pilosa/pilosa/creator" @@ -1164,6 +1165,8 @@ func (cmd *BagentCommand) ParseFlags(args []string) error { bm = &bench.MultiDBSetBits{} case "random-query": bm = &bench.RandomQuery{} + case "import": + bm = bench.NewImport(cmd.Stdin, cmd.Stdout, cmd.Stderr) default: return fmt.Errorf("Unknown benchmark cmd: %v", remArgs[0]) } @@ -1204,9 +1207,7 @@ The following arguments are available: random-set-bits multi-db-set-bits random-query - - - + import `) } diff --git a/pilosactl/import.go b/pilosactl/import.go index 1d9358cce..34d243f1e 100644 --- a/pilosactl/import.go +++ b/pilosactl/import.go @@ -51,6 +51,10 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *ImportCommand } } +func (cmd *ImportCommand) String() string { + return fmt.Sprint(*cmd) +} + // ParseFlags parses command line flags from args. func (cmd *ImportCommand) ParseFlags(args []string) error { fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)