scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-06 08:35:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

wire up import benchmark, fix bug with ImportCommand.Host

Browse source 
also add some config metadata to results in order to distinguish them
This commit is contained in:
jaffee 2016-12-06 10:16:28 -06:00
parent e5edf3e432
commit 838941b26a
5 changed files with 125 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
bench/import.go
View file

@ -11,6 +11,12 @@ import (
"github.com/pilosa/pilosa/pilosactl"
)
func NewImport(stdin io.Reader, stdout, stderr io.Writer) *Import {
return &Import{
ImportCommand: pilosactl.NewImportCommand(stdin, stdout, stderr),
}
}
// Import sets bits with increasing profile id and bitmap id.
type Import struct {
BaseBitmapID int64
@ -22,8 +28,9 @@ type Import struct {
MaxBitsPerMap int64
AgentControls string
Seed int64
numbits int
pilosactl.ImportCommand
*pilosactl.ImportCommand
}
func (b *Import) Usage() string {
@ -94,30 +101,36 @@ func (b *Import) ConsumeFlags(args []string) ([]string, error) {
}
func (b *Import) Init(hosts []string, agentNum int) error {
var err error
b.Client, err = firstHostClient(hosts)
if err != nil {
return err
if len(hosts) == 0 {
return fmt.Errorf("Need at least one host")
}
b.Host = hosts[0]
// generate csv data
baseBitmapID, maxBitmapID, baseProfileID, maxProfileID := b.BaseBitmapID, b.MaxBitmapID, b.BaseProfileID, b.MaxProfileID
if b.AgentControls == "height" {
switch b.AgentControls {
case "height":
numBitmapIDs := (b.MaxBitmapID - b.BaseBitmapID)
baseBitmapID = b.BaseBitmapID + (numBitmapIDs * int64(agentNum))
maxBitmapID = baseBitmapID + numBitmapIDs
}
if b.AgentControls == "height" {
case "width":
numProfileIDs := (b.MaxProfileID - b.BaseProfileID)
baseProfileID = b.BaseProfileID + (numProfileIDs * int64(agentNum))
maxProfileID = baseProfileID + numProfileIDs
case "":
break
default:
return fmt.Errorf("agent-controls: '%v' is not supported", b.AgentControls)
}
f, err := ioutil.TempFile("", "")
if err != nil {
return err
}
GenerateImportCSV(f, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID,
// set b.Paths)
num := GenerateImportCSV(f, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID,
b.MinBitsPerMap, b.MaxBitsPerMap, b.Seed+int64(agentNum), b.RandomBitmapOrder)
b.numbits = num
// set b.Paths
f.Close()
b.Paths = []string{f.Name()}
return nil
}
@ -125,11 +138,16 @@ func (b *Import) Init(hosts []string, agentNum int) error {
// Run runs the Import benchmark
func (b *Import) Run(agentNum int) map[string]interface{} {
results := make(map[string]interface{})
b.ImportCommand.Run(context.TODO())
err := b.ImportCommand.Run(context.TODO())
if err != nil {
results["error"] = err.Error()
}
results["numbits"] = b.numbits
results["config"] = *b
return results
}
func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, minBitsPerMap, maxBitsPerMap, seed int64, randomOrder bool) {
func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, minBitsPerMap, maxBitsPerMap, seed int64, randomOrder bool) int {
src := rand.NewSource(seed)
rng := rand.New(src)
@ -137,6 +155,7 @@ func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, ma
if randomOrder {
bitmapIDs = rng.Perm(int(maxBitmapID - baseBitmapID))
}
numrows := 0
for i := baseBitmapID; i < maxBitmapID; i++ {
var bitmapID int64
if randomOrder {
@ -149,6 +168,8 @@ func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, ma
for j := int64(0); j < numBitsToSet; j++ {
profileID := rng.Int63n(maxProfileID-baseProfileID) + baseProfileID
fmt.Fprintf(w, "%d,%d\n", bitmapID, profileID)
numrows += 1
}
}
return numrows
}

84
bench/import_test.go
View file

@ -2,17 +2,73 @@ package bench_test
import (
"bytes"
"log"
"testing"
"io/ioutil"
"os"
"github.com/pilosa/pilosa/bench"
)
func TestImportInit(t *testing.T) {
imp := bench.Import{
BaseBitmapID: 0,
MaxBitmapID: 10,
BaseProfileID: 0,
MaxProfileID: 10,
RandomBitmapOrder: false,
MinBitsPerMap: 2,
MaxBitsPerMap: 3,
AgentControls: "width",
Seed: 0,
}
imp.Init([]string{"blah"}, 2)
f, err := os.Open(imp.Paths[0])
if err != nil {
t.Fatalf("Couldn't open file: %v, err: %v", imp.Paths[0], err)
}
bytes, err := ioutil.ReadAll(f)
if err != nil {
t.Fatalf("error reading file: %v", err)
}
expected := `
0,21
0,22
1,22
1,20
2,22
2,26
3,21
3,23
4,21
4,22
5,20
5,28
6,23
6,27
7,20
7,20
8,29
8,23
9,29
9,23
`[1:]
if string(bytes) != expected {
t.Fatalf("unexpected result: %v", string(bytes))
}
log.Println(imp)
}
func TestGenerateImportCSVNonRand(t *testing.T) {
b := bytes.NewBuffer(make([]byte, 0))
bench.GenerateImportCSV(b, 0, 10, 21, 29, 2, 3, 0, false)
bench.GenerateImportCSV(b, 0, 10, 20, 30, 2, 3, 2, false)
bytes, err := ioutil.ReadAll(b)
if err != nil {
@ -21,25 +77,25 @@ func TestGenerateImportCSVNonRand(t *testing.T) {
expected := `
0,21
0,24
1,23
1,28
2,21
0,22
1,22
1,20
2,22
2,26
3,21
3,25
4,23
3,23
4,21
4,22
5,20
5,28
5,28
6,25
6,23
7,26
7,23
8,21
6,27
7,20
7,20
8,29
8,23
9,25
9,24
9,29
9,23
`[1:]
if string(bytes) != expected {

15
cmd/pilosactl/import.json Normal file
View file

@ -0,0 +1,15 @@
{
"PilosaHosts": ["localhost:19327"],
"CreatorArgs": ["-type", "local", "-serverN", "1", "-replicaN", "1"],
"Agents": { "Type": "local" },
"Benchmarks": [
{
"Num": 1,
"Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width"]
},
{
"Num": 1,
"Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "-random-bitmap-order", "-db", "randoload"]
}
]
}

7
cmd/pilosactl/main.go
View file

@ -24,6 +24,7 @@ import (
"unsafe"
"encoding/json"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/bench"
"github.com/pilosa/pilosa/creator"
@ -1164,6 +1165,8 @@ func (cmd *BagentCommand) ParseFlags(args []string) error {
bm = &bench.MultiDBSetBits{}
case "random-query":
bm = &bench.RandomQuery{}
case "import":
bm = bench.NewImport(cmd.Stdin, cmd.Stdout, cmd.Stderr)
default:
return fmt.Errorf("Unknown benchmark cmd: %v", remArgs[0])
}
@ -1204,9 +1207,7 @@ The following arguments are available:
random-set-bits
multi-db-set-bits
random-query
import
`)
}

4
pilosactl/import.go
View file

@ -51,6 +51,10 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *ImportCommand
}
}
func (cmd *ImportCommand) String() string {
return fmt.Sprint(*cmd)
}
// ParseFlags parses command line flags from args.
func (cmd *ImportCommand) ParseFlags(args []string) error {
fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)