Merge pull request #191 from jaffee/166-single-slice

166 single slice

bench/import.go

@@ -7,6 +7,9 @@ import (
 	"io"
 	"io/ioutil"
 	"math/rand"
+	"time"
+
+	"sort"
 
 	"github.com/pilosa/pilosa/pilosactl"
 )
@@ -138,15 +141,23 @@ func (b *Import) Init(hosts []string, agentNum int) error {
 // Run runs the Import benchmark
 func (b *Import) Run(agentNum int) map[string]interface{} {
 	results := make(map[string]interface{})
+	results["numbits"] = b.numbits
+	results["db"] = b.Database
+	start := time.Now()
 	err := b.ImportCommand.Run(context.TODO())
 	if err != nil {
 		results["error"] = err.Error()
 	}
-	results["numbits"] = b.numbits
-	results["config"] = *b
+	results["time"] = time.Now().Sub(start)
 	return results
 }
 
+type Int64Slice []int64
+
+func (s Int64Slice) Len() int           { return len(s) }
+func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] }
+func (s Int64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
 func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, minBitsPerMap, maxBitsPerMap, seed int64, randomOrder bool) int {
 	src := rand.NewSource(seed)
 	rng := rand.New(src)
@@ -156,6 +167,7 @@ func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, ma
 		bitmapIDs = rng.Perm(int(maxBitmapID - baseBitmapID))
 	}
 	numrows := 0
+	profileIDs := make(Int64Slice, maxBitsPerMap)
 	for i := baseBitmapID; i < maxBitmapID; i++ {
 		var bitmapID int64
 		if randomOrder {
@@ -165,11 +177,18 @@ func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, ma
 		}
 
 		numBitsToSet := rng.Int63n(maxBitsPerMap-minBitsPerMap) + minBitsPerMap
+		numrows += int(numBitsToSet)
 		for j := int64(0); j < numBitsToSet; j++ {
-			profileID := rng.Int63n(maxProfileID-baseProfileID) + baseProfileID
-			fmt.Fprintf(w, "%d,%d\n", bitmapID, profileID)
-			numrows += 1
+			profileIDs[j] = rng.Int63n(maxProfileID-baseProfileID) + baseProfileID
 		}
+		profIDs := profileIDs[:numBitsToSet]
+		if !randomOrder {
+			sort.Sort(profIDs)
+		}
+		for j := int64(0); j < numBitsToSet; j++ {
+			fmt.Fprintf(w, "%d,%d\n", bitmapID, profIDs[j])
+		}
+
 	}
 	return numrows
 }

bench/import_test.go

@@ -13,17 +13,16 @@ import (
 )
 
 func TestImportInit(t *testing.T) {
-	imp := bench.Import{
-		BaseBitmapID:      0,
-		MaxBitmapID:       10,
-		BaseProfileID:     0,
-		MaxProfileID:      10,
-		RandomBitmapOrder: false,
-		MinBitsPerMap:     2,
-		MaxBitsPerMap:     3,
-		AgentControls:     "width",
-		Seed:              0,
-	}
+	imp := bench.NewImport(os.Stdin, os.Stdout, os.Stderr)
+	imp.BaseBitmapID = 0
+	imp.MaxBitmapID = 10
+	imp.BaseProfileID = 0
+	imp.MaxProfileID = 10
+	imp.RandomBitmapOrder = false
+	imp.MinBitsPerMap = 2
+	imp.MaxBitsPerMap = 3
+	imp.AgentControls = "width"
+	imp.Seed = 0
 
 	imp.Init([]string{"blah"}, 2)
 	f, err := os.Open(imp.Paths[0])
@@ -38,8 +37,8 @@ func TestImportInit(t *testing.T) {
 	expected := `
 0,21
 0,22
-1,22
 1,20
+1,22
 2,22
 2,26
 3,21
@@ -52,10 +51,10 @@ func TestImportInit(t *testing.T) {
 6,27
 7,20
 7,20
-8,29
 8,23
-9,29
+8,29
 9,23
+9,29
 `[1:]
 
 	if string(bytes) != expected {
@@ -78,8 +77,8 @@ func TestGenerateImportCSVNonRand(t *testing.T) {
 	expected := `
 0,21
 0,22
-1,22
 1,20
+1,22
 2,22
 2,26
 3,21
@@ -92,10 +91,10 @@ func TestGenerateImportCSVNonRand(t *testing.T) {
 6,27
 7,20
 7,20
-8,29
 8,23
-9,29
+8,29
 9,23
+9,29
 `[1:]
 
 	if string(bytes) != expected {

bench/query.go

@@ -1,8 +1,9 @@
 package bench
 
 import (
-	"github.com/pilosa/pilosa/pql"
 	"math/rand"
+
+	"github.com/pilosa/pilosa/pql"
 )
 
 func NewQueryGenerator(seed int64) *QueryGenerator {

bench/randquery.go

@@ -9,7 +9,7 @@ import (
 	"time"
 )
 
-// RandomQuery sets bits randomly and deterministically based on a seed.
+// RandomQuery queries randomly and deterministically based on a seed.
 type RandomQuery struct {
 	HasClient
 	MaxDepth      int

bench/sliceheight.go

@@ -0,0 +1,134 @@
+package bench
+
+import (
+	"context"
+	"flag"
+	"io"
+	"io/ioutil"
+	"strconv"
+	"time"
+
+	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/pql"
+)
+
+func NewSliceHeight(stdin io.Reader, stdout, stderr io.Writer) *SliceHeight {
+	return &SliceHeight{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+	}
+}
+
+// SliceHeight benchmark tests the effect of an increasing number of bitmaps in
+// a single slice on query time.
+type SliceHeight struct {
+	MaxTime time.Duration
+	hosts   []string
+
+	MinBitsPerMap int64
+	MaxBitsPerMap int64
+	Seed          int64
+	Database      string
+	Frame         string
+
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+func (b *SliceHeight) Usage() string {
+	return `
+slice-height repeatedly imports more bitmaps into a single slice and tests query times in between.
+
+Usage: slice-height [arguments]
+
+The following arguments are available:
+
+	-max-time int
+		stop benchmark after this many seconds
+
+	-min-bits-per-map int
+		minimum number of bits set per bitmap
+
+	-max-bits-per-map int
+		maximum number of bits set per bitmap
+
+	-seed int
+		seed for RNG
+
+	-db string
+		pilosa db to use
+
+	-frame string
+		frame to import into
+`[1:]
+}
+
+func (b *SliceHeight) ConsumeFlags(args []string) ([]string, error) {
+	fs := flag.NewFlagSet("SliceHeight", flag.ContinueOnError)
+	fs.SetOutput(ioutil.Discard)
+
+	maxTime := fs.Int("max-time", 30, "")
+	fs.Int64Var(&b.MinBitsPerMap, "min-bits-per-map", 0, "")
+	fs.Int64Var(&b.MaxBitsPerMap, "max-bits-per-map", 10, "")
+	fs.Int64Var(&b.Seed, "seed", 0, "")
+	fs.StringVar(&b.Database, "db", "benchdb", "")
+	fs.StringVar(&b.Frame, "frame", "testframe", "")
+
+	if err := fs.Parse(args); err != nil {
+		return nil, err
+	}
+	b.MaxTime = time.Duration(*maxTime) * time.Second
+	return fs.Args(), nil
+}
+
+func (b *SliceHeight) Init(hosts []string, agentNum int) error {
+	b.hosts = hosts
+	return nil
+}
+
+// Run runs the SliceHeight benchmark
+func (b *SliceHeight) Run(agentNum int) map[string]interface{} {
+	results := make(map[string]interface{})
+
+	imp := NewImport(b.Stdin, b.Stdout, b.Stderr)
+	imp.MaxBitmapID = 100
+	imp.MaxProfileID = pilosa.SliceWidth
+	imp.MinBitsPerMap = b.MinBitsPerMap
+	imp.MaxBitsPerMap = b.MaxBitsPerMap
+	imp.Database = b.Database
+	imp.Frame = b.Frame
+
+	start := time.Now()
+
+	for i := 0; i > -1; i++ {
+		iresults := make(map[string]interface{})
+		results["iteration"+strconv.Itoa(i)] = iresults
+
+		genstart := time.Now()
+		imp.Init(b.hosts, agentNum)
+		gendur := time.Now().Sub(genstart)
+		iresults["csvgen"] = gendur
+
+		iresults["import"] = imp.Run(agentNum)
+
+		qstart := time.Now()
+		q := &pql.TopN{Frame: b.Frame, N: 50}
+		_, err := imp.Client.ExecuteQuery(context.TODO(), b.Database, q.String(), true)
+		if err != nil {
+			iresults["query_error"] = err.Error()
+		} else {
+			qdur := time.Now().Sub(qstart)
+			iresults["query"] = qdur
+		}
+		imp.BaseBitmapID = imp.MaxBitmapID
+		imp.MaxBitmapID = imp.MaxBitmapID * 10
+
+		if time.Now().Sub(start) > b.MaxTime {
+			break
+		}
+	}
+
+	return results
+}

client.go

@@ -191,7 +191,7 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire
 		return nil, errors.New(s)
 	}
 
-	return nil, nil
+	return qresp, nil
 }
 
 // Import bulk imports bits for a single slice to a host.

cmd/pilosactl/import.json

@@ -1,15 +1,10 @@
 {
-    "PilosaHosts": ["localhost:19327"],
     "CreatorArgs": ["-type", "local", "-serverN", "1", "-replicaN", "1"],
     "Agents": { "Type": "local" },
     "Benchmarks": [
         {
             "Num": 1,
-            "Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width"]
-        },
-        {
-            "Num": 1,
-            "Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "-random-bitmap-order", "-db", "randoload"]
+            "Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "-random-bitmap-order", "-db", "randoload"]
         }
     ]
 }

cmd/pilosactl/main.go

@@ -1167,6 +1167,8 @@ func (cmd *BagentCommand) ParseFlags(args []string) error {
 			bm = &bench.RandomQuery{}
 		case "import":
 			bm = bench.NewImport(cmd.Stdin, cmd.Stdout, cmd.Stderr)
+		case "slice-height":
+			bm = bench.NewSliceHeight(cmd.Stdin, cmd.Stdout, cmd.Stderr)
 		default:
 			return fmt.Errorf("Unknown benchmark cmd: %v", remArgs[0])
 		}
@@ -1208,6 +1210,7 @@ The following arguments are available:
 		multi-db-set-bits
 		random-query
 		import
+		slice-height
 `)
 }
 
@@ -1220,7 +1223,13 @@ func (cmd *BagentCommand) Run(ctx context.Context) error {
 	}
 
 	res := sbm.Run(cmd.AgentNum)
-	fmt.Fprintln(cmd.Stdout, res)
+	enc := json.NewEncoder(cmd.Stdout)
+	enc.SetIndent("", "  ")
+	err = enc.Encode(res)
+	if err != nil {
+		fmt.Fprintln(cmd.Stderr, err)
+	}
+	// fmt.Fprintln(cmd.Stdout, res)
 	return nil
 }
 

cmd/pilosactl/slice-height.json

@@ -0,0 +1,10 @@
+{
+    "CreatorArgs": ["-type", "local", "-serverN", "1", "-replicaN", "1"],
+    "Agents": { "Type": "local" },
+    "Benchmarks": [
+        {
+            "Num": 1,
+            "Args": ["slice-height", "-max-time", "1", "-max-bits-per-map", "100"]
+        }
+    ]
+}