From af1d35affceba4993e177009c451dc73e3e6ad1b Mon Sep 17 00:00:00 2001
From: jaffee <jaffee@umbel.com>
Date: Tue, 6 Dec 2016 16:11:00 -0600
Subject: [PATCH] import benchmark sort by profile id after bitmap id

---
 bench/import.go           | 22 +++++++++++++++++++---
 bench/import_test.go      | 33 ++++++++++++++++-----------------
 cmd/pilosactl/import.json |  7 +------
 3 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/bench/import.go b/bench/import.go
index e06fea145..651cbb49a 100644
--- a/bench/import.go
+++ b/bench/import.go
@@ -8,6 +8,8 @@ import (
 	"io/ioutil"
 	"math/rand"
 
+	"sort"
+
 	"github.com/pilosa/pilosa/pilosactl"
 )
 
@@ -147,6 +149,12 @@ func (b *Import) Run(agentNum int) map[string]interface{} {
 	return results
 }
 
+type Int64Slice []int64
+
+func (s Int64Slice) Len() int           { return len(s) }
+func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] }
+func (s Int64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
 func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, minBitsPerMap, maxBitsPerMap, seed int64, randomOrder bool) int {
 	src := rand.NewSource(seed)
 	rng := rand.New(src)
@@ -156,6 +164,7 @@ func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, ma
 		bitmapIDs = rng.Perm(int(maxBitmapID - baseBitmapID))
 	}
 	numrows := 0
+	profileIDs := make(Int64Slice, maxBitsPerMap)
 	for i := baseBitmapID; i < maxBitmapID; i++ {
 		var bitmapID int64
 		if randomOrder {
@@ -165,11 +174,18 @@ func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, ma
 		}
 
 		numBitsToSet := rng.Int63n(maxBitsPerMap-minBitsPerMap) + minBitsPerMap
+		numrows += int(numBitsToSet)
 		for j := int64(0); j < numBitsToSet; j++ {
-			profileID := rng.Int63n(maxProfileID-baseProfileID) + baseProfileID
-			fmt.Fprintf(w, "%d,%d\n", bitmapID, profileID)
-			numrows += 1
+			profileIDs[j] = rng.Int63n(maxProfileID-baseProfileID) + baseProfileID
 		}
+		profIDs := profileIDs[:numBitsToSet]
+		if !randomOrder {
+			sort.Sort(profIDs)
+		}
+		for j := int64(0); j < numBitsToSet; j++ {
+			fmt.Fprintf(w, "%d,%d\n", bitmapID, profIDs[j])
+		}
+
 	}
 	return numrows
 }
diff --git a/bench/import_test.go b/bench/import_test.go
index 9bc037a3d..6ef13527b 100644
--- a/bench/import_test.go
+++ b/bench/import_test.go
@@ -13,17 +13,16 @@ import (
 )
 
 func TestImportInit(t *testing.T) {
-	imp := bench.Import{
-		BaseBitmapID:      0,
-		MaxBitmapID:       10,
-		BaseProfileID:     0,
-		MaxProfileID:      10,
-		RandomBitmapOrder: false,
-		MinBitsPerMap:     2,
-		MaxBitsPerMap:     3,
-		AgentControls:     "width",
-		Seed:              0,
-	}
+	imp := bench.NewImport(os.Stdin, os.Stdout, os.Stderr)
+	imp.BaseBitmapID = 0
+	imp.MaxBitmapID = 10
+	imp.BaseProfileID = 0
+	imp.MaxProfileID = 10
+	imp.RandomBitmapOrder = false
+	imp.MinBitsPerMap = 2
+	imp.MaxBitsPerMap = 3
+	imp.AgentControls = "width"
+	imp.Seed = 0
 
 	imp.Init([]string{"blah"}, 2)
 	f, err := os.Open(imp.Paths[0])
@@ -38,8 +37,8 @@ func TestImportInit(t *testing.T) {
 	expected := `
 0,21
 0,22
-1,22
 1,20
+1,22
 2,22
 2,26
 3,21
@@ -52,10 +51,10 @@ func TestImportInit(t *testing.T) {
 6,27
 7,20
 7,20
-8,29
 8,23
-9,29
+8,29
 9,23
+9,29
 `[1:]
 
 	if string(bytes) != expected {
@@ -78,8 +77,8 @@ func TestGenerateImportCSVNonRand(t *testing.T) {
 	expected := `
 0,21
 0,22
-1,22
 1,20
+1,22
 2,22
 2,26
 3,21
@@ -92,10 +91,10 @@ func TestGenerateImportCSVNonRand(t *testing.T) {
 6,27
 7,20
 7,20
-8,29
 8,23
-9,29
+8,29
 9,23
+9,29
 `[1:]
 
 	if string(bytes) != expected {
diff --git a/cmd/pilosactl/import.json b/cmd/pilosactl/import.json
index 23a77593d..0f5ca0d8a 100644
--- a/cmd/pilosactl/import.json
+++ b/cmd/pilosactl/import.json
@@ -1,15 +1,10 @@
 {
-    "PilosaHosts": ["localhost:19327"],
     "CreatorArgs": ["-type", "local", "-serverN", "1", "-replicaN", "1"],
     "Agents": { "Type": "local" },
     "Benchmarks": [
         {
             "Num": 1,
-            "Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width"]
-        },
-        {
-            "Num": 1,
-            "Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "-random-bitmap-order", "-db", "randoload"]
+            "Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "-random-bitmap-order", "-db", "randoload"]
         }
     ]
 }