scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-06 08:35:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

import benchmark sort by profile id after bitmap id

Browse source
This commit is contained in:
jaffee 2016-12-06 16:11:00 -06:00
parent 838941b26a
commit af1d35affc
3 changed files with 36 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
bench/import.go
View file

@ -8,6 +8,8 @@ import (
"io/ioutil"
"math/rand"
"sort"
"github.com/pilosa/pilosa/pilosactl"
)
@ -147,6 +149,12 @@ func (b *Import) Run(agentNum int) map[string]interface{} {
return results
}
type Int64Slice []int64
func (s Int64Slice) Len() int { return len(s) }
func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] }
func (s Int64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, minBitsPerMap, maxBitsPerMap, seed int64, randomOrder bool) int {
src := rand.NewSource(seed)
rng := rand.New(src)
@ -156,6 +164,7 @@ func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, ma
bitmapIDs = rng.Perm(int(maxBitmapID - baseBitmapID))
}
numrows := 0
profileIDs := make(Int64Slice, maxBitsPerMap)
for i := baseBitmapID; i < maxBitmapID; i++ {
var bitmapID int64
if randomOrder {
@ -165,11 +174,18 @@ func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, ma
}
numBitsToSet := rng.Int63n(maxBitsPerMap-minBitsPerMap) + minBitsPerMap
numrows += int(numBitsToSet)
for j := int64(0); j < numBitsToSet; j++ {
profileID := rng.Int63n(maxProfileID-baseProfileID) + baseProfileID
fmt.Fprintf(w, "%d,%d\n", bitmapID, profileID)
numrows += 1
profileIDs[j] = rng.Int63n(maxProfileID-baseProfileID) + baseProfileID
}
profIDs := profileIDs[:numBitsToSet]
if !randomOrder {
sort.Sort(profIDs)
}
for j := int64(0); j < numBitsToSet; j++ {
fmt.Fprintf(w, "%d,%d\n", bitmapID, profIDs[j])
}
}
return numrows
}

33
bench/import_test.go
View file

@ -13,17 +13,16 @@ import (
)
func TestImportInit(t *testing.T) {
imp := bench.Import{
BaseBitmapID: 0,
MaxBitmapID: 10,
BaseProfileID: 0,
MaxProfileID: 10,
RandomBitmapOrder: false,
MinBitsPerMap: 2,
MaxBitsPerMap: 3,
AgentControls: "width",
Seed: 0,
}
imp := bench.NewImport(os.Stdin, os.Stdout, os.Stderr)
imp.BaseBitmapID = 0
imp.MaxBitmapID = 10
imp.BaseProfileID = 0
imp.MaxProfileID = 10
imp.RandomBitmapOrder = false
imp.MinBitsPerMap = 2
imp.MaxBitsPerMap = 3
imp.AgentControls = "width"
imp.Seed = 0
imp.Init([]string{"blah"}, 2)
f, err := os.Open(imp.Paths[0])
@ -38,8 +37,8 @@ func TestImportInit(t *testing.T) {
expected := `
0,21
0,22
1,22
1,20
1,22
2,22
2,26
3,21
@ -52,10 +51,10 @@ func TestImportInit(t *testing.T) {
6,27
7,20
7,20
8,29
8,23
9,29
8,29
9,23
9,29
`[1:]
if string(bytes) != expected {
@ -78,8 +77,8 @@ func TestGenerateImportCSVNonRand(t *testing.T) {
expected := `
0,21
0,22
1,22
1,20
1,22
2,22
2,26
3,21
@ -92,10 +91,10 @@ func TestGenerateImportCSVNonRand(t *testing.T) {
6,27
7,20
7,20
8,29
8,23
9,29
8,29
9,23
9,29
`[1:]
if string(bytes) != expected {

7
cmd/pilosactl/import.json
View file

@ -1,15 +1,10 @@
{
"PilosaHosts": ["localhost:19327"],
"CreatorArgs": ["-type", "local", "-serverN", "1", "-replicaN", "1"],
"Agents": { "Type": "local" },
"Benchmarks": [
{
"Num": 1,
"Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width"]
},
{
"Num": 1,
"Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "-random-bitmap-order", "-db", "randoload"]
"Args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "-random-bitmap-order", "-db", "randoload"]
}
]
}