From c7a0aa371d457f580e9d5ba07a342cb44eef1d55 Mon Sep 17 00:00:00 2001
From: Yuce Tekol <yucetekol@gmail.com>
Date: Mon, 20 Feb 2017 20:11:58 +0300
Subject: [PATCH 01/37] Installs glide in the docker container

---
 Dockerfile | 14 +++++++++++++-
 Makefile   |  2 +-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 257bdcd86..07e13620f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -7,8 +7,20 @@ VOLUME /data
 
 RUN echo 'data-dir = "/data"' > /config
 
+RUN git clone --depth 1 https://github.com/Masterminds/glide.git /go/src/github.com/Masterminds/glide \
+    && cd /go/src/github.com/Masterminds/glide \
+    && git fetch --tags --depth 1 \
+    && git checkout tags/v0.12.3 -b build \
+    && make build \
+    && mv ./glide /go/bin \
+    && cd / \
+    && rm -r /go/src/github.com/Masterminds/glide
+
 COPY . /go/src/github.com/pilosa/pilosa
-RUN CGO_ENABLED=0 go install -a github.com/pilosa/pilosa/cmd/pilosa
+
+RUN cd /go/src/github.com/pilosa/pilosa \
+    && make vendor \
+    && CGO_ENABLED=0 go install -a github.com/pilosa/pilosa/cmd/pilosa
 
 ENTRYPOINT ["/go/bin/pilosa"]
 CMD ["-config", "/config"]
diff --git a/Makefile b/Makefile
index 09b5616be..f55fb8445 100644
--- a/Makefile
+++ b/Makefile
@@ -43,5 +43,5 @@ install: vendor
 	go install $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
 	go install $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosactl
 
-docker: vendor
+docker:
 	docker build -t pilosa:latest .

From 7de02930342e31560a619b5f9e95fa81cf6aeeba Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Mon, 20 Feb 2017 13:55:03 -0700
Subject: [PATCH 02/37] Add basic schema support.

Databases and frames now require explicit creation and have the
option of setting row & column labels. If no labels are provided
then the default `id` and `profileID` labels are used.
---
 client.go               |  97 ++++++++++++++++++++++
 client_test.go          |   3 +
 cmd/pilosa/main_test.go |  51 ++++++++++++
 db.go                   | 113 +++++++++++++++++++-------
 db_test.go              |   6 +-
 executor.go             | 172 ++++++++++++++++++++++++++--------------
 executor_test.go        |  30 ++++++-
 frame.go                |  59 +++++++++++++-
 handler.go              | 133 ++++++++++++++++++++++++++-----
 handler_test.go         |  35 +++++---
 index.go                |  66 ++++++++-------
 index_test.go           |  22 +++--
 internal/internal.pb.go | 172 +++++++++++++++++++++++++++++-----------
 internal/internal.proto |   2 +
 pilosa.go               |  13 ++-
 server.go               |   6 +-
 16 files changed, 770 insertions(+), 210 deletions(-)

diff --git a/client.go b/client.go
index 5c24920ff..e7d0eaf85 100644
--- a/client.go
+++ b/client.go
@@ -108,6 +108,51 @@ func (c *Client) Schema(ctx context.Context) ([]*DBInfo, error) {
 	return rsp.DBs, nil
 }
 
+// CreateDB creates a new database on the server.
+func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error {
+	// Encode query request.
+	buf, err := json.Marshal(&postDBRequest{
+		DB:      db,
+		Options: opt,
+	})
+	if err != nil {
+		return err
+	}
+
+	// Create URL & HTTP request.
+	u := url.URL{Scheme: "http", Host: c.host, Path: "/db"}
+	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")
+
+	// Execute request against the host.
+	resp, err := c.HTTPClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	// Read body.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	// Handle response based on status code.
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return nil // ok
+	case http.StatusConflict:
+		return ErrDatabaseExists
+	default:
+		return errors.New(string(body))
+	}
+}
+
 // FragmentNodes returns a list of nodes that own a slice.
 func (c *Client) FragmentNodes(ctx context.Context, db string, slice uint64) ([]*Node, error) {
 	// Execute request against the host.
@@ -596,6 +641,56 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame str
 	return nil
 }
 
+// CreateFrame creates a new frame on the server.
+func (c *Client) CreateFrame(ctx context.Context, db, frame string, opt FrameOptions) error {
+	if db == "" {
+		return ErrDatabaseRequired
+	}
+
+	// Encode query request.
+	buf, err := json.Marshal(&postFrameRequest{
+		DB:      db,
+		Frame:   frame,
+		Options: opt,
+	})
+	if err != nil {
+		return err
+	}
+
+	// Create URL & HTTP request.
+	u := url.URL{Scheme: "http", Host: c.host, Path: "/frame"}
+	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")
+
+	// Execute request against the host.
+	resp, err := c.HTTPClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	// Read body.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	// Handle response based on status code.
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return nil // ok
+	case http.StatusConflict:
+		return ErrFrameExists
+	default:
+		return errors.New(string(body))
+	}
+}
+
 // RestoreFrame restores an entire frame from a host in another cluster.
 func (c *Client) RestoreFrame(ctx context.Context, host, db, frame string) error {
 	u := url.URL{
@@ -797,6 +892,8 @@ func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []At
 	// Return error if status is not OK.
 	switch resp.StatusCode {
 	case http.StatusOK: // ok
+	case http.StatusNotFound:
+		return nil, ErrFrameNotFound
 	default:
 		return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode)
 	}
diff --git a/client_test.go b/client_test.go
index f37fea6ee..aa8b3b126 100644
--- a/client_test.go
+++ b/client_test.go
@@ -71,6 +71,9 @@ func TestClient_BackupRestore(t *testing.T) {
 	}
 
 	// Restore to a different frame.
+	if _, err := idx.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil {
+		t.Fatal(err)
+	}
 	if err := c.RestoreFrom(context.Background(), &buf, "x", "y"); err != nil {
 		t.Fatal(err)
 	}
diff --git a/cmd/pilosa/main_test.go b/cmd/pilosa/main_test.go
index 1267d44a5..184f5c334 100644
--- a/cmd/pilosa/main_test.go
+++ b/cmd/pilosa/main_test.go
@@ -31,8 +31,20 @@ func TestMain_Set_Quick(t *testing.T) {
 		m := MustRunMain()
 		defer m.Close()
 
+		// Create client.
+		client, err := pilosa.NewClient(m.Server.Host)
+		if err != nil {
+			t.Fatal(err)
+		}
+
 		// Execute SetBit() commands.
 		for _, cmd := range cmds {
+			if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
+				t.Fatal(err)
+			}
+			if err := client.CreateFrame(context.Background(), "d", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists {
+				t.Fatal(err)
+			}
 			if _, err := m.Query("db=d", fmt.Sprintf(`SetBit(id=%d, frame=%q, profileID=%d)`, cmd.ID, cmd.Frame, cmd.ProfileID)); err != nil {
 				t.Fatal(err)
 			}
@@ -95,6 +107,16 @@ func TestMain_SetBitmapAttrs(t *testing.T) {
 	m := MustRunMain()
 	defer m.Close()
 
+	// Create frames.
+	client := m.Client()
+	if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
+		t.Fatal(err)
+	} else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil {
+		t.Fatal(err)
+	} else if err := client.CreateFrame(context.Background(), "d", "z", pilosa.FrameOptions{}); err != nil {
+		t.Fatal(err)
+	}
+
 	// Set bits on different bitmaps in different frames.
 	if _, err := m.Query("db=d", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil {
 		t.Fatal(err)
@@ -144,6 +166,14 @@ func TestMain_SetProfileAttrs(t *testing.T) {
 	m := MustRunMain()
 	defer m.Close()
 
+	// Create frames.
+	client := m.Client()
+	if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
+		t.Fatal(err)
+	} else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil {
+		t.Fatal(err)
+	}
+
 	// Set bits on bitmap.
 	if _, err := m.Query("db=d", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil {
 		t.Fatal(err)
@@ -190,6 +220,14 @@ func TestMain_FrameRestore(t *testing.T) {
 	}
 	m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes
 
+	// Create frames.
+	client := m0.Client()
+	if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
+		t.Fatal(err)
+	} else if err := client.CreateFrame(context.Background(), "d", "f", pilosa.FrameOptions{}); err != nil {
+		t.Fatal(err)
+	}
+
 	// Write data on first cluster.
 	if _, err := m0.Query("db=d", `
 		SetBit(id=1, frame="f", profileID=100)
@@ -218,6 +256,10 @@ func TestMain_FrameRestore(t *testing.T) {
 	client, err := pilosa.NewClient(m2.Server.Host)
 	if err != nil {
 		t.Fatal(err)
+	} else if err := m2.Client().CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
+		t.Fatal(err)
+	} else if err := m2.Client().CreateFrame(context.Background(), "d", "f", pilosa.FrameOptions{}); err != nil {
+		t.Fatal(err)
 	} else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "d", "f"); err != nil {
 		t.Fatal(err)
 	}
@@ -327,6 +369,15 @@ func (m *Main) Reopen() error {
 // URL returns the base URL string for accessing the running program.
 func (m *Main) URL() string { return "http://" + m.Server.Addr().String() }
 
+// Client returns a client to connect to the program.
+func (m *Main) Client() *pilosa.Client {
+	client, err := pilosa.NewClient(m.Server.Host)
+	if err != nil {
+		panic(err)
+	}
+	return client
+}
+
 // Query executes a query against the program through the HTTP API.
 func (m *Main) Query(rawQuery, query string) (string, error) {
 	resp := MustDo("POST", m.URL()+"/query?"+rawQuery, query)
diff --git a/db.go b/db.go
index 2a4f1d6e0..23f678bf6 100644
--- a/db.go
+++ b/db.go
@@ -15,6 +15,11 @@ import (
 	"github.com/pilosa/pilosa/internal"
 )
 
+// Default database settings.
+const (
+	DefaultColumnLabel = "profileID"
+)
+
 // DB represents a container for frames.
 type DB struct {
 	mu   sync.Mutex
@@ -25,6 +30,9 @@ type DB struct {
 	// This can be overridden by individual frames.
 	timeQuantum TimeQuantum
 
+	// Label used for referring to columns in database.
+	columnLabel string
+
 	// Frames by name.
 	frames map[string]*Frame
 
@@ -49,6 +57,8 @@ func NewDB(path, name string) *DB {
 
 		profileAttrStore: NewAttrStore(filepath.Join(path, "data")),
 
+		columnLabel: DefaultColumnLabel,
+
 		stats:     NopStatsClient,
 		LogOutput: ioutil.Discard,
 	}
@@ -63,6 +73,33 @@ func (db *DB) Path() string { return db.path }
 // ProfileAttrStore returns the storage for profile attributes.
 func (db *DB) ProfileAttrStore() *AttrStore { return db.profileAttrStore }
 
+// SetColumnLabel sets the column label. Persists to meta file on update.
+func (db *DB) SetColumnLabel(v string) error {
+	db.mu.Lock()
+	defer db.mu.Unlock()
+
+	// Ignore if no change occurred.
+	if v == "" || db.columnLabel == v {
+		return nil
+	}
+
+	// Persist meta data to disk on change.
+	db.columnLabel = v
+	if err := db.saveMeta(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ColumnLabel returns the column label.
+func (db *DB) ColumnLabel() string {
+	db.mu.Lock()
+	v := db.columnLabel
+	db.mu.Unlock()
+	return v
+}
+
 // Open opens and initializes the database.
 func (db *DB) Open() error {
 	// Ensure the path exists.
@@ -123,6 +160,7 @@ func (db *DB) loadMeta() error {
 	buf, err := ioutil.ReadFile(filepath.Join(db.path, "meta"))
 	if os.IsNotExist(err) {
 		db.timeQuantum = ""
+		db.columnLabel = DefaultColumnLabel
 		return nil
 	} else if err != nil {
 		return err
@@ -134,6 +172,7 @@ func (db *DB) loadMeta() error {
 
 	// Copy metadata fields.
 	db.timeQuantum = TimeQuantum(pb.TimeQuantum)
+	db.columnLabel = pb.ColumnLabel
 
 	return nil
 }
@@ -141,7 +180,10 @@ func (db *DB) loadMeta() error {
 // saveMeta writes meta data for the database.
 func (db *DB) saveMeta() error {
 	// Marshal metadata.
-	buf, err := proto.Marshal(&internal.DB{TimeQuantum: string(db.timeQuantum)})
+	buf, err := proto.Marshal(&internal.DB{
+		TimeQuantum: string(db.timeQuantum),
+		ColumnLabel: db.columnLabel,
+	})
 	if err != nil {
 		return err
 	}
@@ -244,28 +286,48 @@ func (db *DB) Frames() []*Frame {
 	return a
 }
 
-// CreateFrameIfNotExists returns a frame in the database by name.
-func (db *DB) CreateFrameIfNotExists(name string) (*Frame, error) {
+// CreateFrame creates a frame.
+func (db *DB) CreateFrame(name string, opt FrameOptions) (*Frame, error) {
 	db.mu.Lock()
 	defer db.mu.Unlock()
-	return db.createFrameIfNotExists(name)
+
+	// Ensure frame doesn't already exist.
+	if db.frames[name] != nil {
+		return nil, ErrFrameExists
+	}
+	return db.createFrame(name, opt)
 }
 
-func (db *DB) createFrameIfNotExists(name string) (*Frame, error) {
-	if name == "" {
-		return nil, errors.New("frame name required")
-	}
+// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
+func (db *DB) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) {
+	db.mu.Lock()
+	defer db.mu.Unlock()
 
 	// Find frame in cache first.
 	if f := db.frames[name]; f != nil {
 		return f, nil
 	}
 
-	// Initialize and open frame.
+	return db.createFrame(name, opt)
+}
+
+func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) {
+	if name == "" {
+		return nil, errors.New("frame name required")
+	}
+
+	// Initialize frame.
 	f := db.newFrame(db.FramePath(name), name)
+
+	// Open frame.
 	if err := f.Open(); err != nil {
 		return nil, err
 	}
+
+	// Update options.
+	f.SetRowLabel(opt.RowLabel)
+
+	// Add to database's frame lookup.
 	db.frames[name] = f
 
 	db.stats.Count("frameN", 1)
@@ -309,22 +371,13 @@ func (db *DB) DeleteFrame(name string) error {
 	return nil
 }
 
-// CreateFragmentIfNotExists returns a fragment in the database by name/slice.
-func (db *DB) CreateFragmentIfNotExists(name string, slice uint64) (*Fragment, error) {
-	f, err := db.CreateFrameIfNotExists(name)
-	if err != nil {
-		return nil, err
-	}
-	return f.CreateFragmentIfNotExists(slice)
-}
-
 // SetBit sets a bit for a given profile & bitmap.
 // If a timestamp is specified then set all bits for the different quantum units.
 func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool, err error) {
 	// Read frame.
-	f, err := db.CreateFrameIfNotExists(name)
-	if err != nil {
-		return changed, err
+	f := db.Frame(name)
+	if f == nil {
+		return changed, ErrFrameNotFound
 	}
 
 	// If this is a non-time bit then simply set the bit on the frame.
@@ -342,8 +395,9 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha
 	}
 
 	// If a timestamp is specified then set bits across all frames for the quantum.
+	opt := f.Options()
 	for _, subname := range FramesByTime(name, *t, q) {
-		f, err := db.CreateFrameIfNotExists(subname)
+		f, err := db.CreateFrameIfNotExists(subname, opt)
 		if err != nil {
 			return changed, err
 		}
@@ -360,9 +414,9 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha
 // Import bulk imports data.
 func (db *DB) Import(name string, bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error {
 	// Read frame.
-	f, err := db.CreateFrameIfNotExists(name)
-	if err != nil {
-		return err
+	f := db.Frame(name)
+	if f == nil {
+		return ErrFrameNotFound
 	}
 
 	// Determine quantum if timestamps are set.
@@ -405,12 +459,12 @@ func (db *DB) Import(name string, bitmapIDs, profileIDs []uint64, timestamps []*
 
 	// Import into each fragment.
 	for key, data := range dataByFragment {
-		f, err := db.CreateFragmentIfNotExists(key.Frame, key.Slice)
+		frag, err := f.CreateFragmentIfNotExists(key.Slice)
 		if err != nil {
 			return err
 		}
 
-		if err := f.Import(data.BitmapIDs, data.ProfileIDs); err != nil {
+		if err := frag.Import(data.BitmapIDs, data.ProfileIDs); err != nil {
 			return err
 		}
 	}
@@ -472,6 +526,11 @@ func (db *DB) SetRemoteMaxSlice(newmax uint64) {
 	db.remoteMaxSlice = newmax
 }
 
+// DBOptions represents options to set when initializing a db.
+type DBOptions struct {
+	ColumnLabel string `json:"columnLabel,omitempty"`
+}
+
 // hasTime returns true if a contains a non-nil time.
 func hasTime(a []*time.Time) bool {
 	for _, t := range a {
diff --git a/db_test.go b/db_test.go
index db017dcd4..b95f89e62 100644
--- a/db_test.go
+++ b/db_test.go
@@ -15,7 +15,7 @@ func TestDB_CreateFrameIfNotExists(t *testing.T) {
 	defer db.Close()
 
 	// Create frame.
-	f, err := db.CreateFrameIfNotExists("f")
+	f, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
 	if err != nil {
 		t.Fatal(err)
 	} else if f == nil {
@@ -23,7 +23,7 @@ func TestDB_CreateFrameIfNotExists(t *testing.T) {
 	}
 
 	// Retrieve existing frame.
-	other, err := db.CreateFrameIfNotExists("f")
+	other, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
 	if err != nil {
 		t.Fatal(err)
 	} else if f != other {
@@ -41,7 +41,7 @@ func TestDB_DeleteFrame(t *testing.T) {
 	defer db.Close()
 
 	// Create frame.
-	if _, err := db.CreateFrameIfNotExists("f"); err != nil {
+	if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
 		t.Fatal(err)
 	}
 
diff --git a/executor.go b/executor.go
index c6c058244..5ac546805 100644
--- a/executor.go
+++ b/executor.go
@@ -127,12 +127,14 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call
 	// Attach bitmap attributes for Bitmap() calls.
 	bm, _ := other.(*Bitmap)
 	if c.Name == "Bitmap" {
-		id, _ := c.Args["id"].(uint64)
 		frame, _ := c.Args["frame"].(string)
 
 		fr := e.Index.Frame(db, frame)
 		if fr != nil {
-			attrs, err := fr.BitmapAttrStore().Attrs(id)
+			rowLabel := fr.RowLabel()
+			rowID, _ := c.Args[rowLabel].(uint64)
+
+			attrs, err := fr.BitmapAttrStore().Attrs(rowID)
 			if err != nil {
 				return nil, err
 			}
@@ -280,17 +282,27 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql
 }
 
 func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
-	id, _ := c.Args["id"].(uint64)
 	frame, _ := c.Args["frame"].(string)
 	if frame == "" {
 		frame = DefaultFrame
 	}
 
-	f := e.Index.Fragment(db, frame, slice)
+	f := e.Index.Frame(db, frame)
 	if f == nil {
+		return nil, ErrFrameNotFound
+	}
+	rowLabel := f.RowLabel()
+
+	rowID, ok := c.Args[rowLabel].(uint64)
+	if !ok {
+		return nil, fmt.Errorf("Bitmap() field required: %s", rowLabel)
+	}
+
+	frag := e.Index.Fragment(db, frame, slice)
+	if frag == nil {
 		return NewBitmap(), nil
 	}
-	return f.Bitmap(id), nil
+	return frag.Bitmap(rowID), nil
 }
 
 // executeIntersectSlice executes a intersect() call for a local slice.
@@ -314,7 +326,21 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.
 
 // executeRangeSlice executes a range() call for a local slice.
 func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
-	id, _ := c.Args["id"].(uint64)
+	// Parse frame, use default if unset.
+	frame, _ := c.Args["frame"].(string)
+	if frame == "" {
+		frame = DefaultFrame
+	}
+
+	// Retrieve base frame.
+	f := e.Index.Frame(db, frame)
+	if f == nil {
+		return nil, ErrFrameNotFound
+	}
+	rowLabel := f.RowLabel()
+
+	// Read row id.
+	rowID, _ := c.Args[rowLabel].(uint64)
 
 	// Parse start time.
 	startTimeStr, ok := c.Args["start"].(string)
@@ -336,18 +362,6 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call
 		return nil, errors.New("cannot parse Range() end time")
 	}
 
-	// Parse frame, use default if unset.
-	frame, _ := c.Args["frame"].(string)
-	if frame == "" {
-		frame = DefaultFrame
-	}
-
-	// Retrieve base frame.
-	f := e.Index.Frame(db, frame)
-	if f == nil {
-		return &Bitmap{}, nil
-	}
-
 	// If no quantum exists then return an empty bitmap.
 	q := f.TimeQuantum()
 	if q == "" {
@@ -361,7 +375,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call
 		if f == nil {
 			continue
 		}
-		bm = bm.Union(f.Bitmap(id))
+		bm = bm.Union(f.Bitmap(rowID))
 	}
 	return bm, nil
 }
@@ -430,27 +444,42 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call,
 		return false, errors.New("ClearBit() frame required")
 	}
 
-	id, ok := c.Args["id"].(uint64)
+	// Lookup column label.
+	d := e.Index.DB(db)
+	if d == nil {
+		return false, nil
+	}
+	columnLabel := d.ColumnLabel()
+
+	// Lookup row label.
+	f := e.Index.Frame(db, frame)
+	if f == nil {
+		return false, nil
+	}
+	rowLabel := f.RowLabel()
+
+	// Read row & column ids.
+	rowID, ok := c.Args[rowLabel].(uint64)
 	if !ok {
-		return false, errors.New("ClearBit() id required")
+		return false, fmt.Errorf("ClearBit() field required: %s", rowLabel)
 	}
 
-	profileID, ok := c.Args["profileID"].(uint64)
+	colID, ok := c.Args[columnLabel].(uint64)
 	if !ok {
-		return false, errors.New("ClearBit() profileID required")
+		return false, fmt.Errorf("ClearBit() field required: %s", columnLabel)
 	}
 
-	slice := profileID / SliceWidth
+	slice := colID / SliceWidth
 	ret := false
 	for _, node := range e.Cluster.FragmentNodes(db, slice) {
 		// Update locally if host matches.
 		if node.Host == e.Host {
-			f := e.Index.Fragment(db, frame, slice)
-			if f == nil {
+			frag := e.Index.Fragment(db, frame, slice)
+			if frag == nil {
 				return false, nil
 			}
 
-			val, err := f.ClearBit(id, profileID)
+			val, err := frag.ClearBit(rowID, colID)
 			if err != nil {
 				return false, err
 			} else if val {
@@ -477,17 +506,32 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call,
 func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (bool, error) {
 	frame, ok := c.Args["frame"].(string)
 	if !ok {
-		return false, errors.New("SetBit() frame required")
+		return false, errors.New("SetBit() field required: frame")
 	}
 
-	id, ok := c.Args["id"].(uint64)
-	if !ok {
-		return false, errors.New("SetBit() id required")
+	// Retrieve frame.
+	d := e.Index.DB(db)
+	if d == nil {
+		return false, ErrFrameNotFound
+	}
+	f := d.Frame(frame)
+	if f == nil {
+		return false, ErrFrameNotFound
 	}
 
-	profileID, ok := c.Args["profileID"].(uint64)
+	// Retrieve labels.
+	columnLabel := d.ColumnLabel()
+	rowLabel := f.RowLabel()
+
+	// Read fields using labels.
+	rowID, ok := c.Args[rowLabel].(uint64)
 	if !ok {
-		return false, errors.New("SetBit() profileID required")
+		return false, fmt.Errorf("SetBit() field required: %s", rowLabel)
+	}
+
+	colID, ok := c.Args[columnLabel].(uint64)
+	if !ok {
+		return false, fmt.Errorf("SetBit() field required: %s", columnLabel)
 	}
 
 	var timestamp *time.Time
@@ -500,17 +544,18 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op
 		timestamp = &t
 	}
 
-	slice := profileID / SliceWidth
+	slice := colID / SliceWidth
 	ret := false
 
 	for _, node := range e.Cluster.FragmentNodes(db, slice) {
 		// Update locally if host matches.
 		if node.Host == e.Host {
-			db, err := e.Index.CreateDBIfNotExists(db)
-			if err != nil {
-				return false, fmt.Errorf("db: %s", err)
+			d := e.Index.DB(db)
+			if d == nil {
+				return false, ErrDatabaseNotFound
 			}
-			val, err := db.SetBit(frame, id, profileID, timestamp)
+
+			val, err := d.SetBit(frame, rowID, colID, timestamp)
 			if err != nil {
 				return false, err
 			} else if val {
@@ -541,24 +586,26 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.
 		return errors.New("SetBitmapAttrs() frame required")
 	}
 
-	id, ok := c.Args["id"].(uint64)
+	// Retrieve frame.
+	frame := e.Index.Frame(db, frameName)
+	if frame == nil {
+		return ErrFrameNotFound
+	}
+	rowLabel := frame.RowLabel()
+
+	// Parse labels.
+	rowID, ok := c.Args[rowLabel].(uint64)
 	if !ok {
-		return errors.New("SetBitmapAttrs() id required")
+		return fmt.Errorf("SetBitmapAttrs() field required: %s", rowLabel)
 	}
 
 	// Copy args and remove reserved fields.
 	attrs := pql.CopyArgs(c.Args)
 	delete(attrs, "frame")
-	delete(attrs, "id")
-
-	// Retrieve frame.
-	frame, err := e.Index.CreateFrameIfNotExists(db, frameName)
-	if err != nil {
-		return err
-	}
+	delete(attrs, rowLabel)
 
 	// Set attributes.
-	if err := frame.BitmapAttrStore().SetAttrs(id, attrs); err != nil {
+	if err := frame.BitmapAttrStore().SetAttrs(rowID, attrs); err != nil {
 		return err
 	}
 
@@ -597,15 +644,22 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
 			return nil, errors.New("SetBitmapAttrs() frame required")
 		}
 
-		id, ok := c.Args["id"].(uint64)
+		// Retrieve frame.
+		f := e.Index.Frame(db, frame)
+		if f == nil {
+			return nil, ErrFrameNotFound
+		}
+		rowLabel := f.RowLabel()
+
+		rowID, ok := c.Args[rowLabel].(uint64)
 		if !ok {
-			return nil, errors.New("SetBitmapAttrs() id required")
+			return nil, fmt.Errorf("SetBitmapAttrs() field required: %s", rowLabel)
 		}
 
 		// Copy args and remove reserved fields.
 		attrs := pql.CopyArgs(c.Args)
 		delete(attrs, "frame")
-		delete(attrs, "id")
+		delete(attrs, rowLabel)
 
 		// Create frame group, if not exists.
 		frameMap := m[frame]
@@ -615,9 +669,9 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
 		}
 
 		// Set or merge attributes.
-		attr := frameMap[id]
+		attr := frameMap[rowID]
 		if attr == nil {
-			frameMap[id] = cloneAttrs(attrs)
+			frameMap[rowID] = cloneAttrs(attrs)
 		} else {
 			for k, v := range attrs {
 				attr[k] = v
@@ -628,9 +682,9 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
 	// Bulk insert attributes by frame.
 	for name, frameMap := range m {
 		// Retrieve frame.
-		frame, err := e.Index.CreateFrameIfNotExists(db, name)
-		if err != nil {
-			return nil, err
+		frame := e.Index.Frame(db, name)
+		if frame == nil {
+			return nil, ErrFrameNotFound
 		}
 
 		// Set attributes.
@@ -677,9 +731,9 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql
 	delete(attrs, "id")
 
 	// Retrieve database.
-	d, err := e.Index.CreateDBIfNotExists(db)
-	if err != nil {
-		return err
+	d := e.Index.DB(db)
+	if d == nil {
+		return ErrDatabaseNotFound
 	}
 
 	// Set attributes.
diff --git a/executor_test.go b/executor_test.go
index 2cea37d3d..39d7f03c3 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -141,6 +141,14 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) {
 	idx := MustOpenIndex()
 	defer idx.Close()
 
+	// Create frames.
+	db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
+	if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
+		t.Fatal(err)
+	} else if _, err := db.CreateFrameIfNotExists("XXX", pilosa.FrameOptions{}); err != nil {
+		t.Fatal(err)
+	}
+
 	// Set two fields on f/10.
 	// Also set fields on other bitmaps and frames to test isolation.
 	e := NewExecutor(idx.Index, NewCluster(1))
@@ -253,8 +261,16 @@ func TestExecutor_Execute_Range(t *testing.T) {
 	idx := MustOpenIndex()
 	defer idx.Close()
 
-	db := idx.MustCreateDBIfNotExists("d")
+	// Create database.
+	db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
 	db.SetTimeQuantum(pilosa.TimeQuantum("YMDH"))
+
+	// Create frame.
+	if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Set bits.
 	db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-31 00:00"))
 	db.MustSetBit("f", 1, 3, MustParseTimePtr("2000-01-01 00:00"))
 	db.MustSetBit("f", 1, 4, MustParseTimePtr("2000-01-02 00:00"))
@@ -370,6 +386,11 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
 	idx := MustOpenIndex()
 	defer idx.Close()
 
+	// Create frame.
+	if _, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
+		t.Fatal(err)
+	}
+
 	e := NewExecutor(idx.Index, c)
 	if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2)`), nil, nil); err != nil {
 		t.Fatal(err)
@@ -409,7 +430,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
 	// Create local executor data.
 	idx := MustOpenIndex()
 	defer idx.Close()
-	idx.CreateDBIfNotExists("d")
+	idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
 	oldQuantum := idx.DB("d").TimeQuantum()
 	defer func() {
 		// restore db quantum
@@ -418,6 +439,11 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
 	// need to set the quantum otherwise SetBit fails silently
 	idx.DB("d").SetTimeQuantum("Y")
 
+	// Create frame.
+	if _, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
+		t.Fatal(err)
+	}
+
 	e := NewExecutor(idx.Index, c)
 	if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2, timestamp="2016-12-11T10:09:07")`), nil, nil); err != nil {
 		t.Fatal(err)
diff --git a/frame.go b/frame.go
index 9cdb4c707..efbf5f13c 100644
--- a/frame.go
+++ b/frame.go
@@ -18,6 +18,11 @@ const (
 	FrameSuffixRank = ".n"
 )
 
+// Default frame settings.
+const (
+	DefaultRowLabel = "id"
+)
+
 // Frame represents a container for fragments.
 type Frame struct {
 	mu          sync.Mutex
@@ -34,6 +39,9 @@ type Frame struct {
 
 	stats StatsClient
 
+	// Label used for referring to a row.
+	rowLabel string
+
 	LogOutput io.Writer
 }
 
@@ -49,6 +57,8 @@ func NewFrame(path, db, name string) *Frame {
 
 		stats: NopStatsClient,
 
+		rowLabel: DefaultRowLabel,
+
 		LogOutput: ioutil.Discard,
 	}
 }
@@ -79,6 +89,43 @@ func (f *Frame) MaxSlice() uint64 {
 	return max
 }
 
+// SetRowLabel sets the row labels. Persists to meta file on update.
+func (f *Frame) SetRowLabel(v string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	// Ignore if no change occurred.
+	if v == "" || f.rowLabel == v {
+		return nil
+	}
+
+	// Persist meta data to disk on change.
+	f.rowLabel = v
+	if err := f.saveMeta(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// RowLabel returns the row label.
+func (f *Frame) RowLabel() string {
+	f.mu.Lock()
+	v := f.rowLabel
+	f.mu.Unlock()
+	return v
+}
+
+// Options returns all options for this frame.
+func (f *Frame) Options() FrameOptions {
+	f.mu.Lock()
+	opt := FrameOptions{
+		RowLabel: f.rowLabel,
+	}
+	f.mu.Unlock()
+	return opt
+}
+
 // Open opens and initializes the frame.
 func (f *Frame) Open() error {
 	if err := func() error {
@@ -153,6 +200,7 @@ func (f *Frame) loadMeta() error {
 	buf, err := ioutil.ReadFile(filepath.Join(f.path, "meta"))
 	if os.IsNotExist(err) {
 		f.timeQuantum = ""
+		f.rowLabel = DefaultRowLabel
 		return nil
 	} else if err != nil {
 		return err
@@ -164,6 +212,7 @@ func (f *Frame) loadMeta() error {
 
 	// Copy metadata fields.
 	f.timeQuantum = TimeQuantum(pb.TimeQuantum)
+	f.rowLabel = pb.RowLabel
 
 	return nil
 }
@@ -171,7 +220,10 @@ func (f *Frame) loadMeta() error {
 // saveMeta writes meta data for the frame.
 func (f *Frame) saveMeta() error {
 	// Marshal metadata.
-	buf, err := proto.Marshal(&internal.Frame{TimeQuantum: string(f.timeQuantum)})
+	buf, err := proto.Marshal(&internal.Frame{
+		TimeQuantum: string(f.timeQuantum),
+		RowLabel:    f.rowLabel,
+	})
 	if err != nil {
 		return err
 	}
@@ -318,3 +370,8 @@ type frameInfoSlice []*FrameInfo
 func (p frameInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p frameInfoSlice) Len() int           { return len(p) }
 func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
+
+// FrameOptions represents options to set when initializing a frame.
+type FrameOptions struct {
+	RowLabel string `json:"rowLabel,omitempty"`
+}
diff --git a/handler.go b/handler.go
index 110b3892f..381cfeab7 100644
--- a/handler.go
+++ b/handler.go
@@ -108,6 +108,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	case "/db":
 		switch r.Method {
+		case "POST":
+			h.handlePostDB(w, r)
 		case "DELETE":
 			h.handleDeleteDB(w, r)
 		default:
@@ -129,6 +131,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	case "/frame":
 		switch r.Method {
+		case "POST":
+			h.handlePostFrame(w, r)
 		case "DELETE":
 			h.handleDeleteFrame(w, r)
 		default:
@@ -299,6 +303,38 @@ type sliceMaxResponse struct {
 	MaxSlices map[string]uint64 `json:"MaxSlices"`
 }
 
+// handlePostDB handles POST /db request.
+func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) {
+	// Decode request.
+	var req postDBRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	// Create database.
+	_, err := h.Index.CreateDB(req.DB, req.Options)
+	if err == ErrDatabaseExists {
+		http.Error(w, err.Error(), http.StatusConflict)
+		return
+	} else if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// Encode response.
+	if err := json.NewEncoder(w).Encode(postDBResponse{}); err != nil {
+		h.logger().Printf("response encoding error: %s", err)
+	}
+}
+
+type postDBRequest struct {
+	DB      string    `json:"db"`
+	Options DBOptions `json:"options"`
+}
+
+type postDBResponse struct{}
+
 // handleDeleteDB handles DELETE /db request.
 func (h *Handler) handleDeleteDB(w http.ResponseWriter, r *http.Request) {
 	// Decode request.
@@ -343,9 +379,9 @@ func (h *Handler) handlePatchDBTimeQuantum(w http.ResponseWriter, r *http.Reques
 	}
 
 	// Retrieve database by name.
-	db, err := h.Index.CreateDBIfNotExists(req.DB)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
+	db := h.Index.DB(req.DB)
+	if db == nil {
+		http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound)
 		return
 	}
 
@@ -378,9 +414,9 @@ func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Retrieve database from index.
-	db, err := h.Index.CreateDBIfNotExists(req.DB)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
+	db := h.Index.DB(req.DB)
+	if db == nil {
+		http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound)
 		return
 	}
 
@@ -424,6 +460,46 @@ type postDBAttrDiffResponse struct {
 	Attrs map[uint64]map[string]interface{} `json:"attrs"`
 }
 
+// handlePostFrame handles POST /frame request.
+func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) {
+	// Decode request.
+	var req postFrameRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	// Find database.
+	db := h.Index.DB(req.DB)
+	if db == nil {
+		http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound)
+		return
+	}
+
+	// Create frame.
+	_, err := db.CreateFrame(req.Frame, req.Options)
+	if err == ErrFrameExists {
+		http.Error(w, err.Error(), http.StatusConflict)
+		return
+	} else if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// Encode response.
+	if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil {
+		h.logger().Printf("response encoding error: %s", err)
+	}
+}
+
+type postFrameRequest struct {
+	DB      string       `json:"db"`
+	Frame   string       `json:"frame"`
+	Options FrameOptions `json:"options"`
+}
+
+type postFrameResponse struct{}
+
 // handleDeleteFrame handles DELETE /frame request.
 func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) {
 	// Decode request.
@@ -478,9 +554,9 @@ func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Req
 	}
 
 	// Retrieve database by name.
-	f, err := h.Index.CreateFrameIfNotExists(req.DB, req.Frame)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
+	f := h.Index.Frame(req.DB, req.Frame)
+	if f == nil {
+		http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
 		return
 	}
 
@@ -514,9 +590,9 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request
 	}
 
 	// Retrieve database from index.
-	f, err := h.Index.CreateFrameIfNotExists(req.DB, req.Frame)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
+	f := h.Index.Frame(req.DB, req.Frame)
+	if f == nil {
+		http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
 		return
 	}
 
@@ -714,10 +790,10 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
 
 	// Find the correct fragment.
 	h.logger().Println("importing:", req.DB, req.Frame, req.Slice)
-	db, err := h.Index.CreateDBIfNotExists(req.DB)
-	if err != nil {
-		h.logger().Printf("fragment error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, err)
-		http.Error(w, "fragment error", http.StatusInternalServerError)
+	db := h.Index.DB(req.DB)
+	if db == nil {
+		h.logger().Printf("fragment error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, ErrDatabaseNotFound.Error())
+		http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound)
 		return
 	}
 
@@ -725,6 +801,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
 	err = db.Import(req.Frame, req.BitmapIDs, req.ProfileIDs, timestamps)
 	if err != nil {
 		h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ProfileIDs), err)
+		return
 	}
 
 	// Marshal response object.
@@ -847,15 +924,22 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	// Retrieve fragment from index.
-	f, err := h.Index.CreateFragmentIfNotExists(q.Get("db"), q.Get("frame"), slice)
+	// Retrieve frame.
+	f := h.Index.Frame(q.Get("db"), q.Get("frame"))
+	if f == nil {
+		http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
+		return
+	}
+
+	// Retrieve fragment from frame.
+	frag, err := f.CreateFragmentIfNotExists(slice)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
 	// Read fragment in from request body.
-	if _, err := f.ReadFrom(r.Body); err != nil {
+	if _, err := frag.ReadFrom(r.Body); err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
@@ -971,8 +1055,15 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
 			continue
 		}
 
+		// Retrieve frame.
+		f := h.Index.Frame(db, frame)
+		if f == nil {
+			http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
+			return
+		}
+
 		// Otherwise retrieve the local fragment.
-		f, err := h.Index.CreateFragmentIfNotExists(db, frame, slice)
+		frag, err := f.CreateFragmentIfNotExists(slice)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
@@ -990,7 +1081,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
 		// Restore to local frame and always close reader.
 		if err := func() error {
 			defer rd.Close()
-			if _, err := f.ReadFrom(rd); err != nil {
+			if _, err := frag.ReadFrom(rd); err != nil {
 				return err
 			}
 			return nil
diff --git a/handler_test.go b/handler_test.go
index 49fa2160f..3a3ad3047 100644
--- a/handler_test.go
+++ b/handler_test.go
@@ -33,13 +33,17 @@ func TestHandler_NotFound(t *testing.T) {
 func TestHandler_Schema(t *testing.T) {
 	idx := MustOpenIndex()
 	defer idx.Close()
-	if _, err := idx.CreateFrameIfNotExists("d0", "f1"); err != nil {
+
+	d0 := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{})
+	d1 := idx.MustCreateDBIfNotExists("d1", pilosa.DBOptions{})
+
+	if _, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
 		t.Fatal(err)
 	}
-	if _, err := idx.CreateFrameIfNotExists("d1", "f0"); err != nil {
+	if _, err := d1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
 		t.Fatal(err)
 	}
-	if _, err := idx.CreateFrameIfNotExists("d0", "f0"); err != nil {
+	if _, err := d0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
 		t.Fatal(err)
 	}
 
@@ -210,7 +214,7 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) {
 	defer idx.Close()
 
 	// Create database and set profile attributes.
-	db, err := idx.CreateDBIfNotExists("d")
+	db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
 	if err != nil {
 		t.Fatal(err)
 	} else if err := db.ProfileAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil {
@@ -275,7 +279,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
 	defer idx.Close()
 
 	// Create database and set profile attributes.
-	db, err := idx.CreateDBIfNotExists("d")
+	db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
 	if err != nil {
 		t.Fatal(err)
 	} else if err := db.ProfileAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil {
@@ -451,7 +455,7 @@ func TestHandler_DB_Delete(t *testing.T) {
 	defer s.Close()
 
 	// Create database.
-	if _, err := idx.CreateDBIfNotExists("d"); err != nil {
+	if _, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}); err != nil {
 		t.Fatal(err)
 	}
 
@@ -481,7 +485,8 @@ func TestHandler_DB_Delete(t *testing.T) {
 func TestHandler_DeleteFrame(t *testing.T) {
 	idx := MustOpenIndex()
 	defer idx.Close()
-	if _, err := idx.CreateFrameIfNotExists("d0", "f1"); err != nil {
+	d0 := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{})
+	if _, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
 		t.Fatal(err)
 	}
 
@@ -502,6 +507,7 @@ func TestHandler_DeleteFrame(t *testing.T) {
 func TestHandler_SetDBTimeQuantum(t *testing.T) {
 	idx := MustOpenIndex()
 	defer idx.Close()
+	idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{})
 
 	h := NewHandler()
 	h.Index = idx.Index
@@ -521,6 +527,11 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) {
 	idx := MustOpenIndex()
 	defer idx.Close()
 
+	// Create frame.
+	if _, err := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil {
+		t.Fatal(err)
+	}
+
 	h := NewHandler()
 	h.Index = idx.Index
 	w := httptest.NewRecorder()
@@ -544,7 +555,7 @@ func TestHandler_DB_AttrStore_Diff(t *testing.T) {
 	defer s.Close()
 
 	// Set attributes on the database.
-	db, err := idx.CreateDBIfNotExists("d")
+	db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -593,7 +604,8 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
 	defer s.Close()
 
 	// Set attributes on the database.
-	f, err := idx.CreateFrameIfNotExists("d", "f")
+	d := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
+	f, err := d.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -657,6 +669,11 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
 		t.Fatalf("unexpected backup status code: %d", resp.StatusCode)
 	}
 
+	// Create frame.
+	if _, err := idx.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil {
+		t.Fatal(err)
+	}
+
 	// Restore backup to slice x/y/0.
 	if resp, err := http.Post(s.URL+"/fragment/data?db=x&frame=y&slice=0", "application/octet-stream", resp.Body); err != nil {
 		t.Fatal(err)
diff --git a/index.go b/index.go
index 12e3ca9bb..7885d87dd 100644
--- a/index.go
+++ b/index.go
@@ -156,15 +156,33 @@ func (i *Index) DBs() []*DB {
 	return a
 }
 
-// CreateDBIfNotExists returns a database by name.
-// The database is created if it does not already exist.
-func (i *Index) CreateDBIfNotExists(name string) (*DB, error) {
+// CreateDB creates a database.
+func (i *Index) CreateDB(name string, opt DBOptions) (*DB, error) {
 	i.mu.Lock()
 	defer i.mu.Unlock()
-	return i.createDBIfNotExists(name)
+
+	// Ensure db doesn't already exist.
+	if i.dbs[name] != nil {
+		return nil, ErrDatabaseExists
+	}
+	return i.createDB(name, opt)
 }
 
-func (i *Index) createDBIfNotExists(name string) (*DB, error) {
+// CreateDBIfNotExists returns a database by name.
+// The database is created if it does not already exist.
+func (i *Index) CreateDBIfNotExists(name string, opt DBOptions) (*DB, error) {
+	i.mu.Lock()
+	defer i.mu.Unlock()
+
+	// Find frame in cache first.
+	if db := i.dbs[name]; db != nil {
+		return db, nil
+	}
+
+	return i.createDB(name, opt)
+}
+
+func (i *Index) createDB(name string, opt DBOptions) (*DB, error) {
 	if name == "" {
 		return nil, errors.New("database name required")
 	}
@@ -179,6 +197,10 @@ func (i *Index) createDBIfNotExists(name string) (*DB, error) {
 	if err := db.Open(); err != nil {
 		return nil, err
 	}
+
+	// Update options.
+	db.SetColumnLabel(opt.ColumnLabel)
+
 	i.dbs[db.Name()] = db
 
 	i.Stats.Count("dbN", 1)
@@ -231,16 +253,6 @@ func (i *Index) Frame(db, name string) *Frame {
 	return d.Frame(name)
 }
 
-// CreateFrameIfNotExists returns the frame for a database & name.
-// The frame is created if it doesn't already exist.
-func (i *Index) CreateFrameIfNotExists(db, name string) (*Frame, error) {
-	d, err := i.CreateDBIfNotExists(db)
-	if err != nil {
-		return nil, err
-	}
-	return d.CreateFrameIfNotExists(name)
-}
-
 // Fragment returns the fragment for a database, frame & slice.
 func (i *Index) Fragment(db, frame string, slice uint64) *Fragment {
 	f := i.Frame(db, frame)
@@ -250,16 +262,6 @@ func (i *Index) Fragment(db, frame string, slice uint64) *Fragment {
 	return f.Fragment(slice)
 }
 
-// CreateFragmentIfNotExists returns the fragment for a database, frame & slice.
-// The fragment is created if it doesn't already exist.
-func (i *Index) CreateFragmentIfNotExists(db, frame string, slice uint64) (*Fragment, error) {
-	f, err := i.CreateFrameIfNotExists(db, frame)
-	if err != nil {
-		return nil, err
-	}
-	return f.CreateFragmentIfNotExists(slice)
-}
-
 // monitorCacheFlush periodically flushes all fragment caches sequentially.
 // This is run in a goroutine.
 func (i *Index) monitorCacheFlush() {
@@ -434,7 +436,9 @@ func (s *IndexSyncer) syncFrame(db, name string) error {
 		// Retrieve attributes from differing blocks.
 		// Skip update and recomputation if no attributes have changed.
 		m, err := client.BitmapAttrDiff(context.Background(), db, name, blks)
-		if err != nil {
+		if err == ErrFrameNotFound {
+			continue // frame not created remotely yet, skip
+		} else if err != nil {
 			return err
 		} else if len(m) == 0 {
 			continue
@@ -457,15 +461,21 @@ func (s *IndexSyncer) syncFrame(db, name string) error {
 
 // syncFragment synchronizes a fragment with the rest of the cluster.
 func (s *IndexSyncer) syncFragment(db, frame string, slice uint64) error {
+	// Retrieve local frame.
+	f := s.Index.Frame(db, frame)
+	if f == nil {
+		return ErrFrameNotFound
+	}
+
 	// Ensure fragment exists locally.
-	f, err := s.Index.CreateFragmentIfNotExists(db, frame, slice)
+	frag, err := f.CreateFragmentIfNotExists(slice)
 	if err != nil {
 		return err
 	}
 
 	// Sync fragments together.
 	fs := FragmentSyncer{
-		Fragment: f,
+		Fragment: frag,
 		Host:     s.Host,
 		Cluster:  s.Cluster,
 		Closing:  s.Closing,
diff --git a/index_test.go b/index_test.go
index 4707139d7..dace25fcc 100644
--- a/index_test.go
+++ b/index_test.go
@@ -72,6 +72,13 @@ func TestIndexSyncer_SyncIndex(t *testing.T) {
 	cluster.Nodes[0].Host = "localhost:0"
 	cluster.Nodes[1].Host = MustParseURLHost(s.URL)
 
+	// Create frames on nodes.
+	for _, idx := range []*Index{idx0, idx1} {
+		idx.MustCreateFrameIfNotExists("d", "f")
+		idx.MustCreateFrameIfNotExists("d", "f0")
+		idx.MustCreateFrameIfNotExists("y", "z")
+	}
+
 	// Set data on the local index.
 	f := idx0.MustCreateFragmentIfNotExists("d", "f", 0)
 	if _, err := f.SetBit(0, 10); err != nil {
@@ -191,8 +198,8 @@ func (i *Index) Close() error {
 }
 
 // MustCreateDBIfNotExists returns a given db. Panic on error.
-func (i *Index) MustCreateDBIfNotExists(db string) *DB {
-	d, err := i.Index.CreateDBIfNotExists(db)
+func (i *Index) MustCreateDBIfNotExists(db string, opt pilosa.DBOptions) *DB {
+	d, err := i.Index.CreateDBIfNotExists(db, opt)
 	if err != nil {
 		panic(err)
 	}
@@ -201,7 +208,7 @@ func (i *Index) MustCreateDBIfNotExists(db string) *DB {
 
 // MustCreateFrameIfNotExists returns a given frame. Panic on error.
 func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame {
-	f, err := i.Index.CreateFrameIfNotExists(db, frame)
+	f, err := i.MustCreateDBIfNotExists(db, pilosa.DBOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
 	if err != nil {
 		panic(err)
 	}
@@ -210,9 +217,14 @@ func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame {
 
 // MustCreateFragmentIfNotExists returns a given fragment. Panic on error.
 func (i *Index) MustCreateFragmentIfNotExists(db, frame string, slice uint64) *Fragment {
-	f, err := i.Index.CreateFragmentIfNotExists(db, frame, slice)
+	d := i.MustCreateDBIfNotExists(db, pilosa.DBOptions{})
+	f, err := d.CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
 	if err != nil {
 		panic(err)
 	}
-	return &Fragment{Fragment: f}
+	frag, err := f.CreateFragmentIfNotExists(slice)
+	if err != nil {
+		panic(err)
+	}
+	return &Fragment{Fragment: frag}
 }
diff --git a/internal/internal.pb.go b/internal/internal.pb.go
index 6239845ed..f0d8caac0 100644
--- a/internal/internal.pb.go
+++ b/internal/internal.pb.go
@@ -48,6 +48,7 @@ const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
 
 type DB struct {
 	TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
+	ColumnLabel string `protobuf:"bytes,2,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"`
 }
 
 func (m *DB) Reset()                    { *m = DB{} }
@@ -57,6 +58,7 @@ func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0
 
 type Frame struct {
 	TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
+	RowLabel    string `protobuf:"bytes,2,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"`
 }
 
 func (m *Frame) Reset()                    { *m = Frame{} }
@@ -323,6 +325,12 @@ func (m *DB) MarshalTo(dAtA []byte) (int, error) {
 		i = encodeVarintInternal(dAtA, i, uint64(len(m.TimeQuantum)))
 		i += copy(dAtA[i:], m.TimeQuantum)
 	}
+	if len(m.ColumnLabel) > 0 {
+		dAtA[i] = 0x12
+		i++
+		i = encodeVarintInternal(dAtA, i, uint64(len(m.ColumnLabel)))
+		i += copy(dAtA[i:], m.ColumnLabel)
+	}
 	return i, nil
 }
 
@@ -347,6 +355,12 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) {
 		i = encodeVarintInternal(dAtA, i, uint64(len(m.TimeQuantum)))
 		i += copy(dAtA[i:], m.TimeQuantum)
 	}
+	if len(m.RowLabel) > 0 {
+		dAtA[i] = 0x12
+		i++
+		i = encodeVarintInternal(dAtA, i, uint64(len(m.RowLabel)))
+		i += copy(dAtA[i:], m.RowLabel)
+	}
 	return i, nil
 }
 
@@ -1055,6 +1069,10 @@ func (m *DB) Size() (n int) {
 	if l > 0 {
 		n += 1 + l + sovInternal(uint64(l))
 	}
+	l = len(m.ColumnLabel)
+	if l > 0 {
+		n += 1 + l + sovInternal(uint64(l))
+	}
 	return n
 }
 
@@ -1065,6 +1083,10 @@ func (m *Frame) Size() (n int) {
 	if l > 0 {
 		n += 1 + l + sovInternal(uint64(l))
 	}
+	l = len(m.RowLabel)
+	if l > 0 {
+		n += 1 + l + sovInternal(uint64(l))
+	}
 	return n
 }
 
@@ -1425,6 +1447,35 @@ func (m *DB) Unmarshal(dAtA []byte) error {
 			}
 			m.TimeQuantum = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ColumnLabel", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowInternal
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthInternal
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.ColumnLabel = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipInternal(dAtA[iNdEx:])
@@ -1504,6 +1555,35 @@ func (m *Frame) Unmarshal(dAtA []byte) error {
 			}
 			m.TimeQuantum = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field RowLabel", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowInternal
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthInternal
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.RowLabel = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipInternal(dAtA[iNdEx:])
@@ -3831,49 +3911,51 @@ var (
 func init() { proto.RegisterFile("internal.proto", fileDescriptorInternal) }
 
 var fileDescriptorInternal = []byte{
-	// 696 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6a, 0x14, 0x41,
-	0x14, 0xb5, 0xa6, 0xbb, 0xe7, 0x71, 0x27, 0x19, 0x26, 0xc5, 0x28, 0x4d, 0x90, 0xa1, 0x29, 0x54,
-	0x5a, 0xc1, 0x04, 0xe2, 0x46, 0x44, 0x10, 0x3b, 0x13, 0x71, 0x90, 0x84, 0xa4, 0x12, 0xdd, 0xb9,
-	0x68, 0x93, 0x32, 0x69, 0xd2, 0x2f, 0xab, 0xab, 0xc5, 0x59, 0xba, 0x70, 0xe3, 0x17, 0x88, 0x7e,
-	0x81, 0x7f, 0xe2, 0xd2, 0x4f, 0x90, 0xf8, 0x23, 0x52, 0x8f, 0x7e, 0x44, 0x31, 0x66, 0xe1, 0xae,
-	0xef, 0xb9, 0x75, 0x6f, 0x9d, 0x73, 0x1f, 0xd5, 0x30, 0x8a, 0x52, 0xc1, 0x78, 0x1a, 0xc6, 0x6b,
-	0x39, 0xcf, 0x44, 0x86, 0xfb, 0x95, 0x4d, 0x6e, 0x41, 0x67, 0x16, 0x60, 0x0f, 0x86, 0x07, 0x51,
-	0xc2, 0xf6, 0xca, 0x30, 0x15, 0x65, 0xe2, 0x22, 0x0f, 0xf9, 0x03, 0xda, 0x86, 0xc8, 0x6d, 0x70,
-	0x9e, 0xf0, 0x30, 0x61, 0x97, 0x38, 0x1a, 0x40, 0x37, 0x88, 0x44, 0x12, 0xe6, 0x18, 0x83, 0x1d,
-	0x44, 0xa2, 0x70, 0x91, 0x67, 0xf9, 0x36, 0x55, 0xdf, 0xf8, 0x06, 0x38, 0x8f, 0x85, 0xe0, 0x85,
-	0xdb, 0xf1, 0x2c, 0x7f, 0xb8, 0x31, 0x5a, 0xab, 0xa9, 0x49, 0x98, 0x6a, 0x27, 0x59, 0x03, 0x7b,
-	0x37, 0x8c, 0x38, 0x1e, 0x83, 0xf5, 0x8c, 0x2d, 0xd4, 0x2d, 0x36, 0x95, 0x9f, 0x78, 0x02, 0xce,
-	0x66, 0x56, 0xa6, 0xc2, 0xed, 0x28, 0x4c, 0x1b, 0xe4, 0x25, 0x58, 0x41, 0x24, 0xf0, 0x2a, 0xf4,
-	0xf5, 0xd5, 0xf3, 0x99, 0x89, 0xa9, 0x6d, 0x7c, 0x1d, 0x06, 0xbb, 0x3c, 0x7b, 0x1d, 0xc5, 0x6c,
-	0x3e, 0x33, 0xc1, 0x0d, 0x20, 0xbd, 0x52, 0x43, 0x21, 0xc2, 0x24, 0x77, 0x2d, 0x0f, 0xf9, 0x16,
-	0x6d, 0x00, 0xf2, 0x08, 0x7a, 0xe6, 0x28, 0x1e, 0x41, 0xa7, 0x4e, 0xde, 0x99, 0xcf, 0x2e, 0xa9,
-	0xe7, 0x23, 0x02, 0x5b, 0x7e, 0xb5, 0x05, 0x0d, 0xb4, 0x20, 0x0c, 0xf6, 0xc1, 0x22, 0x67, 0x86,
-	0x92, 0xfa, 0x96, 0x45, 0xde, 0x17, 0x3c, 0x4a, 0x8f, 0x5f, 0x84, 0x71, 0xc9, 0x14, 0x9f, 0x01,
-	0x6d, 0x43, 0x92, 0xef, 0xf3, 0x28, 0x15, 0xda, 0x6f, 0x6b, 0x35, 0x35, 0x20, 0xbd, 0x41, 0x96,
-	0xc5, 0xda, 0xeb, 0x78, 0xc8, 0xef, 0xd3, 0x06, 0x20, 0xeb, 0xd0, 0x93, 0x5c, 0xb6, 0xc3, 0xbc,
-	0x61, 0x8f, 0x2e, 0x62, 0xff, 0x19, 0xc1, 0xd2, 0x5e, 0xc9, 0xf8, 0x82, 0xb2, 0x37, 0x25, 0x2b,
-	0x84, 0x2c, 0xc2, 0x2c, 0x30, 0x22, 0xe4, 0xfc, 0x4c, 0xc0, 0x51, 0x7e, 0x25, 0x62, 0x40, 0xb5,
-	0x81, 0xaf, 0x41, 0x77, 0x3f, 0x8e, 0x0e, 0x59, 0xe1, 0x5a, 0x6a, 0x00, 0x8c, 0x25, 0xbb, 0x64,
-	0xaa, 0x59, 0x28, 0xea, 0x7d, 0x5a, 0xdb, 0xd8, 0x85, 0x5e, 0x35, 0x5a, 0x8e, 0xca, 0x55, 0x99,
-	0x32, 0x1b, 0x65, 0x49, 0x26, 0x98, 0xdb, 0x55, 0x31, 0xc6, 0x22, 0xef, 0x11, 0x2c, 0x1b, 0x72,
-	0x45, 0x9e, 0xa5, 0x05, 0x93, 0x35, 0xde, 0xe2, 0xbc, 0xaa, 0xf1, 0x16, 0xe7, 0x78, 0x1d, 0x7a,
-	0x94, 0x15, 0x65, 0x2c, 0xaa, 0x36, 0x5d, 0x6d, 0x84, 0x56, 0xb1, 0x65, 0x2c, 0x68, 0x75, 0x0a,
-	0xdf, 0x6d, 0x51, 0xb4, 0x54, 0xc4, 0x4a, 0x13, 0x61, 0x3c, 0x0d, 0x6b, 0xf2, 0x01, 0xc1, 0xb0,
-	0x95, 0x07, 0xfb, 0xd5, 0x0a, 0x28, 0x12, 0xc3, 0x8d, 0x71, 0x13, 0xac, 0x71, 0x5a, 0xad, 0xc8,
-	0x12, 0xa0, 0x1d, 0xd3, 0x7a, 0xb4, 0x23, 0xdb, 0x21, 0xc7, 0xbe, 0xba, 0xb3, 0xd5, 0x0e, 0x09,
-	0x53, 0xed, 0x94, 0x35, 0xda, 0x3c, 0x09, 0xd3, 0x63, 0x76, 0x64, 0xca, 0x57, 0x99, 0xe4, 0x2b,
-	0x82, 0xe5, 0x79, 0x92, 0x67, 0x5c, 0x5c, 0xd0, 0x29, 0xb5, 0xc7, 0x55, 0xa7, 0xf4, 0x52, 0x4f,
-	0xc0, 0x51, 0xbd, 0x51, 0x93, 0x66, 0x53, 0x6d, 0xa8, 0x29, 0x32, 0xdb, 0x23, 0x1b, 0x25, 0x5b,
-	0xd8, 0x00, 0x78, 0x0a, 0x50, 0xaf, 0x4f, 0xe1, 0x3a, 0xca, 0xdd, 0x42, 0xa4, 0xbf, 0x5e, 0xa0,
-	0xc2, 0xed, 0x7a, 0x96, 0x6f, 0xd1, 0x16, 0x42, 0x08, 0x8c, 0x2a, 0xaa, 0x7f, 0xeb, 0x1b, 0x39,
-	0x82, 0x71, 0x10, 0x67, 0x87, 0xa7, 0xb3, 0x50, 0x84, 0xff, 0x43, 0xd1, 0x04, 0x1c, 0x95, 0xcf,
-	0x6c, 0x8c, 0x36, 0xc8, 0x1e, 0xac, 0xb4, 0x6e, 0x31, 0x64, 0xce, 0x89, 0x47, 0x17, 0x8b, 0xef,
-	0xfc, 0x2e, 0x9e, 0xdc, 0x04, 0x67, 0x33, 0x3c, 0x3c, 0xf9, 0x47, 0x1a, 0xf2, 0x05, 0xc1, 0xca,
-	0x76, 0xf8, 0x4e, 0xef, 0x45, 0x7d, 0xf5, 0x53, 0x18, 0xd4, 0xa0, 0x59, 0xcc, 0x3b, 0xcd, 0x24,
-	0xfc, 0x71, 0xbe, 0x41, 0xb6, 0x52, 0xc1, 0x17, 0xb4, 0x09, 0x5e, 0x7d, 0x08, 0xa3, 0xf3, 0x4e,
-	0x59, 0xe3, 0xd3, 0xe6, 0xfd, 0x39, 0xd5, 0x0f, 0xea, 0x5b, 0xf5, 0x4e, 0x98, 0x07, 0x55, 0x19,
-	0x0f, 0x3a, 0xf7, 0x51, 0x30, 0xfe, 0x76, 0x36, 0x45, 0xdf, 0xcf, 0xa6, 0xe8, 0xc7, 0xd9, 0x14,
-	0x7d, 0xfa, 0x39, 0xbd, 0xf2, 0xaa, 0xab, 0x7e, 0x1f, 0xf7, 0x7e, 0x05, 0x00, 0x00, 0xff, 0xff,
-	0x04, 0xa0, 0xe3, 0x0c, 0x50, 0x06, 0x00, 0x00,
+	// 721 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6a, 0x14, 0x4d,
+	0x14, 0xfe, 0x6b, 0xba, 0x7b, 0x2e, 0x67, 0x92, 0x61, 0x52, 0xcc, 0xff, 0xd3, 0x84, 0x9f, 0x61,
+	0x28, 0x14, 0x06, 0xc1, 0x04, 0xe2, 0x46, 0x44, 0x10, 0x7b, 0x66, 0x24, 0x83, 0x26, 0x24, 0x95,
+	0xe8, 0xce, 0x45, 0x25, 0x29, 0x93, 0x26, 0x7d, 0x19, 0xab, 0xab, 0xd5, 0x59, 0xba, 0x70, 0xe3,
+	0x13, 0x88, 0x3e, 0x81, 0x6f, 0xe2, 0xd2, 0x47, 0x90, 0xf8, 0x22, 0x52, 0x97, 0xbe, 0x44, 0x31,
+	0x66, 0xe1, 0xae, 0xbf, 0xef, 0xd4, 0x39, 0xf5, 0x9d, 0x5b, 0x35, 0xf4, 0xc2, 0x44, 0x72, 0x91,
+	0xb0, 0x68, 0x63, 0x21, 0x52, 0x99, 0xe2, 0x76, 0x81, 0xc9, 0x36, 0x34, 0xa6, 0x01, 0x1e, 0x41,
+	0xf7, 0x30, 0x8c, 0xf9, 0x7e, 0xce, 0x12, 0x99, 0xc7, 0x3e, 0x1a, 0xa1, 0x71, 0x87, 0xd6, 0x29,
+	0x75, 0x62, 0x92, 0x46, 0x79, 0x9c, 0x3c, 0x61, 0x47, 0x3c, 0xf2, 0x1b, 0xe6, 0x44, 0x8d, 0x22,
+	0x33, 0xf0, 0x1e, 0x09, 0x16, 0xf3, 0x6b, 0x04, 0x5b, 0x87, 0x36, 0x4d, 0x5f, 0xd7, 0x23, 0x95,
+	0x98, 0x04, 0xd0, 0x0c, 0x42, 0x19, 0xb3, 0x05, 0xc6, 0xe0, 0x06, 0xa1, 0xcc, 0x7c, 0x34, 0x72,
+	0xc6, 0x2e, 0xd5, 0xdf, 0xf8, 0x06, 0x78, 0x0f, 0xa5, 0x14, 0x99, 0xdf, 0x18, 0x39, 0xe3, 0xee,
+	0x56, 0x6f, 0xa3, 0x4c, 0x4c, 0xd1, 0xd4, 0x18, 0xc9, 0x06, 0xb8, 0x7b, 0x2c, 0x14, 0xb8, 0x0f,
+	0xce, 0x63, 0xbe, 0xd4, 0x0a, 0x5c, 0xaa, 0x3e, 0xf1, 0x00, 0xbc, 0x49, 0x9a, 0x27, 0x52, 0x5f,
+	0xeb, 0x52, 0x03, 0xc8, 0x73, 0x70, 0x82, 0x50, 0x2a, 0x59, 0xe6, 0xea, 0xf9, 0xd4, 0xfa, 0x94,
+	0x18, 0xff, 0x0f, 0x9d, 0x3d, 0x91, 0xbe, 0x08, 0x23, 0x3e, 0x9f, 0x5a, 0xe7, 0x8a, 0x50, 0x56,
+	0x95, 0x5f, 0x26, 0x59, 0xbc, 0xf0, 0x9d, 0x11, 0x1a, 0x3b, 0xb4, 0x22, 0xc8, 0x03, 0x68, 0xd9,
+	0xa3, 0xb8, 0x07, 0x8d, 0x32, 0x78, 0x63, 0x3e, 0xbd, 0x66, 0x3e, 0xef, 0x11, 0xb8, 0xea, 0xab,
+	0x9e, 0x50, 0xc7, 0x24, 0x84, 0xc1, 0x3d, 0x5c, 0x2e, 0xb8, 0x95, 0xa4, 0xbf, 0x55, 0x03, 0x0e,
+	0xa4, 0x08, 0x93, 0xd3, 0x67, 0x2c, 0xca, 0xb9, 0xd6, 0xd3, 0xa1, 0x75, 0x4a, 0xe9, 0x7d, 0x1a,
+	0x26, 0xd2, 0xd8, 0x5d, 0x93, 0x4d, 0x49, 0x28, 0x6b, 0x90, 0xa6, 0x91, 0xb1, 0x7a, 0x23, 0x34,
+	0x6e, 0xd3, 0x8a, 0x20, 0x9b, 0xd0, 0x52, 0x5a, 0x76, 0xd8, 0xa2, 0x52, 0x8f, 0xae, 0x52, 0xff,
+	0x11, 0xc1, 0xca, 0x7e, 0xce, 0xc5, 0x92, 0xf2, 0x97, 0x39, 0xcf, 0xa4, 0x2a, 0xc2, 0x34, 0xb0,
+	0x49, 0xa8, 0xe9, 0x1b, 0x80, 0xa7, 0xed, 0x76, 0x16, 0x0c, 0xc0, 0xff, 0x41, 0xf3, 0x20, 0x0a,
+	0x8f, 0x79, 0xe6, 0x3b, 0x7a, 0x00, 0x2c, 0x52, 0x5d, 0xb2, 0xd5, 0xcc, 0xb4, 0xf4, 0x36, 0x2d,
+	0x31, 0xf6, 0xa1, 0x55, 0x8c, 0x9d, 0xa7, 0x63, 0x15, 0x50, 0x45, 0xa3, 0x3c, 0x4e, 0x25, 0xf7,
+	0x9b, 0xda, 0xc7, 0x22, 0xf2, 0x16, 0xc1, 0xaa, 0x15, 0x97, 0x2d, 0xd2, 0x24, 0xe3, 0xaa, 0xc6,
+	0x33, 0x21, 0x8a, 0x1a, 0xcf, 0x84, 0xc0, 0x9b, 0xd0, 0xa2, 0x3c, 0xcb, 0x23, 0x59, 0xb4, 0xe9,
+	0xdf, 0x2a, 0xd1, 0xc2, 0x37, 0x8f, 0x24, 0x2d, 0x4e, 0xe1, 0xdb, 0x35, 0x89, 0x8e, 0xf6, 0x58,
+	0xab, 0x3c, 0xac, 0xa5, 0x52, 0x4d, 0xde, 0x21, 0xe8, 0xd6, 0xe2, 0xe0, 0x71, 0xb1, 0x02, 0x5a,
+	0x44, 0x77, 0xab, 0x5f, 0x39, 0x1b, 0x9e, 0x16, 0x2b, 0xb2, 0x02, 0x68, 0xd7, 0xb6, 0x1e, 0xed,
+	0xaa, 0x76, 0xa8, 0xb1, 0x2f, 0xee, 0xac, 0xb5, 0x43, 0xd1, 0xd4, 0x18, 0x55, 0x8d, 0x26, 0x67,
+	0x2c, 0x39, 0xe5, 0x27, 0xb6, 0x7c, 0x05, 0x24, 0x9f, 0x11, 0xac, 0xce, 0xe3, 0x45, 0x2a, 0xe4,
+	0x15, 0x9d, 0xd2, 0x3b, 0x5e, 0x74, 0xca, 0x2c, 0xfc, 0x00, 0x3c, 0xdd, 0x1b, 0x3d, 0x69, 0x2e,
+	0x35, 0x40, 0x4f, 0x91, 0xdd, 0x1e, 0xd5, 0x28, 0xd5, 0xc2, 0x8a, 0xc0, 0x43, 0x80, 0x72, 0x7d,
+	0x32, 0xdf, 0xd3, 0xe6, 0x1a, 0xa3, 0xec, 0xe5, 0x02, 0x65, 0x7e, 0x73, 0xe4, 0x8c, 0x1d, 0x5a,
+	0x63, 0x08, 0x81, 0x5e, 0x21, 0xf5, 0x77, 0x7d, 0x23, 0x27, 0xd0, 0x0f, 0xa2, 0xf4, 0xf8, 0x7c,
+	0xca, 0x24, 0xfb, 0x1b, 0x19, 0x0d, 0xc0, 0xd3, 0xf1, 0xec, 0xc6, 0x18, 0x40, 0xf6, 0x61, 0xad,
+	0x76, 0x8b, 0x15, 0x73, 0x29, 0x79, 0x74, 0x75, 0xf2, 0x8d, 0x9f, 0x93, 0x27, 0x37, 0xc1, 0x9b,
+	0xb0, 0xe3, 0xb3, 0x3f, 0x84, 0x21, 0x9f, 0x10, 0xac, 0xed, 0xb0, 0x37, 0x66, 0x2f, 0xca, 0xab,
+	0xb7, 0xa1, 0x53, 0x92, 0x76, 0x31, 0x6f, 0x55, 0x93, 0xf0, 0xcb, 0xf9, 0x8a, 0x99, 0x25, 0x52,
+	0x2c, 0x69, 0xe5, 0xbc, 0x7e, 0x1f, 0x7a, 0x97, 0x8d, 0xaa, 0xc6, 0xe7, 0xd5, 0xfb, 0x73, 0x6e,
+	0x1e, 0xd4, 0x57, 0xfa, 0x9d, 0xb0, 0x0f, 0xaa, 0x06, 0xf7, 0x1a, 0x77, 0x51, 0xd0, 0xff, 0x72,
+	0x31, 0x44, 0x5f, 0x2f, 0x86, 0xe8, 0xdb, 0xc5, 0x10, 0x7d, 0xf8, 0x3e, 0xfc, 0xe7, 0xa8, 0xa9,
+	0x7f, 0x3e, 0x77, 0x7e, 0x04, 0x00, 0x00, 0xff, 0xff, 0xf3, 0x47, 0xed, 0xb3, 0x8e, 0x06, 0x00,
+	0x00,
 }
diff --git a/internal/internal.proto b/internal/internal.proto
index eaffe7307..8acbe0bab 100644
--- a/internal/internal.proto
+++ b/internal/internal.proto
@@ -4,10 +4,12 @@ package internal;
 
 message DB {
 	string TimeQuantum = 1;
+	string ColumnLabel = 2;
 }
 
 message Frame {
 	string TimeQuantum = 1;
+	string RowLabel = 2;
 }
 
 message Bitmap {
diff --git a/pilosa.go b/pilosa.go
index dff78ebb1..a136cb56f 100644
--- a/pilosa.go
+++ b/pilosa.go
@@ -6,21 +6,20 @@ import (
 	"github.com/pilosa/pilosa/internal"
 )
 
+// System errors.
 var (
-	// ErrHostRequired is returned when excuting a remote operation without a host.
 	ErrHostRequired = errors.New("host required")
 
-	// ErrDatabaseRequired is returned when no database is specified.
 	ErrDatabaseRequired = errors.New("database required")
+	ErrDatabaseExists   = errors.New("database already exists")
+	ErrDatabaseNotFound = errors.New("database not found")
 
-	// ErrFrameRequired is returned when no frame is specified.
 	ErrFrameRequired = errors.New("frame required")
+	ErrFrameExists   = errors.New("frame already exists")
+	ErrFrameNotFound = errors.New("frame not found")
 
-	// ErrFragmentNotFound is returned when a fragment does not exist.
 	ErrFragmentNotFound = errors.New("fragment not found")
-
-	// ErrQueryRequired is returned when no query is specified.
-	ErrQueryRequired = errors.New("query required")
+	ErrQueryRequired    = errors.New("query required")
 )
 
 // Profile represents vertical column in a database.
diff --git a/server.go b/server.go
index b520381d7..57ec0e2b5 100644
--- a/server.go
+++ b/server.go
@@ -211,9 +211,9 @@ func (s *Server) monitorMaxSlices() {
 							localdb.SetRemoteMaxSlice(newmax)
 						}
 					} else {
-						d, err := s.Index.CreateDBIfNotExists(db)
-						if err != nil {
-							s.logger().Printf("Failed to create DB locally: %s", db)
+						d := s.Index.DB(db)
+						if d == nil {
+							s.logger().Printf("Local DB not found: %s", db)
 							return
 						}
 						oldmaxslices[db] = newmax

From 7d30b72d432aa9578fee8dcfac0e8760dec2bef2 Mon Sep 17 00:00:00 2001
From: Linh Vo <lvo@umbel.com>
Date: Tue, 21 Feb 2017 10:09:49 -0600
Subject: [PATCH 03/37] add restriction database

---
 client.go             | 10 ++++++++++
 cmd/pilosactl/main.go |  2 ++
 db.go                 |  5 ++++-
 fragment.go           |  6 +++---
 frame.go              |  9 +++++++--
 pilosa.go             | 16 ++++++++++++++++
 pilosactl/import.go   | 10 ++++++++++
 7 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/client.go b/client.go
index 5c24920ff..bc6338053 100644
--- a/client.go
+++ b/client.go
@@ -149,6 +149,11 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire
 		return nil, ErrQueryRequired
 	}
 
+	er := ValidateName(db)
+	if er != nil {
+		return nil, ErrName
+	}
+
 	// Encode query request.
 	buf, err := proto.Marshal(&internal.QueryRequest{
 		DB:     db,
@@ -205,6 +210,11 @@ func (c *Client) ExecutePQL(ctx context.Context, db, query string) (interface{},
 		}.Encode(),
 	}
 
+	er := ValidateName(db)
+	if er != nil {
+		return nil, ErrName
+	}
+
 	req, err := http.NewRequest("POST", u.String(), bytes.NewReader([]byte(query)))
 	if err != nil {
 		return nil, err
diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go
index 331459c31..7c25d62cb 100644
--- a/cmd/pilosactl/main.go
+++ b/cmd/pilosactl/main.go
@@ -290,6 +290,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
 	logger := log.New(cmd.Stderr, "", log.LstdFlags)
 
 	// Validate arguments.
+	fmt.Print("1213224")
 	if cmd.Database == "" {
 		return pilosa.ErrDatabaseRequired
 	} else if cmd.Frame == "" {
@@ -959,6 +960,7 @@ func (cmd *BenchCommand) Run(ctx context.Context) error {
 
 // runSetBit executes a benchmark of random SetBit() operations.
 func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) error {
+	fmt.Print("Nodb")
 	if cmd.N == 0 {
 		return errors.New("operation count required")
 	} else if cmd.Database == "" {
diff --git a/db.go b/db.go
index 2a4f1d6e0..9d4c6532a 100644
--- a/db.go
+++ b/db.go
@@ -274,7 +274,10 @@ func (db *DB) createFrameIfNotExists(name string) (*Frame, error) {
 }
 
 func (db *DB) newFrame(path, name string) *Frame {
-	f := NewFrame(path, db.name, name)
+	f, err := NewFrame(path, db.name, name)
+	if err != nil {
+		return nil
+	}
 	f.LogOutput = db.LogOutput
 	f.stats = db.stats.WithTags(fmt.Sprintf("frame:%s", name))
 	return f
diff --git a/fragment.go b/fragment.go
index 1181f1739..9bc542f93 100644
--- a/fragment.go
+++ b/fragment.go
@@ -213,11 +213,11 @@ func (f *Fragment) openCache() error {
 	// Determine cache type from frame name.
 	if strings.HasSuffix(f.frame, FrameSuffixRank) {
 		c := NewRankCache()
-		c.ThresholdLength = 50000
-		c.ThresholdIndex = 45000
+		c.ThresholdLength = 500000
+		c.ThresholdIndex = 450000
 		f.cache = c
 	} else {
-		f.cache = NewLRUCache(50000)
+		f.cache = NewLRUCache(500000)
 	}
 
 	// Read cache data from disk.
diff --git a/frame.go b/frame.go
index 9cdb4c707..1971900e4 100644
--- a/frame.go
+++ b/frame.go
@@ -38,7 +38,12 @@ type Frame struct {
 }
 
 // NewFrame returns a new instance of frame.
-func NewFrame(path, db, name string) *Frame {
+func NewFrame(path, db, name string) (*Frame, error) {
+	err := ValidateName(db)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Frame{
 		path: path,
 		db:   db,
@@ -50,7 +55,7 @@ func NewFrame(path, db, name string) *Frame {
 		stats: NopStatsClient,
 
 		LogOutput: ioutil.Discard,
-	}
+	}, nil
 }
 
 // Name returns the name the frame was initialized with.
diff --git a/pilosa.go b/pilosa.go
index dff78ebb1..1a8bba840 100644
--- a/pilosa.go
+++ b/pilosa.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 
 	"github.com/pilosa/pilosa/internal"
+	"regexp"
 )
 
 var (
@@ -16,6 +17,9 @@ var (
 	// ErrFrameRequired is returned when no frame is specified.
 	ErrFrameRequired = errors.New("frame required")
 
+	// ErrFrameRequired is returned when no frame is specified.
+	ErrName = errors.New("name restricted to [a-z0-9_-.]")
+
 	// ErrFragmentNotFound is returned when a fragment does not exist.
 	ErrFragmentNotFound = errors.New("fragment not found")
 
@@ -75,3 +79,15 @@ func decodeProfile(pb *internal.Profile) *Profile {
 
 // TimeFormat is the go-style time format used to parse string dates.
 const TimeFormat = "2006-01-02T15:04"
+
+
+// Restrict name using regex
+func ValidateName(name string) error {
+	expr := regexp.MustCompile(`^([a-z0-9._-]{2,64}$)`)
+	validName := expr.FindStringSubmatchIndex(name)
+
+	if len(validName) == 0 {
+		return ErrName
+	}
+	return nil
+}
\ No newline at end of file
diff --git a/pilosactl/import.go b/pilosactl/import.go
index 9d4ad63f4..40a735c61 100644
--- a/pilosactl/import.go
+++ b/pilosactl/import.go
@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa"
+	"regexp"
 )
 
 // ImportCommand represents a command for bulk importing data.
@@ -104,7 +105,16 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 	} else if len(cmd.Paths) == 0 {
 		return errors.New("path required")
 	}
+	// Restrict frame name and database name with regex
+	dbError := pilosa.ValidateName(cmd.Database)
+	if dbError != nil {
+		return dbError
+	}
 
+	frameError := pilosa.ValidateName(cmd.Frame)
+	if frameError != nil {
+		return frameError
+	}
 	// Create a client to the server.
 	client, err := pilosa.NewClient(cmd.Host)
 	if err != nil {

From c2afa6491da31abdafba54a995a32307fa7c6c58 Mon Sep 17 00:00:00 2001
From: Yuce Tekol <yucetekol@gmail.com>
Date: Tue, 21 Feb 2017 23:08:41 +0300
Subject: [PATCH 04/37] Updated README for docker usage

---
 README.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/README.md b/README.md
index 2079010ec..19221e003 100644
--- a/README.md
+++ b/README.md
@@ -65,6 +65,20 @@ The remaining configuration options should be the same on every node in the clus
 
 `[[cluster.node]]`: specifies each node within the cluster
 
+## Docker
+
+You can create a Pilosa container using `make docker` or equivalently:
+```
+docker build -t pilosa:latest .
+```
+
+You can run a temporary container using:
+```
+docker run -it --rm --name pilosa -p 15000:15000 pilosa:latest
+```
+
+When you click `Ctrl+C` to stop the container, the container and the data in the container will be erased. You can leave out `--rm` flag to keep the data in the container. See [Docker documentation](https://docs.docker.com) for other options.
+
 ## Usage
 
 You can interact with Pilosa via HTTP requests to the host:port on which you have Pilosa running.

From 7a7869aea66e7e5d2d4889b81aac3d3d33e933d3 Mon Sep 17 00:00:00 2001
From: Cody Soyland <cody@soyland.org>
Date: Thu, 23 Feb 2017 09:35:27 -0600
Subject: [PATCH 05/37] Add build status to README

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 2079010ec..bd6aacbe2 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,7 @@
 
 Pilosa is a bitmap index database.
 
+[![Build Status](https://travis-ci.com/pilosa/pilosa.svg?token=Peb4jvQ3kLbjUEhpU5aR&branch=master)](https://travis-ci.com/pilosa/pilosa)
 
 ## Getting Started
 

From 6f645b9db5649d85abf0a511227e20f72bb0e1c5 Mon Sep 17 00:00:00 2001
From: Yuce Tekol <yucetekol@gmail.com>
Date: Fri, 24 Feb 2017 16:35:41 +0300
Subject: [PATCH 06/37] SetBit uses TimeFormat. Implements 315

---
 executor.go      | 2 +-
 executor_test.go | 4 ++--
 pql/ast.go       | 3 +--
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/executor.go b/executor.go
index c6c058244..19222cf51 100644
--- a/executor.go
+++ b/executor.go
@@ -493,7 +493,7 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op
 	var timestamp *time.Time
 	sTimestamp, ok := c.Args["timestamp"].(string)
 	if ok {
-		t, err := time.Parse("2006-01-02T15:04:05", sTimestamp)
+		t, err := time.Parse(TimeFormat, sTimestamp)
 		if err != nil {
 			return false, fmt.Errorf("invalid date: %s", sTimestamp)
 		}
diff --git a/executor_test.go b/executor_test.go
index 2cea37d3d..d51a9c996 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -399,7 +399,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
 	s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
 		if db != `d` {
 			t.Fatalf("unexpected db: %s", db)
-		} else if query.String() != `SetBit(frame="f", id=10, profileID=2, timestamp="2016-12-11T10:09:07")` {
+		} else if query.String() != `SetBit(frame="f", id=10, profileID=2, timestamp="2016-12-11T10:09")` {
 			t.Fatalf("unexpected query: %s", query.String())
 		}
 		remoteCalled = true
@@ -419,7 +419,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
 	idx.DB("d").SetTimeQuantum("Y")
 
 	e := NewExecutor(idx.Index, c)
-	if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2, timestamp="2016-12-11T10:09:07")`), nil, nil); err != nil {
+	if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
 		t.Fatal(err)
 	}
 
diff --git a/pql/ast.go b/pql/ast.go
index dec73ac2b..3168e5ec0 100644
--- a/pql/ast.go
+++ b/pql/ast.go
@@ -100,8 +100,7 @@ func (c *Call) String() string {
 		case []uint64:
 			fmt.Fprintf(&buf, "%v=%s", key, joinUint64Slice(v))
 		case time.Time:
-			layout := "2006-01-02T15:04:05"
-			fmt.Fprintf(&buf, "%v=\"%s\"", key, v.Format(layout))
+			fmt.Fprintf(&buf, "%v=\"%s\"", key, v.Format(TimeFormat))
 		default:
 			fmt.Fprintf(&buf, "%v=%v", key, v)
 		}

From 0c21c583fb74baf5143280e25dda445ef717a03b Mon Sep 17 00:00:00 2001
From: Cody Soyland <cody@soyland.org>
Date: Fri, 24 Feb 2017 09:08:49 -0600
Subject: [PATCH 07/37] Install glide from ppa before running Travis tests
 (#343)

---
 .travis.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index cea0d3d1a..56dec77f7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,6 +2,10 @@ language: go
 go:
   - 1.7.x
   - master
+before_install:
+  - sudo add-apt-repository ppa:masterminds/glide -y
+  - sudo apt-get update -q
+  - sudo apt-get install glide -y
 script: make test
 notifications:
   slack:

From 77372fe3cc3e505b0228cbece89555df875ab2e5 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Thu, 9 Feb 2017 16:51:45 -0600
Subject: [PATCH 08/37] remove creator/bagent/bspawn

---
 bench/LICENSE                      |  30 --
 bench/bench.go                     |  41 --
 bench/client.go                    |  67 ---
 bench/diagonal.go                  | 105 -----
 bench/doc.go                       |  39 --
 bench/errgroup.go                  |  67 ---
 bench/errgroup_test.go             | 176 --------
 bench/import.go                    | 203 ---------
 bench/import_test.go               | 136 ------
 bench/multidb.go                   |  97 ----
 bench/permutations.go              |  64 ---
 bench/prettify.go                  |  41 --
 bench/prettify_test.go             |  93 ----
 bench/query_test.go                |  20 -
 bench/random.go                    | 117 -----
 bench/sliceheight.go               | 151 -------
 bench/stats.go                     |  65 ---
 bench/store.go                     |  39 --
 bench/zipf.go                      | 191 --------
 build/build.go                     |  29 --
 cmd/pilosactl/import.json          |   9 -
 cmd/pilosactl/main.go              | 692 -----------------------------
 cmd/pilosactl/multidbspawn.json    |   8 -
 cmd/pilosactl/randSetAndQuery.json |  11 -
 cmd/pilosactl/randspawn.json       |   8 -
 cmd/pilosactl/slice-height.json    |   8 -
 cmd/pilosactl/spawn.json           |  14 -
 cmd/pilosactl/zipfspawn.json       |  12 -
 creator/creator.go                 |  11 -
 creator/local.go                   |  94 ----
 creator/remote.go                  | 187 --------
 31 files changed, 2825 deletions(-)
 delete mode 100644 bench/LICENSE
 delete mode 100644 bench/bench.go
 delete mode 100644 bench/client.go
 delete mode 100644 bench/diagonal.go
 delete mode 100644 bench/doc.go
 delete mode 100644 bench/errgroup.go
 delete mode 100644 bench/errgroup_test.go
 delete mode 100644 bench/import.go
 delete mode 100644 bench/import_test.go
 delete mode 100644 bench/multidb.go
 delete mode 100644 bench/permutations.go
 delete mode 100644 bench/prettify.go
 delete mode 100644 bench/prettify_test.go
 delete mode 100644 bench/query_test.go
 delete mode 100644 bench/random.go
 delete mode 100644 bench/sliceheight.go
 delete mode 100644 bench/stats.go
 delete mode 100644 bench/store.go
 delete mode 100644 bench/zipf.go
 delete mode 100644 build/build.go
 delete mode 100644 cmd/pilosactl/import.json
 delete mode 100644 cmd/pilosactl/multidbspawn.json
 delete mode 100644 cmd/pilosactl/randSetAndQuery.json
 delete mode 100644 cmd/pilosactl/randspawn.json
 delete mode 100644 cmd/pilosactl/slice-height.json
 delete mode 100644 cmd/pilosactl/spawn.json
 delete mode 100644 cmd/pilosactl/zipfspawn.json
 delete mode 100644 creator/creator.go
 delete mode 100644 creator/local.go
 delete mode 100644 creator/remote.go

diff --git a/bench/LICENSE b/bench/LICENSE
deleted file mode 100644
index e9de11fa5..000000000
--- a/bench/LICENSE
+++ /dev/null
@@ -1,30 +0,0 @@
-The following applies to the files errgroup.go and errgroup_test.go
-in the same directory this file resides:
-
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/bench/bench.go b/bench/bench.go
deleted file mode 100644
index 000909bde..000000000
--- a/bench/bench.go
+++ /dev/null
@@ -1,41 +0,0 @@
-package bench
-
-import "context"
-
-// Benchmark is an interface to guide the creation of new pilosa benchmarks or
-// benchmark components. It defines 2 methods, Init, and Run. These are separate
-// methods so that benchmark running code can time only the running of the
-// benchmark, and not any setup.
-type Benchmark interface {
-	// Init takes a list of hosts and an agent number. It is generally expected
-	// to set up a connection to pilosa using whatever client it chooses. The
-	// agentNum should be used to parameterize the benchmark's configuration if
-	// it is being run simultaneously on multiple "agents". E.G. the agentNum
-	// might be used to make a random seed different for each agent, or have
-	// each agent set a different set of bits. Init's doc string should document
-	// how the agentNum affects it. Every benchmark should have a 'Name' field
-	// set by init, which appears when the benchmark is marshalled to json as
-	// "name".
-	Init(hosts []string, agentNum int) error
-
-	// Run runs the benchmark. The return value of Run is kept generic so that
-	// any relevant statistics or metrics that may be specific to the benchmark
-	// in question can be reported. TODO guidelines for what gets included in
-	// results and what will get added by other stuff. Run does not need to
-	// report total run time in `results`, as that will be added by calling
-	// code.
-	Run(ctx context.Context) map[string]interface{}
-}
-
-// Command extends Benchmark by adding methods for configuring via command line flags and returning usage information.
-type Command interface {
-	Benchmark
-
-	// ConsumeFlags sets and parses flags, and then returns flagSet.Args(). This
-	// is so that multiple benchmarks can be specified at the command line.
-	ConsumeFlags(args []string) ([]string, error)
-
-	// Usage returns information on how to use this benchmark. The usage string
-	// should explain how the agent num affects the benchmark's operation.
-	Usage() string
-}
diff --git a/bench/client.go b/bench/client.go
deleted file mode 100644
index 101f62d4b..000000000
--- a/bench/client.go
+++ /dev/null
@@ -1,67 +0,0 @@
-package bench
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"github.com/pilosa/pilosa"
-)
-
-func firstHostClient(hosts []string) (*pilosa.Client, error) {
-	client, err := pilosa.NewClient(hosts[0])
-	if err != nil {
-		return nil, err
-	}
-	return client, nil
-}
-
-func roundRobinClient(hosts []string, agentNum int) (*pilosa.Client, error) {
-	clientNum := agentNum % len(hosts)
-	return firstHostClient(hosts[clientNum:])
-}
-
-// HasClient provides a reusable component for Benchmark implementations which
-// provides the Init method, a ClientType argument and a cli internal variable.
-type HasClient struct {
-	client      *pilosa.Client
-	ClientType  string `json:"client-type"`
-	ContentType string `json:"content-type"`
-}
-
-// Init for HasClient looks at the ClientType field and creates a pilosa client
-// either using the first host in the list of hosts or based on the agent
-// number mod len(hosts)
-func (h *HasClient) Init(hosts []string, agentNum int) error {
-	var err error
-	switch h.ClientType {
-	case "single":
-		h.client, err = firstHostClient(hosts)
-	case "round_robin":
-		h.client, err = roundRobinClient(hosts, agentNum)
-	default:
-		err = fmt.Errorf("Unsupported ClientType: %v", h.ClientType)
-	}
-	if err != nil {
-		return err
-	}
-
-	switch h.ContentType {
-	case "protobuf":
-		return nil
-	case "pql":
-		return nil
-	default:
-		return fmt.Errorf("Unsupported ContentType: %v", h.ContentType)
-
-	}
-}
-
-func (h *HasClient) ExecuteQuery(contentType, db, query string, ctx context.Context) (interface{}, error) {
-	if contentType == "protobuf" {
-		return h.client.ExecuteQuery(ctx, db, query, true)
-	} else if contentType == "pql" {
-		return h.client.ExecutePQL(ctx, db, query)
-	} else {
-		return nil, errors.New("unsupport content type")
-	}
-}
diff --git a/bench/diagonal.go b/bench/diagonal.go
deleted file mode 100644
index 393b732b3..000000000
--- a/bench/diagonal.go
+++ /dev/null
@@ -1,105 +0,0 @@
-package bench
-
-import (
-	"fmt"
-
-	"flag"
-	"io/ioutil"
-
-	"context"
-	"time"
-)
-
-// DiagonalSetBits sets bits with increasing profile id and bitmap id.
-type DiagonalSetBits struct {
-	HasClient
-	Name          string `json:"name"`
-	BaseBitmapID  int    `json:"base-bitmap-id"`
-	BaseProfileID int    `json:"base-profile-id"`
-	Iterations    int    `json:"iterations"`
-	DB            string `json:"db"`
-}
-
-// Init sets up the pilosa client and modifies the configured values based on
-// the agent num.
-func (b *DiagonalSetBits) Init(hosts []string, agentNum int) error {
-	b.Name = "diagonal-set-bits"
-	b.BaseBitmapID = b.BaseBitmapID + (agentNum * b.Iterations)
-	b.BaseProfileID = b.BaseProfileID + (agentNum * b.Iterations)
-	return b.HasClient.Init(hosts, agentNum)
-}
-
-// Usage returns the usage message to be printed.
-func (b *DiagonalSetBits) Usage() string {
-	return `
-diagonal-set-bits sets bits with increasing profile id and bitmap id.
-
-Agent num offsets both the base profile id and base bitmap id by the number of
-iterations, so that only bits on the main diagonal are set, and agents don't
-overlap at all.
-
-Usage: diagonal-set-bits [arguments]
-
-The following arguments are available:
-
-	-base-bitmap-id int
-		bits being set will all be greater than BaseBitmapID
-
-	-base-profile-id int
-		profile id num to start from
-
-	-iterations int
-		number of bits to set
-
-	-db string
-		pilosa db to use
-
-	-client-type string
-		Can be 'single' (all agents hitting one host) or 'round_robin'
-
-	-content-type string
-		protobuf or pql
-`[1:]
-}
-
-// ConsumeFlags parses all flags up to the next non flag argument (argument does
-// not start with "-" and isn't the value of a flag). It returns the remaining
-// args.
-func (b *DiagonalSetBits) ConsumeFlags(args []string) ([]string, error) {
-	fs := flag.NewFlagSet("DiagonalSetBits", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	fs.IntVar(&b.BaseBitmapID, "base-bitmap-id", 0, "")
-	fs.IntVar(&b.BaseProfileID, "base-profile-id", 0, "")
-	fs.IntVar(&b.Iterations, "iterations", 100, "")
-	fs.StringVar(&b.DB, "db", "benchdb", "")
-	fs.StringVar(&b.ClientType, "client-type", "single", "")
-	fs.StringVar(&b.ContentType, "content-type", "protobuf", "")
-
-	if err := fs.Parse(args); err != nil {
-		return nil, err
-	}
-	return fs.Args(), nil
-}
-
-// Run runs the DiagonalSetBits benchmark
-func (b *DiagonalSetBits) Run(ctx context.Context) map[string]interface{} {
-	results := make(map[string]interface{})
-	if b.client == nil {
-		results["error"] = fmt.Errorf("No client set for DiagonalSetBits")
-		return results
-	}
-	s := NewStats()
-	var start time.Time
-	for n := 0; n < b.Iterations; n++ {
-		query := fmt.Sprintf("SetBit(%d, 'frame.n', %d)", b.BaseBitmapID+n, b.BaseProfileID+n)
-		start = time.Now()
-		_, err := b.client.ExecuteQuery(ctx, b.DB, query, true)
-		if err != nil {
-			results["error"] = err
-			return results
-		}
-		s.Add(time.Now().Sub(start))
-	}
-	AddToResults(s, results)
-	return results
-}
diff --git a/bench/doc.go b/bench/doc.go
deleted file mode 100644
index 6554ffe80..000000000
--- a/bench/doc.go
+++ /dev/null
@@ -1,39 +0,0 @@
-// bench contains benchmarks and common utilities useful to benchmarks
-//
-// In order to write new benchmarks, one must satisfy the Benchmark and Command
-// interfaces in bench.go. In order to use the benchmark from pilosactl, it
-// needs to be wired in in two places. The first is BagentCommand.ParseFlags,
-// where a case statement needs to be added, and the second is just adding the
-// benchmark to the BagentCommand.Usage usage string.
-//
-// When writing a new benchmark, there are a few things to keep in mind other
-// than just implementing the interface:
-//
-// 1. The benchmark should modify its own configuration in its Init method based
-// on the agentNum it is given. How it modifies is specific to the benchmark,
-// but the idea is that it should make sense to call the benchmark with the same
-// configuration, but multiple different agent numbers, and it should do useful
-// work each time (i.e. not just setting the same bits, or running the same
-// queries).
-//
-// 2. The Init method should do everything that needs to be done to get the
-// benchmark to a runnable state - all code in run should be the stuff that we
-// actually want to time.
-//
-// 3. The Run method does not need to report the total runtime - that is collected
-// by calling code.
-//
-// 4. Usage should follow the format in other benchmarks, and explain how the
-// benchmark uses agentNum to modify its behavior
-//
-//
-// Files:
-//
-// 1. client.go contains pilosa client code which is shared by many benchmarks
-//
-// 2. errgroup.go contains the ErrGroup implementation copied from golang.org/x/
-// so as not to pull in a bunch of useless deps.
-//
-// 3. stats.go contains useful code for gathering stats about a series of timed
-// operations.
-package bench
diff --git a/bench/errgroup.go b/bench/errgroup.go
deleted file mode 100644
index ac0bb4819..000000000
--- a/bench/errgroup.go
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package errgroup provides synchronization, error propagation, and Context
-// cancelation for groups of goroutines working on subtasks of a common task.
-package bench
-
-import (
-	"sync"
-
-	"context"
-)
-
-// An ErrGroup is a collection of goroutines working on subtasks that are part of
-// the same overall task.
-//
-// A zero ErrGroup is valid and does not cancel on error.
-type ErrGroup struct {
-	cancel func()
-
-	wg sync.WaitGroup
-
-	errOnce sync.Once
-	err     error
-}
-
-// WithContext returns a new ErrGroup and an associated Context derived from ctx.
-//
-// The derived Context is canceled the first time a function passed to Go
-// returns a non-nil error or the first time Wait returns, whichever occurs
-// first.
-func WithContext(ctx context.Context) (*ErrGroup, context.Context) {
-	ctx, cancel := context.WithCancel(ctx)
-	return &ErrGroup{cancel: cancel}, ctx
-}
-
-// Wait blocks until all function calls from the Go method have returned, then
-// returns the first non-nil error (if any) from them.
-func (g *ErrGroup) Wait() error {
-	g.wg.Wait()
-	if g.cancel != nil {
-		g.cancel()
-	}
-	return g.err
-}
-
-// Go calls the given function in a new goroutine.
-//
-// The first call to return a non-nil error cancels the group; its error will be
-// returned by Wait.
-func (g *ErrGroup) Go(f func() error) {
-	g.wg.Add(1)
-
-	go func() {
-		defer g.wg.Done()
-
-		if err := f(); err != nil {
-			g.errOnce.Do(func() {
-				g.err = err
-				if g.cancel != nil {
-					g.cancel()
-				}
-			})
-		}
-	}()
-}
diff --git a/bench/errgroup_test.go b/bench/errgroup_test.go
deleted file mode 100644
index 546cceb60..000000000
--- a/bench/errgroup_test.go
+++ /dev/null
@@ -1,176 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package bench_test
-
-import (
-	"errors"
-	"fmt"
-	"net/http"
-	"os"
-	"testing"
-
-	"context"
-	"github.com/pilosa/pilosa/bench"
-)
-
-var (
-	Web   = fakeSearch("web")
-	Image = fakeSearch("image")
-	Video = fakeSearch("video")
-)
-
-type Result string
-type Search func(ctx context.Context, query string) (Result, error)
-
-func fakeSearch(kind string) Search {
-	return func(_ context.Context, query string) (Result, error) {
-		return Result(fmt.Sprintf("%s result for %q", kind, query)), nil
-	}
-}
-
-// JustErrors illustrates the use of a ErrGroup in place of a sync.WaitGroup to
-// simplify goroutine counting and error handling. This example is derived from
-// the sync.WaitGroup example at https://golang.org/pkg/sync/#example_WaitGroup.
-func ExampleGroup_justErrors() {
-	var g bench.ErrGroup
-	var urls = []string{
-		"http://www.golang.org/",
-		"http://www.google.com/",
-		"http://www.somestupidname.com/",
-	}
-	for _, url := range urls {
-		// Launch a goroutine to fetch the URL.
-		url := url // https://golang.org/doc/faq#closures_and_goroutines
-		g.Go(func() error {
-			// Fetch the URL.
-			resp, err := http.Get(url)
-			if err == nil {
-				resp.Body.Close()
-			}
-			return err
-		})
-	}
-	// Wait for all HTTP fetches to complete.
-	if err := g.Wait(); err == nil {
-		fmt.Println("Successfully fetched all URLs.")
-	}
-}
-
-// Parallel illustrates the use of a ErrGroup for synchronizing a simple parallel
-// task: the "Google Search 2.0" function from
-// https://talks.golang.org/2012/concurrency.slide#46, augmented with a Context
-// and error-handling.
-func ExampleGroup_parallel() {
-	Google := func(ctx context.Context, query string) ([]Result, error) {
-		g, ctx := bench.WithContext(ctx)
-
-		searches := []Search{Web, Image, Video}
-		results := make([]Result, len(searches))
-		for i, search := range searches {
-			i, search := i, search // https://golang.org/doc/faq#closures_and_goroutines
-			g.Go(func() error {
-				result, err := search(ctx, query)
-				if err == nil {
-					results[i] = result
-				}
-				return err
-			})
-		}
-		if err := g.Wait(); err != nil {
-			return nil, err
-		}
-		return results, nil
-	}
-
-	results, err := Google(context.Background(), "golang")
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		return
-	}
-	for _, result := range results {
-		fmt.Println(result)
-	}
-
-	// Output:
-	// web result for "golang"
-	// image result for "golang"
-	// video result for "golang"
-}
-
-func TestZeroGroup(t *testing.T) {
-	err1 := errors.New("errgroup_test: 1")
-	err2 := errors.New("errgroup_test: 2")
-
-	cases := []struct {
-		errs []error
-	}{
-		{errs: []error{}},
-		{errs: []error{nil}},
-		{errs: []error{err1}},
-		{errs: []error{err1, nil}},
-		{errs: []error{err1, nil, err2}},
-	}
-
-	for _, tc := range cases {
-		var g bench.ErrGroup
-
-		var firstErr error
-		for i, err := range tc.errs {
-			err := err
-			g.Go(func() error { return err })
-
-			if firstErr == nil && err != nil {
-				firstErr = err
-			}
-
-			if gErr := g.Wait(); gErr != firstErr {
-				t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+
-					"g.Wait() = %v; want %v",
-					g, tc.errs[:i+1], err, firstErr)
-			}
-		}
-	}
-}
-
-func TestWithContext(t *testing.T) {
-	errDoom := errors.New("group_test: doomed")
-
-	cases := []struct {
-		errs []error
-		want error
-	}{
-		{want: nil},
-		{errs: []error{nil}, want: nil},
-		{errs: []error{errDoom}, want: errDoom},
-		{errs: []error{errDoom, nil}, want: errDoom},
-	}
-
-	for _, tc := range cases {
-		g, ctx := bench.WithContext(context.Background())
-
-		for _, err := range tc.errs {
-			err := err
-			g.Go(func() error { return err })
-		}
-
-		if err := g.Wait(); err != tc.want {
-			t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+
-				"g.Wait() = %v; want %v",
-				g, tc.errs, err, tc.want)
-		}
-
-		canceled := false
-		select {
-		case <-ctx.Done():
-			canceled = true
-		default:
-		}
-		if !canceled {
-			t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+
-				"ctx.Done() was not closed",
-				g, tc.errs)
-		}
-	}
-}
diff --git a/bench/import.go b/bench/import.go
deleted file mode 100644
index 2d88b23a3..000000000
--- a/bench/import.go
+++ /dev/null
@@ -1,203 +0,0 @@
-package bench
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"math/rand"
-
-	"sort"
-
-	"github.com/pilosa/pilosa/pilosactl"
-)
-
-// NewImport returns an Import Benchmark which pilosactl importer configured.
-func NewImport(stdin io.Reader, stdout, stderr io.Writer) *Import {
-	return &Import{
-		ImportCommand: pilosactl.NewImportCommand(stdin, stdout, stderr),
-	}
-}
-
-// Import sets bits with increasing profile id and bitmap id.
-type Import struct {
-	Name              string `json:"name"`
-	BaseBitmapID      int64  `json:"base-bitmap-id"`
-	MaxBitmapID       int64  `json:"max-bitmap-id"`
-	BaseProfileID     int64  `json:"base-profile-id"`
-	MaxProfileID      int64  `json:"max-profile-id"`
-	RandomBitmapOrder bool   `json:"random-bitmap-order"`
-	MinBitsPerMap     int64  `json:"min-bits-per-map"`
-	MaxBitsPerMap     int64  `json:"max-bits-per-map"`
-	AgentControls     string `json:"agent-controls"`
-	Seed              int64  `json:"seed"`
-	numbits           int
-
-	*pilosactl.ImportCommand
-}
-
-// Usage returns the usage message to be printed.
-func (b *Import) Usage() string {
-	return `
-import generates an import file and imports using pilosa's bulk import interface
-
-Agent num can have various effects - see -agent-controls flag.
-
-Usage: import [arguments]
-
-The following arguments are available:
-
-	-base-bitmap-id int
-		bits being set will all be greater than this
-
-	-max-bitmap-id int
-		bits being set will all be less than this
-
-	-base-profile-id int
-		profile id num to start from
-
-	-max-profile-id int
-		maximum profile id to generate
-
-	-random-bitmap-order
-		if this option is set, the import file will not be sorted by bitmap id
-
-	-min-bits-per-map int
-		minimum number of bits set per bitmap
-
-	-max-bits-per-map int
-		maximum number of bits set per bitmap
-
-	-agent-controls string
-		can be 'height', 'width', or empty (TODO or square?)- increasing
-		number of agents modulates bitmap id range, profile id range,
-		or just sets more bits in the same range.
-
-	-seed int
-		seed for RNG
-
-	-db string
-		pilosa db to use
-
-	-frame string
-		frame to import into
-`[1:]
-}
-
-// ConsumeFlags parses all flags up to the next non flag argument (argument does
-// not start with "-" and isn't the value of a flag). It returns the remaining
-// args.
-func (b *Import) ConsumeFlags(args []string) ([]string, error) {
-	fs := flag.NewFlagSet("Import", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	fs.Int64Var(&b.BaseBitmapID, "base-bitmap-id", 0, "")
-	fs.Int64Var(&b.MaxBitmapID, "max-bitmap-id", 1000, "")
-	fs.Int64Var(&b.BaseProfileID, "base-profile-id", 0, "")
-	fs.Int64Var(&b.MaxProfileID, "max-profile-id", 1000, "")
-	fs.BoolVar(&b.RandomBitmapOrder, "random-bitmap-order", false, "")
-	fs.Int64Var(&b.MinBitsPerMap, "min-bits-per-map", 0, "")
-	fs.Int64Var(&b.MaxBitsPerMap, "max-bits-per-map", 10, "")
-	fs.StringVar(&b.AgentControls, "agent-controls", "", "")
-	fs.Int64Var(&b.Seed, "seed", 0, "")
-	fs.StringVar(&b.Database, "db", "benchdb", "")
-	fs.StringVar(&b.Frame, "frame", "testframe", "")
-	fs.IntVar(&b.BufferSize, "buffer-size", 10000000, "")
-
-	if err := fs.Parse(args); err != nil {
-		return nil, err
-	}
-	return fs.Args(), nil
-}
-
-// Init generates import data based on the agent num and fields of 'b'.
-func (b *Import) Init(hosts []string, agentNum int) error {
-	if len(hosts) == 0 {
-		return fmt.Errorf("Need at least one host")
-	}
-	b.Name = "import"
-	b.Host = hosts[0]
-	// generate csv data
-	b.Seed = b.Seed + int64(agentNum)
-	switch b.AgentControls {
-	case "height":
-		numBitmapIDs := (b.MaxBitmapID - b.BaseBitmapID)
-		b.BaseBitmapID = b.BaseBitmapID + (numBitmapIDs * int64(agentNum))
-		b.MaxBitmapID = b.BaseBitmapID + numBitmapIDs
-	case "width":
-		numProfileIDs := (b.MaxProfileID - b.BaseProfileID)
-		b.BaseProfileID = b.BaseProfileID + (numProfileIDs * int64(agentNum))
-		b.MaxProfileID = b.BaseProfileID + numProfileIDs
-	case "":
-		break
-	default:
-		return fmt.Errorf("agent-controls: '%v' is not supported", b.AgentControls)
-	}
-	f, err := ioutil.TempFile("", "")
-	if err != nil {
-		return err
-	}
-	// set b.Paths)
-	num := GenerateImportCSV(f, b.BaseBitmapID, b.MaxBitmapID, b.BaseProfileID, b.MaxProfileID,
-		b.MinBitsPerMap, b.MaxBitsPerMap, b.Seed, b.RandomBitmapOrder)
-	b.numbits = num
-	// set b.Paths
-	b.Paths = []string{f.Name()}
-	return f.Close()
-}
-
-// Run runs the Import benchmark
-func (b *Import) Run(ctx context.Context) map[string]interface{} {
-	results := make(map[string]interface{})
-	results["numbits"] = b.numbits
-	results["db"] = b.Database
-	err := b.ImportCommand.Run(ctx)
-
-	if err != nil {
-		results["error"] = err.Error()
-	}
-	return results
-}
-
-// Int64Slice is a sortable slice of 64 bit signed ints
-type Int64Slice []int64
-
-func (s Int64Slice) Len() int           { return len(s) }
-func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] }
-func (s Int64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
-
-// GenerateImportCSV writes a generated csv to 'w' which is in the form pilosactl expects for imports.
-func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, minBitsPerMap, maxBitsPerMap, seed int64, randomOrder bool) int {
-	src := rand.NewSource(seed)
-	rng := rand.New(src)
-
-	var bitmapIDs []int
-	if randomOrder {
-		bitmapIDs = rng.Perm(int(maxBitmapID - baseBitmapID))
-	}
-	numrows := 0
-	profileIDs := make(Int64Slice, maxBitsPerMap)
-	for i := baseBitmapID; i < maxBitmapID; i++ {
-		var bitmapID int64
-		if randomOrder {
-			bitmapID = int64(bitmapIDs[i-baseBitmapID])
-		} else {
-			bitmapID = int64(i)
-		}
-
-		numBitsToSet := rng.Int63n(maxBitsPerMap-minBitsPerMap) + minBitsPerMap
-		numrows += int(numBitsToSet)
-		for j := int64(0); j < numBitsToSet; j++ {
-			profileIDs[j] = rng.Int63n(maxProfileID-baseProfileID) + baseProfileID
-		}
-		profIDs := profileIDs[:numBitsToSet]
-		if !randomOrder {
-			sort.Sort(profIDs)
-		}
-		for j := int64(0); j < numBitsToSet; j++ {
-			fmt.Fprintf(w, "%d,%d\n", bitmapID, profIDs[j])
-		}
-
-	}
-	return numrows
-}
diff --git a/bench/import_test.go b/bench/import_test.go
deleted file mode 100644
index 07074f125..000000000
--- a/bench/import_test.go
+++ /dev/null
@@ -1,136 +0,0 @@
-package bench_test
-
-import (
-	"bytes"
-	"testing"
-
-	"io/ioutil"
-
-	"os"
-
-	"github.com/pilosa/pilosa/bench"
-)
-
-func TestImportInit(t *testing.T) {
-	imp := bench.NewImport(os.Stdin, os.Stdout, os.Stderr)
-	imp.BaseBitmapID = 0
-	imp.MaxBitmapID = 10
-	imp.BaseProfileID = 0
-	imp.MaxProfileID = 10
-	imp.RandomBitmapOrder = false
-	imp.MinBitsPerMap = 2
-	imp.MaxBitsPerMap = 3
-	imp.AgentControls = "width"
-	imp.Seed = 0
-
-	imp.Init([]string{"blah"}, 2)
-	f, err := os.Open(imp.Paths[0])
-	if err != nil {
-		t.Fatalf("Couldn't open file: %v, err: %v", imp.Paths[0], err)
-	}
-	bytes, err := ioutil.ReadAll(f)
-	if err != nil {
-		t.Fatalf("error reading file: %v", err)
-	}
-
-	expected := `
-0,21
-0,22
-1,20
-1,22
-2,22
-2,26
-3,21
-3,23
-4,21
-4,22
-5,20
-5,28
-6,23
-6,27
-7,20
-7,20
-8,23
-8,29
-9,23
-9,29
-`[1:]
-
-	if string(bytes) != expected {
-		t.Fatalf("unexpected result: %v", string(bytes))
-	}
-}
-
-func TestGenerateImportCSVNonRand(t *testing.T) {
-	b := bytes.NewBuffer(make([]byte, 0))
-
-	bench.GenerateImportCSV(b, 0, 10, 20, 30, 2, 3, 2, false)
-
-	bytes, err := ioutil.ReadAll(b)
-	if err != nil {
-		t.Fatalf("Error reading buffer: %v", err)
-	}
-
-	expected := `
-0,21
-0,22
-1,20
-1,22
-2,22
-2,26
-3,21
-3,23
-4,21
-4,22
-5,20
-5,28
-6,23
-6,27
-7,20
-7,20
-8,23
-8,29
-9,23
-9,29
-`[1:]
-
-	if string(bytes) != expected {
-		t.Fatalf("unexpected value for generated csv: \n%v", string(bytes))
-	}
-}
-
-func TestGenerateImportCSVRand(t *testing.T) {
-	b := bytes.NewBuffer(make([]byte, 0))
-
-	bench.GenerateImportCSV(b, 0, 10, 21, 29, 1, 4, 0, true)
-
-	bytes, err := ioutil.ReadAll(b)
-	if err != nil {
-		t.Fatalf("Error reading buffer: %v", err)
-	}
-
-	expected := `
-8,25
-2,23
-3,22
-3,28
-3,28
-0,25
-0,23
-5,26
-5,23
-7,21
-7,23
-1,25
-6,23
-6,27
-6,25
-9,26
-4,22
-4,24
-`[1:]
-
-	if string(bytes) != expected {
-		t.Fatalf("unexpected value for generated csv: \n%v", string(bytes))
-	}
-}
diff --git a/bench/multidb.go b/bench/multidb.go
deleted file mode 100644
index aa275ac17..000000000
--- a/bench/multidb.go
+++ /dev/null
@@ -1,97 +0,0 @@
-package bench
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"strconv"
-	"time"
-)
-
-// MultiDBSetBits sets bits with increasing profile id and bitmap id.
-type MultiDBSetBits struct {
-	HasClient
-	Name          string `json:"name"`
-	BaseBitmapID  int    `json:"base-bitmap-id"`
-	BaseProfileID int    `json:"base-profile-id"`
-	Iterations    int    `json:"iterations"`
-	Database      string `json:"database"`
-}
-
-// Init sets up the db name based on the agentNum and sets up the pilosa client.
-func (b *MultiDBSetBits) Init(hosts []string, agentNum int) error {
-	b.Name = "multi-db-set-bits"
-	b.Database = b.Database + strconv.Itoa(agentNum)
-	return b.HasClient.Init(hosts, agentNum)
-}
-
-// Usage returns the usage message to be printed.
-func (b *MultiDBSetBits) Usage() string {
-	return `
-multi-db-set-bits sets bits with increasing profile id and bitmap id using a different DB for each agent.
-
-Agent num changes the database being written to.
-
-Usage: multi-db-set-bits [arguments]
-
-The following arguments are available:
-
-	-base-bitmap-id int
-		bits being set will all be greater than base-bitmap-id
-
-	-base-profile-id int
-		profile id num to start from
-
-	-iterations int
-		number of bits to set
-
-	-client-type string
-		Can be 'single' (all agents hitting one host) or 'round_robin'
-
-	-content-type string
-		protobuf or pql
-
-`[1:]
-}
-
-// ConsumeFlags parses all flags up to the next non flag argument (argument does
-// not start with "-" and isn't the value of a flag). It returns the remaining
-// args.
-func (b *MultiDBSetBits) ConsumeFlags(args []string) ([]string, error) {
-	fs := flag.NewFlagSet("MultiDBSetBits", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	fs.IntVar(&b.BaseBitmapID, "base-bitmap-id", 0, "")
-	fs.IntVar(&b.BaseProfileID, "base-profile-id", 0, "")
-	fs.IntVar(&b.Iterations, "iterations", 100, "")
-	fs.StringVar(&b.ClientType, "client-type", "single", "")
-	fs.StringVar(&b.ContentType, "content-type", "protobuf", "")
-
-	if err := fs.Parse(args); err != nil {
-		return nil, err
-	}
-	return fs.Args(), nil
-}
-
-// Run runs the MultiDBSetBits benchmark
-func (b *MultiDBSetBits) Run(ctx context.Context) map[string]interface{} {
-	results := make(map[string]interface{})
-	if b.client == nil {
-		results["error"] = fmt.Errorf("No client set for MultiDBSetBits")
-		return results
-	}
-	s := NewStats()
-	var start time.Time
-	for n := 0; n < b.Iterations; n++ {
-		query := fmt.Sprintf("SetBit(%d, 'frame.n', %d)", b.BaseBitmapID+n, b.BaseProfileID+n)
-		start = time.Now()
-		_, err := b.client.ExecuteQuery(ctx, b.Database, query, true)
-		if err != nil {
-			results["error"] = err
-			return results
-		}
-		s.Add(time.Now().Sub(start))
-	}
-	AddToResults(s, results)
-	return results
-}
diff --git a/bench/permutations.go b/bench/permutations.go
deleted file mode 100644
index cbe4162e4..000000000
--- a/bench/permutations.go
+++ /dev/null
@@ -1,64 +0,0 @@
-package bench
-
-// PermutationGenerator provides a way to pass integer IDs through a permutation
-// map that is pseudorandom but repeatable. This could be done with rand.Perm,
-// but that would require storing a [Iterations]int64 array, which we want to avoid
-// for large values of Iterations.
-// It works by using a Linear Congruence Generator (https://en.wikipedia.org/wiki/Linear_congruential_generator)
-// with modulus m = Iterations,
-// c = an arbitrary prime,
-// a = computed to ensure the full period.
-// relevant stackoverflow: http://cs.stackexchange.com/questions/29822/lazily-computing-a-random-permutation-of-the-positive-integers
-type PermutationGenerator struct {
-	a int64
-	c int64
-	m int64
-}
-
-func NewPermutationGenerator(m int64, seed int64) *PermutationGenerator {
-	// figure out 'a' and 'c', return PermutationGenerator
-	a := LCGmultiplierFromModulus(m, seed)
-	c := int64(22695479)
-	return &PermutationGenerator{a, c, m}
-}
-
-func (p *PermutationGenerator) Next(n int64) int64 {
-	// run one step of the LCG
-	return (n*p.a + p.c) % p.m
-}
-
-// LCG parameters must satisfy three conditions:
-// 1. m and c are relatively prime (satisfied for prime c != m)
-// 2. a-1 is divisible by all prime factors of m
-// 3. a-1 is divisible by 4 if m is divisible by 4
-// Additionally, a seed can be used to select between different permutations
-func LCGmultiplierFromModulus(m int64, seed int64) int64 {
-	factors := primeFactors(m)
-	product := int64(1)
-	for p := range factors {
-		// satisfy condition 2
-		product *= p
-	}
-
-	if m%4 == 0 {
-		// satisfy condition 3
-		product *= 2
-	}
-
-	return product*seed + 1
-}
-
-// Returns map of {integerFactor: count, ...}
-// This is a naive algorithm that will not work well for large prime n.
-func primeFactors(n int64) map[int64]int {
-	factors := make(map[int64]int)
-	for i := int64(2); i <= n; i++ {
-		div, mod := n/i, n%i
-		for mod == 0 {
-			factors[i] += 1
-			n = div
-			div, mod = n/i, n%i
-		}
-	}
-	return factors
-}
diff --git a/bench/prettify.go b/bench/prettify.go
deleted file mode 100644
index 1902d412b..000000000
--- a/bench/prettify.go
+++ /dev/null
@@ -1,41 +0,0 @@
-package bench
-
-import "time"
-
-// wrapper type to force human-readable JSON output
-type PrettyDuration time.Duration
-
-// MarshalJSON returns a nicely formatted duration, instead of it just being
-// treated like an int.
-func (d PrettyDuration) MarshalJSON() ([]byte, error) {
-	s := time.Duration(d).String()
-	return []byte("\"" + s + "\""), nil
-}
-
-// Recursively replaces elements of ugly types with their pretty wrappers
-func Prettify(m map[string]interface{}) map[string]interface{} {
-	newmap := make(map[string]interface{})
-	for k, v := range m {
-		switch v.(type) {
-		case map[string]interface{}:
-			newmap[k] = Prettify(v.(map[string]interface{}))
-		case []time.Duration:
-			newslice := make([]PrettyDuration, len(v.([]time.Duration)))
-			slice := v.([]time.Duration)
-			for n, e := range slice {
-				newslice[n] = PrettyDuration(e)
-			}
-			newmap[k] = newslice
-		case time.Duration:
-			newmap[k] = PrettyDuration(v.(time.Duration))
-		default:
-			if interv, ok := v.([]map[string]interface{}); ok {
-				for i, iv := range interv {
-					interv[i] = Prettify(iv)
-				}
-			}
-			newmap[k] = v
-		}
-	}
-	return newmap
-}
diff --git a/bench/prettify_test.go b/bench/prettify_test.go
deleted file mode 100644
index 20269d73e..000000000
--- a/bench/prettify_test.go
+++ /dev/null
@@ -1,93 +0,0 @@
-package bench_test
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/pilosa/pilosa/bench"
-)
-
-func prettyEncode(data map[string]interface{}) string {
-	pretty := bench.Prettify(data)
-	jsonString := new(bytes.Buffer)
-	enc := json.NewEncoder(jsonString)
-	enc.SetIndent("", "  ")
-	err := enc.Encode(pretty)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-	}
-
-	return jsonString.String()
-}
-
-func TestPrettifyString(t *testing.T) {
-	res := make(map[string]interface{}, 1)
-	res["0"] = "foobar"
-	pretty := prettyEncode(res)
-
-	expected := `
-{
-  "0": "foobar"
-}
-`[1:]
-
-	if pretty != expected {
-		t.Fatalf("Pretty string doesn't match")
-	}
-}
-
-func TestPrettifyInt(t *testing.T) {
-	res := make(map[string]interface{}, 1)
-	res["0"] = 234567
-	pretty := prettyEncode(res)
-
-	expected := `
-{
-  "0": 234567
-}
-`[1:]
-
-	if pretty != expected {
-		t.Fatalf("Pretty int doesn't match")
-	}
-}
-
-func TestPrettifyDuration(t *testing.T) {
-	res := make(map[string]interface{}, 1)
-	res["0"] = time.Duration(234567)
-	pretty := prettyEncode(res)
-
-	expected := `
-{
-  "0": "234.567µs"
-}
-`[1:]
-
-	if pretty != expected {
-		t.Fatalf("Pretty duration doesn't match")
-	}
-}
-
-func TestPrettifyDurationSlice(t *testing.T) {
-	res := make(map[string]interface{}, 1)
-	res["0"] = []time.Duration{123, 234567, 34567890}
-	pretty := prettyEncode(res)
-
-	expected := `
-{
-  "0": [
-    "123ns",
-    "234.567µs",
-    "34.56789ms"
-  ]
-}
-`[1:]
-
-	if pretty != expected {
-		t.Fatalf("Pretty duration slice doesn't match")
-	}
-}
diff --git a/bench/query_test.go b/bench/query_test.go
deleted file mode 100644
index 13b0ad416..000000000
--- a/bench/query_test.go
+++ /dev/null
@@ -1,20 +0,0 @@
-package bench_test
-
-import (
-	"github.com/pilosa/pilosa/bench"
-	"testing"
-)
-
-func TestRandomBitmapCall(t *testing.T) {
-	qm := bench.NewQueryGenerator(5)
-	bmc := qm.RandomBitmapCall(4, 3, 0, 1000)
-	t.Log(bmc.String())
-}
-
-func TestRandom(t *testing.T) {
-	for i := 99; i < 120; i++ {
-		qm := bench.NewQueryGenerator(int64(i))
-		call := qm.Random(10, 4, 3, 0, 1000)
-		t.Log(call)
-	}
-}
diff --git a/bench/random.go b/bench/random.go
deleted file mode 100644
index 3bb74e886..000000000
--- a/bench/random.go
+++ /dev/null
@@ -1,117 +0,0 @@
-package bench
-
-import (
-	"fmt"
-
-	"flag"
-	"io/ioutil"
-
-	"context"
-	"math/rand"
-	"time"
-)
-
-// RandomSetBits sets bits randomly and deterministically based on a seed.
-type RandomSetBits struct {
-	HasClient
-	Name           string `json:"name"`
-	BaseBitmapID   int64  `json:"base-bitmap-id"`
-	BaseProfileID  int64  `json:"base-profile-id"`
-	BitmapIDRange  int64  `json:"bitmap-id-range"`
-	ProfileIDRange int64  `json:"profile-id-range"`
-	Iterations     int    `json:"iterations"`
-	Seed           int64  `json:"seed"`
-	DB             string `json:"db"`
-}
-
-// Init adds the agent num to the random seed and initializes the client.
-func (b *RandomSetBits) Init(hosts []string, agentNum int) error {
-	b.Name = "random-set-bits"
-	b.Seed = b.Seed + int64(agentNum)
-	return b.HasClient.Init(hosts, agentNum)
-}
-
-// Usage returns the usage message to be printed.
-func (b *RandomSetBits) Usage() string {
-	return `
-random-set-bits sets random bits
-
-Agent number modifies the random seed.
-
-Usage: random-set-bits [arguments]
-
-The following arguments are available:
-
-	-base-bitmap-id int
-		bits being set will all be greater than BaseBitmapID
-
-	-bitmap-id-range int
-		number of possible bitmap ids that can be set
-
-	-base-profile-id int
-		profile id num to start from
-
-	-profile-id-range int
-		number of possible profile ids that can be set
-
-	-iterations int
-		number of bits to set
-
-	-seed int
-		Seed for RNG
-
-	-db string
-		pilosa db to use
-
-	-client-type string
-		Can be 'single' (all agents hitting one host) or 'round_robin'
-
-	-content-type string
-		protobuf or pql
-`[1:]
-}
-
-// ConsumeFlags parses all flags up to the next non flag argument (argument does
-// not start with "-" and isn't the value of a flag). It returns the remaining
-// args.
-func (b *RandomSetBits) ConsumeFlags(args []string) ([]string, error) {
-	fs := flag.NewFlagSet("RandomSetBits", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	fs.Int64Var(&b.BaseBitmapID, "base-bitmap-id", 0, "")
-	fs.Int64Var(&b.BitmapIDRange, "bitmap-id-range", 100000, "")
-	fs.Int64Var(&b.BaseProfileID, "base-profile-id", 0, "")
-	fs.Int64Var(&b.ProfileIDRange, "profile-id-range", 100000, "")
-	fs.Int64Var(&b.Seed, "seed", 1, "")
-	fs.IntVar(&b.Iterations, "iterations", 100, "")
-	fs.StringVar(&b.DB, "db", "benchdb", "")
-	fs.StringVar(&b.ClientType, "client-type", "single", "")
-	fs.StringVar(&b.ContentType, "content-type", "protobuf", "")
-
-	if err := fs.Parse(args); err != nil {
-		return nil, err
-	}
-	return fs.Args(), nil
-}
-
-// Run runs the RandomSetBits benchmark
-func (b *RandomSetBits) Run(ctx context.Context) map[string]interface{} {
-	src := rand.NewSource(b.Seed)
-	rng := rand.New(src)
-	results := make(map[string]interface{})
-	if b.client == nil {
-		results["error"] = fmt.Errorf("No client set for RandomSetBits")
-		return results
-	}
-	s := NewStats()
-	var start time.Time
-	for n := 0; n < b.Iterations; n++ {
-		bitmapID := rng.Int63n(b.BitmapIDRange)
-		profID := rng.Int63n(b.ProfileIDRange)
-		query := fmt.Sprintf("SetBit(%d, 'frame.n', %d)", b.BaseBitmapID+bitmapID, b.BaseProfileID+profID)
-		start = time.Now()
-		b.client.ExecuteQuery(ctx, b.DB, query, true)
-		s.Add(time.Now().Sub(start))
-	}
-	AddToResults(s, results)
-	return results
-}
diff --git a/bench/sliceheight.go b/bench/sliceheight.go
deleted file mode 100644
index 5b870aff1..000000000
--- a/bench/sliceheight.go
+++ /dev/null
@@ -1,151 +0,0 @@
-package bench
-
-import (
-	"context"
-	"flag"
-	"io"
-	"io/ioutil"
-	"strconv"
-	"time"
-
-	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/pql"
-)
-
-// NewSliceHeight creates a new slice height benchmark with stdin/out/err
-// initialized.
-func NewSliceHeight(stdin io.Reader, stdout, stderr io.Writer) *SliceHeight {
-	return &SliceHeight{
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	}
-}
-
-// SliceHeight benchmark tests the effect of an increasing number of bitmaps in
-// a single slice on query time.
-type SliceHeight struct {
-	MaxTime time.Duration `json:"max-time"`
-	hosts   []string
-
-	Name          string `json:"name"`
-	MinBitsPerMap int64  `json:"min-bits-per-map"`
-	MaxBitsPerMap int64  `json:"max-bits-per-map"`
-	Seed          int64  `json:"seed"`
-	Database      string `json:"database"`
-	Frame         string `json:"frame"`
-
-	Stdin  io.Reader `json:"-"`
-	Stdout io.Writer `json:"-"`
-	Stderr io.Writer `json:"-"`
-}
-
-// Usage returns the usage message to be printed.
-func (b *SliceHeight) Usage() string {
-	return `
-slice-height repeatedly imports more bitmaps into a single slice and tests query times in between.
-
-Agent number has no effect on this benchmark.
-
-Usage: slice-height [arguments]
-
-The following arguments are available:
-
-	-max-time int
-		stop benchmark after this many seconds
-
-	-min-bits-per-map int
-		minimum number of bits set per bitmap
-
-	-max-bits-per-map int
-		maximum number of bits set per bitmap
-
-	-seed int
-		seed for RNG
-
-	-db string
-		pilosa db to use
-
-	-frame string
-		frame to import into
-`[1:]
-}
-
-// ConsumeFlags parses all flags up to the next non flag argument (argument does
-// not start with "-" and isn't the value of a flag). It returns the remaining
-// args.
-func (b *SliceHeight) ConsumeFlags(args []string) ([]string, error) {
-	fs := flag.NewFlagSet("SliceHeight", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-
-	maxTime := fs.Int("max-time", 30, "")
-	fs.Int64Var(&b.MinBitsPerMap, "min-bits-per-map", 0, "")
-	fs.Int64Var(&b.MaxBitsPerMap, "max-bits-per-map", 10, "")
-	fs.Int64Var(&b.Seed, "seed", 0, "")
-	fs.StringVar(&b.Database, "db", "benchdb", "")
-	fs.StringVar(&b.Frame, "frame", "testframe", "")
-
-	if err := fs.Parse(args); err != nil {
-		return nil, err
-	}
-	b.MaxTime = time.Duration(*maxTime) * time.Second
-	return fs.Args(), nil
-}
-
-// Init sets up the slice height benchmark.
-func (b *SliceHeight) Init(hosts []string, agentNum int) error {
-	b.Name = "slice-height"
-	b.hosts = hosts
-	return nil
-}
-
-// Run runs the SliceHeight benchmark
-func (b *SliceHeight) Run(ctx context.Context) map[string]interface{} {
-	results := make(map[string]interface{})
-
-	imp := NewImport(b.Stdin, b.Stdout, b.Stderr)
-	imp.MaxBitmapID = 100
-	imp.MaxProfileID = pilosa.SliceWidth
-	imp.MinBitsPerMap = b.MinBitsPerMap
-	imp.MaxBitsPerMap = b.MaxBitsPerMap
-	imp.Database = b.Database
-	imp.Frame = b.Frame
-
-	start := time.Now()
-
-	for i := 0; i > -1; i++ {
-		iresults := make(map[string]interface{})
-		results["iteration"+strconv.Itoa(i)] = iresults
-
-		genstart := time.Now()
-		imp.Init(b.hosts, 0)
-		gendur := time.Now().Sub(genstart)
-		iresults["csvgen"] = gendur
-
-		iresults["import"] = imp.Run(ctx)
-
-		qstart := time.Now()
-		q := &pql.Call{
-			Name: "TopN",
-			Args: map[string]interface{}{
-				"frame": b.Frame,
-				"n":     50,
-			},
-		}
-		_, err := imp.Client.ExecuteQuery(ctx, b.Database, q.String(), true)
-		if err != nil {
-			iresults["query_error"] = err.Error()
-		} else {
-			qdur := time.Now().Sub(qstart)
-			iresults["query"] = qdur
-		}
-		imp.BaseBitmapID = imp.MaxBitmapID
-		imp.MaxBitmapID = imp.MaxBitmapID * 10
-
-		if time.Now().Sub(start) > b.MaxTime {
-			break
-		}
-	}
-
-	return results
-}
diff --git a/bench/stats.go b/bench/stats.go
deleted file mode 100644
index 7a32987f9..000000000
--- a/bench/stats.go
+++ /dev/null
@@ -1,65 +0,0 @@
-package bench
-
-import (
-	"math"
-	"time"
-)
-
-// Stats object helps track timing stats.
-type Stats struct {
-	Min            time.Duration
-	Max            time.Duration
-	Mean           time.Duration
-	sumSquareDelta float64
-	Total          time.Duration
-	Num            int64
-	All            []time.Duration
-	SaveAll        bool
-}
-
-// NewStats gets a Stats object.
-func NewStats() *Stats {
-	return &Stats{
-		Min: 1<<63 - 1,
-		All: make([]time.Duration, 0),
-	}
-}
-
-// Add adds a new time to the stats object.
-func (s *Stats) Add(td time.Duration) {
-	if s.SaveAll {
-		s.All = append(s.All, td)
-	}
-	s.Num += 1
-	s.Total += td
-	if td < s.Min {
-		s.Min = td
-	}
-	if td > s.Max {
-		s.Max = td
-	}
-
-	// online variance calculation
-	// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm
-	delta := td - s.Mean
-	s.Mean += delta / time.Duration(s.Num)
-	s.sumSquareDelta += float64(delta * (td - s.Mean))
-}
-
-// Avg returns the average of all durations Added to the Stats object.
-func (s *Stats) Avg() time.Duration {
-	return s.Total / time.Duration(s.Num)
-}
-
-// AddToResults serializes the summary of Stats and adds them to the results
-// map.
-func AddToResults(s *Stats, results map[string]interface{}) {
-	results["min"] = s.Min
-	results["max"] = s.Max
-	results["avg"] = s.Mean
-	variance := s.sumSquareDelta / float64(s.Num)
-	results["std"] = time.Duration(math.Sqrt(variance))
-	if s.SaveAll {
-		results["all"] = s.All
-	}
-}
diff --git a/bench/store.go b/bench/store.go
deleted file mode 100644
index 6f752cc08..000000000
--- a/bench/store.go
+++ /dev/null
@@ -1,39 +0,0 @@
-package bench
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-)
-
-// NewS3Uploader creates an S3Uploader with specified bucket and key
-func NewS3Uploader(bucket string, key string) *S3Uploader {
-	return &S3Uploader{
-		bucket,
-		s3.New(session.New(&aws.Config{})),
-		key,
-	}
-}
-
-// S3Uploader is an io.Writer for sending output to AWS S3 storage.
-type S3Uploader struct {
-	bucket  string
-	service *s3.S3
-	key     string
-}
-
-// Write writes data to the uploader's bucket/key
-func (u *S3Uploader) Write(data []byte) (int, error) {
-	// first return value of PutObject contains an ETag (hash) of the uploaded object, not needed here
-	fmt.Println(string(data))
-	_, err := u.service.PutObject(&s3.PutObjectInput{
-		Body:   strings.NewReader(string(data)),
-		Bucket: &u.bucket,
-		Key:    &u.key,
-	})
-
-	return 0, err
-}
diff --git a/bench/zipf.go b/bench/zipf.go
deleted file mode 100644
index 46832a75f..000000000
--- a/bench/zipf.go
+++ /dev/null
@@ -1,191 +0,0 @@
-package bench
-
-import (
-	"fmt"
-
-	"flag"
-	"io/ioutil"
-
-	"context"
-	"math"
-	"math/rand"
-	"time"
-)
-
-// Zipf sets random bits according to the Zipf-Mandelbrot distribution.
-// This distribution accepts two parameters, Exponent and Ratio, for both bitmaps and profiles.
-// It also uses PermutationGenerator to permute IDs randomly.
-type Zipf struct {
-	HasClient
-	Name            string  `json:"name"`
-	BaseBitmapID    int64   `json:"base-bitmap-id"`
-	BaseProfileID   int64   `json:"base-profile-id"`
-	BitmapIDRange   int64   `json:"bitmap-id-range"`
-	ProfileIDRange  int64   `json:"profile-id-range"`
-	Iterations      int     `json:"iterations"`
-	Seed            int64   `json:"seed"`
-	DB              string  `json:"db"`
-	BitmapExponent  float64 `json:"bitmap-exponent"`
-	BitmapRatio     float64 `json:"bitmap-ratio"`
-	ProfileExponent float64 `json:"profile-exponent"`
-	ProfileRatio    float64 `json:"profile-ratio"`
-	Operation       string  `json:"operation"`
-	bitmapRng       *rand.Zipf
-	profileRng      *rand.Zipf
-	bitmapPerm      *PermutationGenerator
-	profilePerm     *PermutationGenerator
-}
-
-// Usage returns the usage message to be printed.
-func (b *Zipf) Usage() string {
-	return `
-zipf sets random bits according to the Zipf distribution.
-This is a power-law distribution controlled by two parameters.
-Exponent, in the range (1, inf), with a default value of 1.001, controls
-the "sharpness" of the distribution, with higher exponent being sharper.
-Ratio, in the range (0, 1), with a default value of 0.25, controls the
-maximum variation of the distribution, with higher ratio being more uniform.
-
-Agent number modifies random seed.
-
-Usage: zipf [arguments]
-
-The following arguments are available:
-
-	-base-bitmap-id int
-		bits being set will all be greater than BaseBitmapID
-
-	-bitmap-id-range int
-		number of possible bitmap ids that can be set
-
-	-base-profile-id int
-		profile id num to start from
-
-	-profile-id-range int
-		number of possible profile ids that can be set
-
-	-iterations int
-		number of bits to set
-
-	-seed int
-		Seed for RNG
-
-	-db string
-		pilosa db to use
-
-	-bitmap-exponent float64
-		zipf exponent parameter for bitmap IDs
-
-	-bitmap-ratio float64
-		zipf probability ratio parameter for bitmap IDs
-
-	-profile-exponent float64
-		zipf exponent parameter for profile IDs
-
-	-profile-ratio float64
-		zipf probability ratio parameter for profile IDs
-
-	-client-type string
-		Can be 'single' (all agents hitting one host) or 'round_robin'
-
-	-operation string
-		Can be 'set' or 'clear'
-
-	-content-type string
-		protobuf or pql
-`[1:]
-}
-
-// ConsumeFlags parses all flags up to the next non flag argument (argument does
-// not start with "-" and isn't the value of a flag). It returns the remaining
-// args.
-func (b *Zipf) ConsumeFlags(args []string) ([]string, error) {
-	fs := flag.NewFlagSet("Zipf", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	fs.Int64Var(&b.BaseBitmapID, "base-bitmap-id", 0, "")
-	fs.Int64Var(&b.BitmapIDRange, "bitmap-id-range", 100000, "")
-	fs.Int64Var(&b.BaseProfileID, "base-profile-id", 0, "")
-	fs.Int64Var(&b.ProfileIDRange, "profile-id-range", 100000, "")
-	fs.Int64Var(&b.Seed, "seed", 1, "")
-	fs.IntVar(&b.Iterations, "iterations", 100, "")
-	fs.StringVar(&b.DB, "db", "benchdb", "")
-	fs.Float64Var(&b.BitmapExponent, "bitmap-exponent", 1.01, "")
-	fs.Float64Var(&b.BitmapRatio, "bitmap-ratio", 0.25, "")
-	fs.Float64Var(&b.ProfileExponent, "profile-exponent", 1.01, "")
-	fs.Float64Var(&b.ProfileRatio, "profile-ratio", 0.25, "")
-	fs.StringVar(&b.ClientType, "client-type", "single", "")
-	fs.StringVar(&b.Operation, "operation", "set", "")
-	fs.StringVar(&b.ContentType, "content-type", "protobuf", "")
-
-	if err := fs.Parse(args); err != nil {
-		return nil, err
-	}
-	return fs.Args(), nil
-}
-
-// Offset is the true parameter used by the Zipf distribution, but the ratio,
-// as defined here, is a simpler, readable way to define the distribution.
-// Offset is in [1, inf), and its meaning depends on N (a pain for updating benchmark configs)
-// ratio is in (0, 1), and its meaning does not depend on N.
-// it is the ratio of the lowest probability in the distribution to the highest.
-// ratio=0.01 corresponds to a very small offset - the most skewed distribution for a given pair (N, exp)
-// ratio=0.99 corresponds to a very large offset - the most nearly uniform distribution for a given (N, exp)
-func getZipfOffset(N int64, exp, ratio float64) float64 {
-	z := math.Pow(ratio, 1/exp)
-	return z * float64(N-1) / (1 - z)
-}
-
-// Init sets up the benchmark based on the agent number and initializes the
-// client.
-func (b *Zipf) Init(hosts []string, agentNum int) error {
-	b.Name = "zipf"
-	b.Seed = b.Seed + int64(agentNum)
-	rnd := rand.New(rand.NewSource(b.Seed))
-	bitmapOffset := getZipfOffset(b.BitmapIDRange, b.BitmapExponent, b.BitmapRatio)
-	b.bitmapRng = rand.NewZipf(rnd, b.BitmapExponent, bitmapOffset, uint64(b.BitmapIDRange-1))
-	profileOffset := getZipfOffset(b.ProfileIDRange, b.ProfileExponent, b.ProfileRatio)
-	b.profileRng = rand.NewZipf(rnd, b.ProfileExponent, profileOffset, uint64(b.ProfileIDRange-1))
-
-	b.bitmapPerm = NewPermutationGenerator(b.BitmapIDRange, b.Seed)
-	b.profilePerm = NewPermutationGenerator(b.ProfileIDRange, b.Seed+1)
-
-	if b.Operation != "set" && b.Operation != "clear" {
-		return fmt.Errorf("Unsupported operation: \"%s\" (must be \"set\" or \"clear\")", b.Operation)
-	}
-
-	return b.HasClient.Init(hosts, agentNum)
-}
-
-// Run runs the Zipf benchmark
-func (b *Zipf) Run(ctx context.Context) map[string]interface{} {
-	results := make(map[string]interface{})
-	if b.client == nil {
-		results["error"] = fmt.Errorf("No client set for Zipf")
-		return results
-	}
-	operation := "SetBit"
-	if b.Operation == "clear" {
-		operation = "ClearBit"
-	}
-	s := NewStats()
-	var start time.Time
-	for n := 0; n < b.Iterations; n++ {
-		// generate IDs from Zipf distribution
-		bitmapIDOriginal := b.bitmapRng.Uint64()
-		profIDOriginal := b.profileRng.Uint64()
-		// permute IDs randomly, but repeatably
-		bitmapID := b.bitmapPerm.Next(int64(bitmapIDOriginal))
-		profID := b.profilePerm.Next(int64(profIDOriginal))
-
-		query := fmt.Sprintf("%s(%d, 'frame.n', %d)", operation, b.BaseBitmapID+int64(bitmapID), b.BaseProfileID+int64(profID))
-		start = time.Now()
-		_, err := b.client.ExecuteQuery(ctx, b.DB, query, true)
-		if err != nil {
-			results["error"] = fmt.Sprintf("Error executing query in zipf: %v", err)
-			return results
-		}
-		s.Add(time.Now().Sub(start))
-	}
-	AddToResults(s, results)
-	return results
-}
diff --git a/build/build.go b/build/build.go
deleted file mode 100644
index eb25af446..000000000
--- a/build/build.go
+++ /dev/null
@@ -1,29 +0,0 @@
-package build
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"os/exec"
-)
-
-func Binary(pkg, goos, goarch string) (io.Reader, error) {
-	binFile, err := ioutil.TempFile("", "pilosactl")
-	if err != nil {
-		return nil, fmt.Errorf("build binary: %v", err)
-	}
-	com := exec.Command("go", "build", "-o", binFile.Name(), pkg)
-	com.Env = append([]string{"GOOS=" + goos, "GOARCH=" + goarch}, os.Environ()...)
-
-	err = com.Run()
-	if err != nil {
-		return nil, err
-	}
-
-	f, err := os.Open(binFile.Name())
-	if err != nil {
-		return nil, err
-	}
-	return f, nil
-}
diff --git a/cmd/pilosactl/import.json b/cmd/pilosactl/import.json
deleted file mode 100644
index 8962ae063..000000000
--- a/cmd/pilosactl/import.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
-    "benchmarks": [
-        {
-            "num": 1,
-            "name": "import",
-            "args": ["import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "import", "-max-bitmap-id", "100000", "-max-profile-id", "10000", "-max-bits-per-map", "100", "-seed", "0", "-agent-controls", "width", "-random-bitmap-order", "-db", "randoload"]
-        }
-    ]
-}
diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go
index bb22d14ac..88fa6fea5 100644
--- a/cmd/pilosactl/main.go
+++ b/cmd/pilosactl/main.go
@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"context"
 	"encoding/csv"
-	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -13,27 +12,18 @@ import (
 	"log"
 	"math/rand"
 	"os"
-	"os/signal"
 	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"text/tabwriter"
 	"time"
 	"unsafe"
 
 	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/bench"
-	"github.com/pilosa/pilosa/build"
-	"github.com/pilosa/pilosa/creator"
 	"github.com/pilosa/pilosa/pilosactl"
 	"github.com/pilosa/pilosa/roaring"
-	pssh "github.com/pilosa/pilosa/ssh"
-
-	"github.com/satori/go.uuid"
-	"golang.org/x/crypto/ssh"
 )
 
 var (
@@ -115,9 +105,6 @@ The commands are:
 	inspect    inspects fragment data files
 	check      performs a consistency check of data files
 	bench      benchmarks operations
-	create     create pilosa clusters
-	bagent     run a benchmarking agent
-	bspawn     create a cluster and agents and run benchmarks based on config file
 
 Use the "-h" flag with any command for more information.
 `)
@@ -157,12 +144,6 @@ func (m *Main) ParseFlags(args []string) error {
 		m.Cmd = NewCheckCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "bench":
 		m.Cmd = NewBenchCommand(m.Stdin, m.Stdout, m.Stderr)
-	case "create":
-		m.Cmd = NewCreateCommand(m.Stdin, m.Stdout, m.Stderr)
-	case "bagent":
-		m.Cmd = NewBagentCommand(m.Stdin, m.Stdout, m.Stderr)
-	case "bspawn":
-		m.Cmd = NewBspawnCommand(m.Stdin, m.Stdout, m.Stderr)
 	default:
 		return ErrUnknownCommand
 	}
@@ -1004,672 +985,6 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) e
 	return nil
 }
 
-// CreateCommand represents a command for creating a pilosa cluster.
-type CreateCommand struct {
-	ServerN       int      `json:"serverN"`
-	ReplicaN      int      `json:"replicaN"`
-	LogFilePrefix string   `json:"log-file-prefix"`
-	Hosts         []string `json:"hosts"`
-	GoMaxProcs    int      `json:"gomaxprocs"`
-
-	SSHUser string `json:"ssh-user"`
-
-	CopyBinary bool   `json:"copy-binary"`
-	GOOS       string `json:"goos"`
-	GOARCH     string `json:"goarch"`
-
-	// The following are set by CreateCommand.Run once they are known and exist
-	// to appear in the output
-	LogFiles   []string `json:"log-files"`
-	FinalHosts []string `json:"final-hosts"`
-
-	// Standard input/output
-	Stdin  io.Reader `json:"-"`
-	Stdout io.Writer `json:"-"`
-	Stderr io.Writer `json:"-"`
-}
-
-// NewCreateCommand returns a new instance of CreateCommand.
-func NewCreateCommand(stdin io.Reader, stdout, stderr io.Writer) *CreateCommand {
-	return &CreateCommand{
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	}
-}
-
-// ParseFlags parses command line flags from args.
-func (cmd *CreateCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	fs.IntVar(&cmd.ServerN, "serverN", 3, "")
-	fs.IntVar(&cmd.ReplicaN, "replicaN", 1, "")
-	fs.StringVar(&cmd.LogFilePrefix, "log-file-prefix", "", "")
-	var hosts string
-	fs.IntVar(&cmd.GoMaxProcs, "gomaxprocs", 0, "")
-	fs.StringVar(&hosts, "hosts", "", "")
-	fs.StringVar(&cmd.SSHUser, "ssh-user", "", "")
-	fs.BoolVar(&cmd.CopyBinary, "copy-binary", false, "")
-	fs.StringVar(&cmd.GOOS, "goos", "linux", "")
-	fs.StringVar(&cmd.GOARCH, "goarch", "amd64", "")
-
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if len(fs.Args()) > 0 {
-		fmt.Fprintf(cmd.Stderr, "Uknown args: %v\n", strings.Join(fs.Args(), " "))
-		return flag.ErrHelp
-	}
-	cmd.Hosts = customSplit(hosts, ",")
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *CreateCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl create [args]
-
-Creates a pilosa cluster. Defaults to a 3-node in-process cluster.
-
-The following flags are allowed:
-
-	-serverN
-		number of hosts in cluster. Disregarded if 'hosts' is set
-
-	-replicaN
-		replication factor for cluster
-
-	-hosts
-		comma separated host:port list. If hosts is set, create will start
-		pilosa on these pre-existing hosts. The same host may be listed multiple
-		times with different ports
-
-	-log-file-prefix
-		output from the started cluster will go into files with
-		this prefix (one per node)
-
-	-ssh-user
-		username to use when contacting remote hosts
-
-	-gomaxprocs
-		when starting a cluster on remote hosts, this will set the value
-		of GOMAXPROCS.
-
-	-copy-binary
-		controls whether or not to build and copy pilosa to hosts
-
-	-goos
-		when using copy-binary, GOOS to use while building binary
-
-	-goarch
-		when using copy-binary, GOARCH to use while building binary
-`)
-}
-
-// Run executes cluster creation.
-func (cmd *CreateCommand) Run(ctx context.Context) error {
-	var clus creator.Cluster
-	if len(cmd.Hosts) == 0 {
-		fmt.Fprintf(cmd.Stderr, "create: no hosts specified - creating cluster in-process\n")
-		clus = &creator.LocalCluster{
-			ReplicaN: cmd.ReplicaN,
-			ServerN:  cmd.ServerN,
-		}
-	} else {
-		if cmd.ServerN != 0 {
-			fmt.Fprintf(cmd.Stderr, "create: hosts were specified, so ignoring serverN\n")
-		}
-		clus = &creator.RemoteCluster{
-			ClusterHosts: cmd.Hosts,
-			ReplicaN:     cmd.ReplicaN,
-			SSHUser:      cmd.SSHUser,
-			Stderr:       cmd.Stderr,
-			GoMaxProcs:   cmd.GoMaxProcs,
-			CopyBinary:   cmd.CopyBinary,
-			GOOS:         cmd.GOOS,
-			GOARCH:       cmd.GOARCH,
-		}
-
-	}
-
-	err := clus.Start()
-	if err != nil {
-		return fmt.Errorf("starting cluster: %v", err)
-	}
-
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt)
-	go func() {
-		for range c {
-			fmt.Fprintf(cmd.Stderr, "\ncreate: caught signal - shutting down\n")
-			err := clus.Shutdown()
-			code := 0
-			if err != nil {
-				code = 1
-			}
-			os.Exit(code)
-		}
-	}()
-
-	defer clus.Shutdown()
-	cmd.FinalHosts = clus.Hosts()
-
-	logReaders := clus.Logs()
-	if cmd.LogFilePrefix != "" {
-		cmd.LogFiles = make([]string, len(clus.Hosts()))
-	}
-	for i, _ := range clus.Hosts() {
-		var f io.Writer = cmd.Stderr
-		var err error
-		if cmd.LogFilePrefix != "" {
-			f, err = os.Create(cmd.LogFilePrefix + strconv.Itoa(i))
-			if err != nil {
-				return err
-			}
-			cmd.LogFiles[i] = f.(*os.File).Name()
-		}
-
-		go func(i int, f io.Writer) {
-			_, err := io.Copy(f, logReaders[i])
-			if err != nil {
-				fmt.Fprintf(cmd.Stderr, "create: error copying cluster logs: '%v'\n", err)
-			}
-		}(i, f)
-	}
-
-	enc := json.NewEncoder(cmd.Stdout)
-	enc.SetIndent("", "  ")
-	err = enc.Encode(cmd)
-	if err != nil {
-		return err
-	}
-	fmt.Fprintf(cmd.Stderr, "create: cluster started.\n")
-	select {}
-
-}
-
-// BagentCommand represents a command for running a benchmark agent. A benchmark
-// agent runs multiple benchmarks in series in the order that they are specified
-// on the command line.
-type BagentCommand struct {
-	// Slice of Benchmarks which will be run serially.
-	Benchmarks []bench.Benchmark `json:"benchmarks"`
-	// AgentNum will be passed to each benchmark's Run method so that it can
-	// parameterize its behavior.
-	AgentNum int `json:"agent-num"`
-
-	// Enable pretty printing of results, for human consumption.
-	HumanReadable bool `json:"human-readable"`
-
-	// Slice of pilosa hosts to run the Benchmarks against.
-	Hosts []string `json:"hosts"`
-
-	Stdin  io.Reader `json:"-"`
-	Stdout io.Writer `json:"-"`
-	Stderr io.Writer `json:"-"`
-}
-
-// NewBagentCommand returns a new instance of BagentCommand.
-func NewBagentCommand(stdin io.Reader, stdout, stderr io.Writer) *BagentCommand {
-	return &BagentCommand{
-		Benchmarks:    []bench.Benchmark{},
-		Hosts:         []string{},
-		AgentNum:      0,
-		HumanReadable: false,
-
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	}
-}
-
-// ParseFlags parses command line flags for the BagentCommand. First the command
-// wide flags `hosts` and `agent-num` are parsed. The rest of the flags should be
-// a series of subcommands along with their flags. ParseFlags runs each
-// subcommand's `ConsumeFlags` method which parses the flags for that command
-// and returns the rest of the argument slice which should contain further
-// subcommands.
-func (cmd *BagentCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-
-	var pilosaHosts string
-	fs.StringVar(&pilosaHosts, "hosts", "localhost:15000", "")
-	fs.IntVar(&cmd.AgentNum, "agent-num", 0, "")
-	fs.BoolVar(&cmd.HumanReadable, "human", true, "")
-
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	remArgs := fs.Args()
-	if len(remArgs) == 0 {
-		return flag.ErrHelp
-	}
-	for len(remArgs) > 0 {
-		var bm bench.Command
-		var err error
-		switch remArgs[0] {
-		case "-help", "-h":
-			return flag.ErrHelp
-		case "diagonal-set-bits":
-			bm = &bench.DiagonalSetBits{}
-		case "random-set-bits":
-			bm = &bench.RandomSetBits{}
-		case "zipf":
-			bm = &bench.Zipf{}
-		case "multi-db-set-bits":
-			bm = &bench.MultiDBSetBits{}
-		case "random-query":
-			bm = &bench.RandomQuery{}
-		case "import":
-			bm = bench.NewImport(cmd.Stdin, cmd.Stdout, cmd.Stderr)
-		case "slice-height":
-			bm = bench.NewSliceHeight(cmd.Stdin, cmd.Stdout, cmd.Stderr)
-		default:
-			return fmt.Errorf("Unknown benchmark cmd: %v", remArgs[0])
-		}
-		remArgs, err = bm.ConsumeFlags(remArgs[1:])
-		cmd.Benchmarks = append(cmd.Benchmarks, bm)
-		if err != nil {
-			if err == flag.ErrHelp {
-				fmt.Fprintln(cmd.Stderr, bm.Usage())
-				return fmt.Errorf("")
-			}
-			return fmt.Errorf("BagentCommand.ParseFlags: %v", err)
-		}
-	}
-	cmd.Hosts = customSplit(pilosaHosts, ",")
-
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *BagentCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl bagent [options] <subcommand [options]>...
-
-Runs benchmarks against a pilosa cluster.
-
-The following flags are allowed:
-
-	-hosts ("localhost:15000")
-		comma separated list of host:port describing all hosts in the cluster
-
-	-agent-num (0)
-		an integer differentiating this agent from others in the fleet
-
-	-human (true)
-		boolean to enable human-readable format
-
-	subcommands:
-		diagonal-set-bits
-		random-set-bits
-		zipf
-		multi-db-set-bits
-		random-query
-		import
-		slice-height
-`)
-}
-
-// Run executes the benchmark agent.
-func (cmd *BagentCommand) Run(ctx context.Context) error {
-	sbm := serial(cmd.Benchmarks...)
-	err := sbm.Init(cmd.Hosts, cmd.AgentNum)
-	if err != nil {
-		return fmt.Errorf("in cmd.Run initialization: %v", err)
-	}
-
-	res := sbm.Run(ctx)
-	res["agent-num"] = cmd.AgentNum
-	enc := json.NewEncoder(cmd.Stdout)
-	if cmd.HumanReadable {
-		enc.SetIndent("", "  ")
-		res = bench.Prettify(res)
-	}
-	err = enc.Encode(res)
-	if err != nil {
-		fmt.Fprintln(cmd.Stderr, err)
-	}
-
-	return nil
-}
-
-// BspawnCommand represents a command for spawning complex benchmarks. This
-// includes cluster creation and teardown, agent creation and teardown, running
-// multiple benchmarks in series and/or parallel, and collecting all the
-// results.
-type BspawnCommand struct {
-	// If PilosaHosts is specified, CreatorArgs is ignored and the existing
-	// cluster specified here is used.
-	PilosaHosts []string `json:"pilosa-hosts"`
-
-	// CreateCommand will be used with these arguments to create a cluster -
-	// the cluster will be used to populate the PilosaHosts field. This
-	// should include everything that comes after `pilosactl create`
-	CreatorArgs []string `json:"creator-args"`
-
-	// List of hosts to run agents on. If this is empty, agents will be run
-	// locally.
-	AgentHosts []string `json:"agent-hosts"`
-
-	// Makes output human readable
-	HumanReadable bool `json:"human-readable"`
-
-	// Result destination, ["stdout", "s3"]
-	Output string `json:"output"`
-
-	// If this is true, build and copy pilosactl binary to agent hosts.
-	CopyBinary bool   `json:"copy-binary"`
-	GOOS       string `json:"goos"`
-	GOARCH     string `json:"goarch"`
-
-	// Benchmarks is a slice of Spawns which specifies all of the bagent
-	// commands to run. These will all be run in parallel, started on each
-	// of the agents in a round robin fashion.
-	Benchmarks []Spawn `json:"benchmarks"`
-
-	SSHUser string `json:"ssh-user"`
-
-	Stdin  io.Reader `json:"-"`
-	Stdout io.Writer `json:"-"`
-	Stderr io.Writer `json:"-"`
-}
-
-// Spawn represents a bagent command run in parallel across Num agents. The
-// bagent command can run multiple Benchmarks serially within itself.
-type Spawn struct {
-	Num  int      `json:"num"`  // number of agents to run
-	Name string   `json:"name"` // Should describe what this Spawn does
-	Args []string `json:"args"` // everything that comes after `pilosactl bagent [arguments]`
-}
-
-// NewBspawnCommand returns a new instance of BspawnCommand.
-func NewBspawnCommand(stdin io.Reader, stdout, stderr io.Writer) *BspawnCommand {
-	return &BspawnCommand{
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	}
-}
-
-// ParseFlags parses command line flags from args.
-func (cmd *BspawnCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	creatorHosts := fs.String("creator.hosts", "", "")
-	creatorLFP := fs.String("creator.log-file-prefix", "", "")
-	creatorCopyBinary := fs.Bool("creator.copy-binary", false, "")
-	pilosaHosts := fs.String("pilosa-hosts", "", "")
-	agentHosts := fs.String("agent-hosts", "", "")
-	sshUser := fs.String("ssh-user", "", "")
-	fs.BoolVar(&cmd.HumanReadable, "human", true, "")
-	fs.StringVar(&cmd.Output, "output", "stdout", "")
-	fs.BoolVar(&cmd.CopyBinary, "copy-binary", false, "")
-	fs.StringVar(&cmd.GOOS, "goos", "linux", "")
-	fs.StringVar(&cmd.GOARCH, "goarch", "amd64", "")
-
-	err := fs.Parse(args)
-	if err != nil {
-		return err
-	}
-	if len(fs.Args()) != 1 {
-		return flag.ErrHelp
-	}
-	f, err := os.Open(fs.Args()[0])
-	if err != nil {
-		return err
-	}
-	dec := json.NewDecoder(f)
-	err = dec.Decode(cmd)
-	if err != nil {
-		return err
-	}
-	if *pilosaHosts != "" {
-		cmd.PilosaHosts = customSplit(*pilosaHosts, ",")
-	}
-	if *agentHosts != "" {
-		cmd.AgentHosts = customSplit(*agentHosts, ",")
-	}
-	if *sshUser != "" {
-		cmd.SSHUser = *sshUser
-	}
-	if *pilosaHosts == "" {
-		cmd.CreatorArgs = []string{"-hosts=" + *creatorHosts, "-log-file-prefix=" + *creatorLFP, "-ssh-user=" + cmd.SSHUser, "-goos=" + cmd.GOOS, "-goarch=" + cmd.GOARCH}
-		if *creatorCopyBinary {
-			cmd.CreatorArgs = append(cmd.CreatorArgs, "-copy-binary")
-		}
-	}
-
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *BspawnCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl spawn [flags] <configfile>
-
-Benchmark orchestration tool - runs 'create' and potentially multiple instances
-of 'bagent' spread across a number of hosts.
-
-The following flags are allowed and will override the values in the config file:
-
-	-creator.hosts ()
-		hosts argument for pilosactl create
-
-	-creator.log-file-prefix ()
-		log-file-prefix argument for pilosactl create. If empty, log to stderr.
-
-	-creator.copy-binary (false)
-		pilosactl create should build and copy pilosa binary to cluster
-
-	-pilosa-hosts ([])
-		pilosa hosts to run against (will ignore creator args)
-
-	-agent-hosts ("localhost")
-		hosts to use for benchmark agents
-
-	-ssh-user (current username)
-		username to use when contacting remote hosts
-
-	-human (true)
-		toggle human readable output (indented json)
-
-	-output ("stdout")
-		string to select output destination, "stdout" or "s3"
-
-	-copy-binary (false)
-		controls whether or not to build and copy pilosactl to agents
-
-	-goos (linux)
-		when using copy-binary, GOOS to use while building binary
-
-	-goarch (amd64)
-		when using copy-binary, GOARCH to use while building binary
-
-
-`)
-}
-
-// Run executes the main program execution.
-func (cmd *BspawnCommand) Run(ctx context.Context) error {
-	runUUID := uuid.NewV1()
-	output := make(map[string]interface{})
-	output["run-uuid"] = runUUID.String()
-	if len(cmd.PilosaHosts) == 0 {
-		fmt.Fprintln(cmd.Stderr, "bspawn: pilosa-hosts not specified - using create command to build cluster")
-		r, w := io.Pipe()
-		createCmd := NewCreateCommand(cmd.Stdin, w, cmd.Stderr)
-		err := createCmd.ParseFlags(cmd.CreatorArgs)
-		if err != nil {
-			return err
-		}
-		go func() {
-			err := createCmd.Run(ctx)
-			if err != nil {
-				fmt.Fprintf(cmd.Stderr, "bspawn: cluster creation error while spawning: %v\n", err)
-			}
-		}()
-		clus := &CreateCommand{}
-		dec := json.NewDecoder(r)
-		err = dec.Decode(clus)
-		if err != nil {
-			return err
-		}
-		cmd.PilosaHosts = clus.FinalHosts
-		output["cluster"] = clus
-	}
-	if len(cmd.AgentHosts) == 0 {
-		fmt.Fprintln(cmd.Stderr, "bpspawn: no agent-hosts specified; all agents will be spawned on localhost")
-		cmd.AgentHosts = []string{"localhost"}
-	}
-	output["spawn"] = cmd
-	res, err := cmd.spawnRemote(ctx)
-	if err != nil {
-		return err
-	}
-	output["results"] = res
-
-	var writer io.Writer
-	if cmd.Output == "s3" {
-		writer = bench.NewS3Uploader("benchmarks-pilosa", runUUID.String()+".json")
-	} else if cmd.Output == "stdout" {
-		writer = cmd.Stdout
-	} else {
-		return fmt.Errorf("invalid bspawn output destination")
-	}
-
-	enc := json.NewEncoder(writer)
-	if cmd.HumanReadable {
-		enc.SetIndent("", "  ")
-		output = bench.Prettify(output)
-	}
-	return enc.Encode(output)
-}
-
-func (cmd *BspawnCommand) spawnRemote(ctx context.Context) (map[string]interface{}, error) {
-	agentIdx := 0
-	agentFleet, err := pssh.NewFleet(cmd.AgentHosts, cmd.SSHUser, "", cmd.Stderr)
-	if err != nil {
-		return nil, err
-	}
-
-	cmdName := "pilosactl"
-	if cmd.CopyBinary {
-		cmdName = "/tmp/pilosactl" + strconv.Itoa(rand.Int())
-		fmt.Fprintf(cmd.Stderr, "bspawn: building pilosactl binary with GOOS=%v and GOARCH=%v to copy to agents at %v\n", cmd.GOOS, cmd.GOARCH, cmdName)
-		pkg := "github.com/pilosa/pilosa/cmd/pilosactl"
-		bin, err := build.Binary(pkg, cmd.GOOS, cmd.GOARCH)
-		if err != nil {
-			return nil, err
-		}
-		err = agentFleet.WriteFile(cmdName, "+x", bin)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	sessions := make([]*ssh.Session, 0)
-	results := make(map[string]interface{})
-	resLock := sync.Mutex{}
-	wg := sync.WaitGroup{}
-	fmt.Fprintln(cmd.Stderr, "bspawn: running benchmarks")
-	for _, sp := range cmd.Benchmarks {
-		results[sp.Name] = make(map[int]interface{})
-		for i := 0; i < sp.Num; i++ {
-			agentIdx %= len(cmd.AgentHosts)
-			sess, err := agentFleet[cmd.AgentHosts[agentIdx]].NewSession()
-			if err != nil {
-				return nil, err
-			}
-			agentIdx += 1
-			sessions = append(sessions, sess)
-			stdout, err := sess.StdoutPipe()
-			if err != nil {
-				return nil, err
-			}
-			wg.Add(1)
-			go func(stdout io.Reader, name string, num int) {
-				defer wg.Done()
-				dec := json.NewDecoder(stdout)
-				var v interface{}
-				err := dec.Decode(&v)
-				if err != nil {
-					fmt.Fprintf(cmd.Stderr, "error decoding json: %v, spawn: %v\n", err, name)
-				}
-				resLock.Lock()
-				results[name].(map[int]interface{})[num] = v
-				resLock.Unlock()
-			}(stdout, sp.Name, i)
-			sess.Stderr = cmd.Stderr
-			err = sess.Start(cmdName + " bagent -agent-num=" + strconv.Itoa(i) + " -hosts=" + strings.Join(cmd.PilosaHosts, ",") + " " + strings.Join(sp.Args, " "))
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-
-	for _, sess := range sessions {
-		err = sess.Wait()
-		if err != nil {
-			return nil, fmt.Errorf("error waiting for remote bagent: %v", err)
-		}
-	}
-	wg.Wait()
-	return results, nil
-}
-
-type serialBenchmark struct {
-	benchmarkers []bench.Benchmark
-}
-
-// Init calls Init for each benchmark. If there are any errors, it will return a
-// non-nil error value.
-func (sb *serialBenchmark) Init(hosts []string, agentNum int) error {
-	errors := make([]error, len(sb.benchmarkers))
-	hadErr := false
-	for i, b := range sb.benchmarkers {
-		errors[i] = b.Init(hosts, agentNum)
-		if errors[i] != nil {
-			hadErr = true
-		}
-	}
-	if hadErr {
-		return fmt.Errorf("Had errs in serialBenchmark.Init: %v", errors)
-	}
-	return nil
-}
-
-// Run runs the serial benchmark and returns it's results in a nested map - the
-// top level keys are the indices of each benchmark in the list of benchmarks,
-// and the values are the results of each benchmark's Run method.
-func (sb *serialBenchmark) Run(ctx context.Context) map[string]interface{} {
-	benchmarks := make([]map[string]interface{}, len(sb.benchmarkers))
-	results := map[string]interface{}{"benchmarks": benchmarks}
-
-	total_start := time.Now()
-	for i, b := range sb.benchmarkers {
-		start := time.Now()
-		output := b.Run(ctx)
-		if _, ok := output["runtime"]; ok {
-			panic(fmt.Sprintf("Benchmark %v added 'runtime' to its results", b))
-		}
-		output["runtime"] = time.Now().Sub(start)
-		ret := map[string]interface{}{"output": output, "metadata": b}
-		benchmarks[i] = ret
-	}
-	results["total_runtime"] = time.Now().Sub(total_start)
-	return results
-}
-
-// serial takes a variable number of Benchmarks and returns a Benchmark
-// which combines then and will run each serially.
-func serial(bs ...bench.Benchmark) bench.Benchmark {
-	return &serialBenchmark{
-		benchmarkers: bs,
-	}
-}
-
 // readCSVRow reads a bitmap/profile pair from a CSV row.
 func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err error) {
 	// Read CSV row.
@@ -1709,12 +1024,5 @@ func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err
 	return bitmapID, profileID, timestamp, nil
 }
 
-func customSplit(s string, sep string) []string {
-	if s == "" {
-		return []string{}
-	}
-	return strings.Split(s, sep)
-}
-
 // errBlank indicates a blank row in a CSV file.
 var errBlank = errors.New("blank row")
diff --git a/cmd/pilosactl/multidbspawn.json b/cmd/pilosactl/multidbspawn.json
deleted file mode 100644
index e1c1a6759..000000000
--- a/cmd/pilosactl/multidbspawn.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-    "benchmarks": [
-        {
-            "num": 3,
-            "args": ["multi-db-set-bits", "-iterations", "30000", "-client-type", "round_robin"]
-        }
-    ]
-}
diff --git a/cmd/pilosactl/randSetAndQuery.json b/cmd/pilosactl/randSetAndQuery.json
deleted file mode 100644
index 92a95e038..000000000
--- a/cmd/pilosactl/randSetAndQuery.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
-    "benchmarks": [
-        {
-            "num": 3,
-            "args": [
-                    "random-set-bits", "-iterations", "30000", "-profile-id-range", "1000000", "-bitmap-id-range", "1000000", "-seed", "2345", "-client-type", "round_robin", "-db", "randsetandquery",
-                    "random-query", "-iterations", "1000", "-bitmap-id-range", "1000000", "-seed", "1239", "-dbs", "randsetandquery"
-            ]
-        }
-    ]
-}
diff --git a/cmd/pilosactl/randspawn.json b/cmd/pilosactl/randspawn.json
deleted file mode 100644
index 154cca521..000000000
--- a/cmd/pilosactl/randspawn.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-    "benchmarks": [
-        {
-            "num": 3,
-            "args": ["random-set-bits", "-iterations", "30000", "-profile-id-range", "1000000", "-bitmap-id-range", "1000000", "-seed", "2345", "-client-type", "round_robin"]
-        }
-    ]
-}
diff --git a/cmd/pilosactl/slice-height.json b/cmd/pilosactl/slice-height.json
deleted file mode 100644
index ac77e501b..000000000
--- a/cmd/pilosactl/slice-height.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-    "benchmarks": [
-        {
-            "num": 1,
-            "args": ["-human", "slice-height", "-max-time", "1", "-max-bits-per-map", "100"]
-        }
-    ]
-}
diff --git a/cmd/pilosactl/spawn.json b/cmd/pilosactl/spawn.json
deleted file mode 100644
index d1c37e2b3..000000000
--- a/cmd/pilosactl/spawn.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-    "benchmarks": [
-        {
-            "num": 3,
-            "name": "set-diags",
-            "args": ["diagonal-set-bits", "-iterations", "30000", "-client-type", "round_robin"]
-        },
-        {
-            "num": 2,
-            "name": "rand-plus-zipf",
-            "args": ["random-set-bits", "-iterations", "20000", "zipf", "-iterations", "100"]
-        }
-    ]
-}
diff --git a/cmd/pilosactl/zipfspawn.json b/cmd/pilosactl/zipfspawn.json
deleted file mode 100644
index 7c6770bdf..000000000
--- a/cmd/pilosactl/zipfspawn.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-    "benchmarks": [
-        {
-            "num": 1,
-            "args": ["zipf", "-iterations", "10000", "-profile-id-range", "100", "-bitmap-id-range", "100", "-seed", "2345", "-client-type", "round_robin", "-bitmap-exponent", "1.001", "-bitmap-ratio", ".9", "-profile-exponent", "1.001", "-profile-ratio", ".3"]
-        },
-        {
-            "num": 1,
-            "args": ["zipf", "-iterations", "10000", "-profile-id-range", "100", "-bitmap-id-range", "100", "-seed", "2345", "-client-type", "round_robin", "-bitmap-exponent", "1.001", "-bitmap-ratio", ".9", "-profile-exponent", "1.001", "-profile-ratio", ".3", "-operation", "clear"]
-        }
-    ]
-}
diff --git a/creator/creator.go b/creator/creator.go
deleted file mode 100644
index a7840dad2..000000000
--- a/creator/creator.go
+++ /dev/null
@@ -1,11 +0,0 @@
-// creator contains code for standing up pilosa clusters
-package creator
-
-import "io"
-
-type Cluster interface {
-	Start() error
-	Hosts() []string
-	Shutdown() error
-	Logs() []io.Reader
-}
diff --git a/creator/local.go b/creator/local.go
deleted file mode 100644
index 4d4b6b8ef..000000000
--- a/creator/local.go
+++ /dev/null
@@ -1,94 +0,0 @@
-package creator
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strconv"
-
-	"github.com/pilosa/pilosa"
-)
-
-type LocalCluster struct {
-	ReplicaN int
-	ServerN  int
-	hosts    []string
-	logs     []io.Reader
-	servers  []*pilosa.Server
-	cluster  *pilosa.Cluster
-	path     string
-}
-
-func (localCluster *LocalCluster) Start() error {
-	BasePort := 19327
-
-	localCluster.hosts = make([]string, localCluster.ServerN)
-	localCluster.servers = make([]*pilosa.Server, localCluster.ServerN)
-	localCluster.logs = make([]io.Reader, localCluster.ServerN)
-
-	path, err := ioutil.TempDir("", "pilosa-bench-")
-	if err != nil {
-		return err
-	}
-	localCluster.path = path
-
-	// Build cluster configuration.
-	cluster := pilosa.NewCluster()
-	cluster.ReplicaN = localCluster.ReplicaN
-
-	for i := 0; i < localCluster.ServerN; i++ {
-		cluster.Nodes = append(cluster.Nodes, &pilosa.Node{
-			Host: fmt.Sprintf("localhost:%d", BasePort+i),
-		})
-	}
-	localCluster.cluster = cluster
-
-	// Build servers.
-	for i := range localCluster.servers {
-		// Make server work directory.
-		if err := os.MkdirAll(filepath.Join(path, strconv.Itoa(i)), 0777); err != nil {
-			return err
-		}
-
-		// Build server.
-		s := pilosa.NewServer()
-		s.Host = fmt.Sprintf("localhost:%d", BasePort+i)
-		s.Cluster = cluster
-		s.Index.Path = filepath.Join(path, strconv.Itoa(i), "data")
-
-		// Create log stream
-		localCluster.logs[i], s.LogOutput = io.Pipe()
-
-		localCluster.servers[i] = s
-	}
-
-	// Open all servers.
-	for i, s := range localCluster.servers {
-		if err := s.Open(); err != nil {
-			return err
-		}
-		localCluster.hosts[i] = s.Host
-	}
-
-	return nil
-}
-
-func (c *LocalCluster) Hosts() []string   { return c.hosts }
-func (c *LocalCluster) Logs() []io.Reader { return c.logs }
-func (c *LocalCluster) Shutdown() error {
-	errs := ""
-	for _, s := range c.servers {
-		if err := s.Close(); err != nil {
-			errs = errs + err.Error() + "; "
-		}
-	}
-	if err := os.RemoveAll(c.path); err != nil {
-		errs = errs + err.Error() + ";"
-	}
-	if errs != "" {
-		return fmt.Errorf(errs)
-	}
-	return nil
-}
diff --git a/creator/remote.go b/creator/remote.go
deleted file mode 100644
index ebd3a5bcf..000000000
--- a/creator/remote.go
+++ /dev/null
@@ -1,187 +0,0 @@
-package creator
-
-import (
-	"fmt"
-	"io"
-	"math/rand"
-	"net"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/BurntSushi/toml"
-	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/build"
-	pssh "github.com/pilosa/pilosa/ssh"
-	"golang.org/x/crypto/ssh"
-)
-
-type RemoteCluster struct {
-	ClusterHosts []string
-	ReplicaN     int
-	SSHUser      string
-	Keyfile      string
-	Key          []byte
-	GoMaxProcs   int
-	CopyBinary   bool
-	GOOS         string
-	GOARCH       string
-	Stderr       io.Writer
-	wg           *sync.WaitGroup
-	logs         []io.Reader
-	sessions     []*ssh.Session
-	pipeRs       []*io.PipeReader
-	pipeWs       []*io.PipeWriter
-	stdins       []io.WriteCloser
-}
-
-// Start creates a configuration for each host in the cluster, copies it to the
-// node, and starts the pilosa process on the remote host.
-func (c *RemoteCluster) Start() error {
-	c.logs = make([]io.Reader, 0)
-	if len(c.ClusterHosts) == 0 {
-		return fmt.Errorf("no type or hosts specified - cannot continue")
-	}
-
-	fleet, err := pssh.NewFleet(c.ClusterHosts, c.SSHUser, c.Keyfile, c.Stderr)
-	if err != nil {
-		return fmt.Errorf("connecting to cluster hosts: %v", err)
-	}
-	cmdName := "pilosa"
-	if c.CopyBinary {
-		cmdName = "/tmp/pilosa" + strconv.Itoa(rand.Int())
-		fmt.Fprintf(c.Stderr, "create: building pilosa binary with GOOS=%v and GOARCH=%v to copy to hosts at %v", c.GOOS, c.GOARCH, cmdName)
-
-		pkg := "github.com/pilosa/pilosa/cmd/pilosa"
-		bin, err := build.Binary(pkg, c.GOOS, c.GOARCH)
-		if err != nil {
-			return fmt.Errorf("building binary: %v", err)
-		}
-
-		err = fleet.WriteFile(cmdName, "+x", bin)
-		if err != nil {
-			return fmt.Errorf("writing binary to fleet: %v", err)
-		}
-
-	}
-
-	// build config
-	conf := pilosa.NewConfigForHosts(c.ClusterHosts)
-	conf.Cluster.ReplicaN = c.ReplicaN
-
-	// copy config to remote hosts and start pilosa
-	c.wg = &sync.WaitGroup{}
-	for _, hostport := range c.ClusterHosts {
-
-		// Set up config for this host
-		host, port, err := net.SplitHostPort(hostport)
-		if err != nil {
-			return fmt.Errorf("splitting hostport: %v", err)
-		}
-		conf.Host = hostport
-		conf.DataDir = "~/.pilosa" + port
-
-		// Get client for host
-		client, err := fleet.Get(host)
-		if err != nil {
-			return fmt.Errorf("connecting to host: %v", err)
-		}
-		configname := "pilosa" + port + ".conf"
-		w, err := client.OpenFile(configname, "")
-		if err != nil {
-			return fmt.Errorf("opening remote config file: %v", err)
-		}
-		enc := toml.NewEncoder(w)
-		err = enc.Encode(conf)
-		if err != nil {
-			return fmt.Errorf("encoding config: %v", err)
-		}
-		err = w.Close()
-		if err != nil {
-			return fmt.Errorf("closing config writer: %v", err)
-		}
-
-		// Start pilosa on remote host
-		sess, err := client.NewSession()
-		if err != nil {
-			return err
-		}
-		// Have to request pty in order to be able to kill remote process
-		// reliably.
-		modes := ssh.TerminalModes{
-			ssh.ISIG: 1,
-			ssh.ECHO: 0,
-		}
-		err = sess.RequestPty("vt100", 40, 80, modes)
-		if err != nil {
-			return fmt.Errorf("request pty error: %v", err)
-		}
-		pipeR, pipeW := io.Pipe()
-		sess.Stdout = pipeW
-		sess.Stderr = pipeW
-		inpipe, err := sess.StdinPipe()
-		if err != nil {
-			return err
-		}
-		c.logs = append(c.logs, pipeR)
-		c.sessions = append(c.sessions, sess)
-		c.pipeRs = append(c.pipeRs, pipeR)
-		c.pipeWs = append(c.pipeWs, pipeW)
-		c.stdins = append(c.stdins, inpipe)
-
-		gomaxprocsString := ""
-		if c.GoMaxProcs != 0 {
-			gomaxprocsString = "GOMAXPROCS=" + strconv.Itoa(c.GoMaxProcs) + " "
-		}
-
-		err = sess.Start(gomaxprocsString + cmdName + " -config " + configname)
-		if err != nil {
-			return err
-		}
-		c.wg.Add(1)
-		go func() {
-			defer c.wg.Done()
-			err = sess.Wait()
-			if err != nil {
-				fmt.Fprintf(c.Stderr, "problem with remote pilosa process: %v\n", err)
-			}
-		}()
-	}
-	return nil
-}
-
-func (c *RemoteCluster) Hosts() []string   { return c.ClusterHosts }
-func (c *RemoteCluster) Logs() []io.Reader { return c.logs }
-func (c *RemoteCluster) Shutdown() error {
-	for i, sess := range c.sessions {
-		var err error
-		_, err = c.stdins[i].Write([]byte{3}) // Send Control C
-		if err != nil {
-			fmt.Fprintf(c.Stderr, "Error write-signaling remote process: %v\n", err)
-		}
-		// signaling isn't supported by many ssh servers - hence the hack above
-		err = sess.Signal(ssh.SIGINT)
-		if err != nil {
-			fmt.Fprintf(c.Stderr, "Error signaling remote process: %v\n", err)
-		}
-	}
-
-	done := make(chan struct{}, 1)
-	go func() {
-		c.wg.Wait()
-		close(done)
-	}()
-	select {
-	case <-done:
-		return nil
-	case <-time.After(time.Second * 5):
-		for _, sess := range c.sessions {
-			err := sess.Close()
-			if err != nil {
-				fmt.Fprintf(c.Stderr, "Error closing remote session: %v\n", err)
-			}
-		}
-		return fmt.Errorf("timed out waiting for remote processes to exit")
-	}
-
-}

From 33d2ecf6b7942bcf8532aece0ff0a6ac3f342eca Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 10 Feb 2017 09:08:58 -0600
Subject: [PATCH 09/37] remove unused vendor stuff

---
 glide.lock |  48 +-----------
 glide.yaml |   2 -
 ssh/ssh.go | 225 -----------------------------------------------------
 3 files changed, 3 insertions(+), 272 deletions(-)
 delete mode 100644 ssh/ssh.go

diff --git a/glide.lock b/glide.lock
index d83c65c94..af94d5502 100644
--- a/glide.lock
+++ b/glide.lock
@@ -1,34 +1,6 @@
-hash: d11cedeb5b4d17ad5bc6d13441cfbb8769c74fbb5fe33839dd0988ee4e73cd4b
-updated: 2017-01-12T14:02:12.015478278-06:00
+hash: 469de49a1736f34a11e9b0e490f7c1da1d8cb0219fed4bf3ad9e71344ca7f58a
+updated: 2017-02-09T17:03:01.816613507-06:00
 imports:
-- name: github.com/aws/aws-sdk-go
-  version: 63ce630574a5ec05ecd8e8de5cea16332a5a684d
-  subpackages:
-  - aws
-  - aws/awserr
-  - aws/awsutil
-  - aws/client
-  - aws/client/metadata
-  - aws/corehandlers
-  - aws/credentials
-  - aws/credentials/ec2rolecreds
-  - aws/credentials/endpointcreds
-  - aws/credentials/stscreds
-  - aws/defaults
-  - aws/ec2metadata
-  - aws/endpoints
-  - aws/request
-  - aws/session
-  - aws/signer/v4
-  - private/protocol
-  - private/protocol/query
-  - private/protocol/query/queryutil
-  - private/protocol/rest
-  - private/protocol/restxml
-  - private/protocol/xml/xmlutil
-  - private/waiter
-  - service/s3
-  - service/sts
 - name: github.com/boltdb/bolt
   version: 4b1ebc1869ad66568b313d0dc410e2be72670dda
 - name: github.com/BurntSushi/toml
@@ -41,8 +13,6 @@ imports:
   version: 346938d642f2ec3594ed81d874461961cd0faa76
   subpackages:
   - spew
-- name: github.com/go-ini/ini
-  version: 6f66b0e091edb3c7b380f7c4f0f884274d550b67
 - name: github.com/gogo/protobuf
   version: a9cd0c35b97daf74d0ebf3514c5254814b2703b4
   subpackages:
@@ -52,21 +22,9 @@ imports:
   subpackages:
   - lru
 - name: github.com/golang/protobuf
-  version: 8ee79997227bf9b34611aee7946ae64735e6fd93
-  subpackages:
-  - proto
-- name: github.com/jmespath/go-jmespath
-  version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d
+  version: 888eb0692c857ec880338addf316bd662d5e630e
 - name: github.com/satori/go.uuid
   version: 879c5887cd475cd7864858769793b2ceb0d44feb
-- name: golang.org/x/crypto
-  version: c3b1d0d6d8690eaebe3064711b026770cc37efa3
-  subpackages:
-  - curve25519
-  - ed25519
-  - ed25519/internal/edwards25519
-  - ssh
-  - ssh/agent
 - name: golang.org/x/sys
   version: c200b10b5d5e122be351b67af224adc6128af5bf
   subpackages:
diff --git a/glide.yaml b/glide.yaml
index 0f87ca297..10c8509d0 100644
--- a/glide.yaml
+++ b/glide.yaml
@@ -27,5 +27,3 @@ import:
 - package: github.com/golang/protobuf
 - package: github.com/satori/go.uuid
   version: ^1.1.0
-- package: github.com/aws/aws-sdk-go
-  version: ^1.6.10
diff --git a/ssh/ssh.go b/ssh/ssh.go
deleted file mode 100644
index f6121fec9..000000000
--- a/ssh/ssh.go
+++ /dev/null
@@ -1,225 +0,0 @@
-package ssh
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"net"
-	"os"
-	"os/user"
-	"strings"
-	"syscall"
-
-	"golang.org/x/crypto/ssh"
-	"golang.org/x/crypto/ssh/agent"
-	"golang.org/x/crypto/ssh/terminal"
-)
-
-type Client struct {
-	client   *ssh.Client
-	Stderr   io.Writer
-	password string
-}
-
-// NewClient wraps up some of the complexity of using the crypto/ssh pacakge
-// directly assuming you want to connect using public key auth and you can pass
-// a keyfile or your key is accessible through ssh agent.
-func NewClient(host, username, keyfile string, stderr io.Writer) (*Client, error) {
-	if username == "" {
-		user, err := user.Current()
-		if err != nil {
-			return nil, err
-		}
-		username = user.Username
-	}
-
-	var authkey ssh.AuthMethod
-	if keyfile == "" {
-		sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
-		if err != nil {
-			return nil, err
-		}
-		authkey = ssh.PublicKeysCallback(agent.NewClient(sshAgent).Signers)
-	} else {
-		return nil, fmt.Errorf("using a keyfile is unimplemented")
-	}
-
-	authpw := ssh.PasswordCallback(func() (secret string, err error) {
-		fmt.Fprintf(stderr, "password for %v@%v: ", username, host)
-		pwbytes, err := terminal.ReadPassword(syscall.Stdin)
-		fmt.Fprintln(stderr)
-		return string(pwbytes), err
-	})
-
-	config := &ssh.ClientConfig{
-		User: username,
-		Auth: []ssh.AuthMethod{authkey, authpw},
-	}
-
-	if strings.Index(host, ":") == -1 {
-		host = host + ":22"
-	}
-
-	client, err := ssh.Dial("tcp", host, config)
-	if err != nil {
-		return nil, fmt.Errorf("NewSSH failed Dial - host: %v, config: %v, err: %v ", host, config, err)
-	}
-
-	return &Client{client: client, Stderr: stderr}, nil
-}
-
-type Fleet map[string]*Client
-
-func NewFleet(hosts []string, username, keyfile string, stderr io.Writer) (Fleet, error) {
-	hosts = DedupHosts(hosts)
-	clients := make(map[string]*Client, len(hosts))
-	for _, host := range hosts {
-		client, err := NewClient(host, username, keyfile, stderr)
-		if err != nil {
-			return nil, err
-		}
-		clients[host] = client
-	}
-	return clients, nil
-}
-
-// DedupHosts takes a slice of hosts, strips off any specified ports, and
-// returns a de-duplicated slice of hosts.
-func DedupHosts(hosts []string) []string {
-	seenHosts := make(map[string]bool)
-	ret := []string{}
-	for _, h := range hosts {
-		colonIdx := strings.Index(h, ":")
-		if colonIdx != -1 {
-			h = h[:colonIdx]
-		}
-		if !seenHosts[h] {
-			ret = append(ret, h)
-		}
-		seenHosts[h] = true
-	}
-	return ret
-}
-
-func (s *Client) NewSession() (*ssh.Session, error) {
-	return s.client.NewSession()
-}
-
-type remoteFile struct {
-	w    io.WriteCloser
-	sess *ssh.Session
-}
-
-func (r *remoteFile) Write(p []byte) (n int, err error) {
-	return r.w.Write(p)
-}
-
-func (r *remoteFile) Close() error {
-	errc := r.w.Close()
-	errw := r.sess.Wait()
-	if errc != nil || errw != nil {
-		return fmt.Errorf("error closing remote file - close: '%v', wait: '%v'", errc, errw)
-	}
-	return nil
-}
-
-// OpenFile creates or truncates an existing file of the given name on the
-// remote host, and returns a WriteCloser which will write to that file. perm
-// will be passed directly to chmod to set the file permissions. rm, touch,
-// chmod, cat and support for semicolons, double ampersand, and output
-// redirection (>>) must be available in the remote shell.
-func (s *Client) OpenFile(name, perm string) (io.WriteCloser, error) {
-	sess, err := s.NewSession()
-	if err != nil {
-		return nil, err
-	}
-	w, err := sess.StdinPipe()
-	if err != nil {
-		return nil, err
-	}
-	if perm == "" {
-		perm = "0664"
-	}
-	err = sess.Start(fmt.Sprintf("rm %v; touch %v && chmod %v %v && cat >> %v", name, name, perm, name, name))
-	if err != nil {
-		return nil, err
-	}
-
-	return &remoteFile{w: w, sess: sess}, nil
-}
-
-type multiWriteCloser struct {
-	wcs []io.WriteCloser
-	ws  []io.Writer
-}
-
-func newMultiWriteCloser() *multiWriteCloser {
-	return &multiWriteCloser{
-		wcs: make([]io.WriteCloser, 0),
-		ws:  make([]io.Writer, 0),
-	}
-}
-
-func (mwc *multiWriteCloser) add(wc io.WriteCloser) {
-	mwc.wcs = append(mwc.wcs, wc)
-	mwc.ws = append(mwc.ws, wc)
-}
-
-func (mwc *multiWriteCloser) Write(p []byte) (n int, err error) {
-	mw := io.MultiWriter(mwc.ws...)
-	return mw.Write(p)
-}
-
-func (mwc *multiWriteCloser) Close() error {
-	errStr := ""
-	for _, wc := range mwc.wcs {
-		err := wc.Close()
-		if err != nil {
-			errStr = errStr + "; " + err.Error()
-		}
-	}
-	if errStr != "" {
-		return errors.New(errStr)
-	}
-	return nil
-}
-
-func (sf Fleet) OpenFile(name, perm string) (io.WriteCloser, error) {
-	writers := newMultiWriteCloser()
-	for _, cli := range sf {
-		wc, err := cli.OpenFile(name, "+x")
-		if err != nil {
-			return nil, err
-		}
-		writers.add(wc)
-	}
-	return writers, nil
-}
-
-// WriteFile writes all of data (until EOF) into a file with the given name on
-// each of the hosts in the fleet. It sets the permissions on the file to <perm>
-// which is any valid input to chmod. If <perm> is the empty string, it defaults
-// to 0664
-func (sf Fleet) WriteFile(name, perm string, data io.Reader) error {
-	wc, err := sf.OpenFile(name, perm)
-	if err != nil {
-		return fmt.Errorf("opening: %v", err)
-	}
-
-	_, err = io.Copy(wc, data)
-	if err != nil {
-		return fmt.Errorf("copying: %v", err)
-	}
-	return wc.Close()
-}
-
-func (sf Fleet) Get(host string) (*Client, error) {
-	colonIdx := strings.Index(host, ":")
-	if colonIdx != -1 {
-		host = host[:colonIdx]
-	}
-	if client, ok := sf[host]; ok {
-		return client, nil
-	}
-	return nil, fmt.Errorf("No client found in fleet: %v for host: %v", sf, host)
-}

From bcff80abad0627b4e59d5e8bcfb0fd51ec52caa5 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 24 Feb 2017 16:13:46 -0600
Subject: [PATCH 10/37] remove some leftovers from benchmark removal

---
 bench/query.go     | 197 ---------------------------------------------
 bench/randquery.go | 119 ---------------------------
 2 files changed, 316 deletions(-)
 delete mode 100644 bench/query.go
 delete mode 100644 bench/randquery.go

diff --git a/bench/query.go b/bench/query.go
deleted file mode 100644
index 166ef17ad..000000000
--- a/bench/query.go
+++ /dev/null
@@ -1,197 +0,0 @@
-package bench
-
-import (
-	"math/rand"
-
-	"github.com/pilosa/pilosa/pql"
-)
-
-// NewQueryGenerator initializes a new QueryGenerator
-func NewQueryGenerator(seed int64) *QueryGenerator {
-	return &QueryGenerator{
-		IDToFrameFn: func(id uint64) string { return "frame.n" },
-		R:           rand.New(rand.NewSource(seed)),
-		Frames:      []string{"frame.n"},
-	}
-}
-
-// QueryGenerator holds the configuration and state for randomly generating
-// queries.
-type QueryGenerator struct {
-	IDToFrameFn func(id uint64) string
-	R           *rand.Rand
-	Frames      []string
-}
-
-// Random returns a randomly generated query.
-func (q *QueryGenerator) Random(maxN, depth, maxargs int, idmin, idmax uint64) *pql.Call {
-	// TODO: handle depth==1 or 0
-	val := q.R.Intn(5)
-	switch val {
-	case 0:
-		return q.RandomTopN(maxN, depth, maxargs, idmin, idmax)
-	default:
-		return q.RandomBitmapCall(depth, maxargs, idmin, idmax)
-	}
-}
-
-// RandomTopN returns a randomly generated TopN query.
-func (q *QueryGenerator) RandomTopN(maxN, depth, maxargs int, idmin, idmax uint64) *pql.Call {
-	frameIdx := q.R.Intn(len(q.Frames))
-	return &pql.Call{
-		Args: map[string]interface{}{
-			"frame": q.Frames[frameIdx],
-			"n":     uint64(q.R.Intn(maxN-1) + 1),
-		},
-		Children: []*pql.Call{q.RandomBitmapCall(depth, maxargs, idmin, idmax)},
-	}
-}
-
-// RandomBitmapCall returns a randomly generate query which returns a bitmap.
-func (q *QueryGenerator) RandomBitmapCall(depth, maxargs int, idmin, idmax uint64) *pql.Call {
-	if depth <= 1 {
-		bitmapID := q.R.Int63n(int64(idmax)-int64(idmin)) + int64(idmin)
-		return Bitmap(uint64(bitmapID), q.IDToFrameFn(uint64(bitmapID)))
-	}
-	call := q.R.Intn(4)
-	if call == 0 {
-		return q.RandomBitmapCall(1, 0, idmin, idmax)
-	}
-
-	var numargs int
-	if maxargs <= 2 {
-		numargs = 2
-	} else {
-		numargs = q.R.Intn(maxargs-2) + 2
-	}
-	calls := make([]*pql.Call, numargs)
-	for i := 0; i < numargs; i++ {
-		calls[i] = q.RandomBitmapCall(depth-1, maxargs, idmin, idmax)
-	}
-
-	switch call {
-	case 1:
-		return Difference(calls...)
-	case 2:
-		return Intersect(calls...)
-	case 3:
-		return Union(calls...)
-	}
-	return nil
-}
-
-///////////////////////////////////////////////////
-// Helpers TODO: move elsewhere
-///////////////////////////////////////////////////
-
-func ClearBit(id uint64, frame string, profileID uint64) *pql.Call {
-	return &pql.Call{
-		Name: "ClearBit",
-		Args: map[string]interface{}{
-			"id":        id,
-			"frame":     frame,
-			"profileID": profileID,
-		},
-	}
-}
-
-func Count(child *pql.Call) *pql.Call {
-	return &pql.Call{
-		Name:     "Count",
-		Children: []*pql.Call{child},
-	}
-}
-
-func Profile(id uint64) *pql.Call {
-	return &pql.Call{
-		Name: "Profile",
-		Args: map[string]interface{}{"id": id},
-	}
-}
-
-func SetBit(id uint64, frame string, profileID uint64) *pql.Call {
-	return &pql.Call{
-		Name: "SetBit",
-		Args: map[string]interface{}{
-			"id":        id,
-			"frame":     frame,
-			"profileID": profileID,
-		},
-	}
-}
-
-func SetBitmapAttrs(id uint64, frame string, attrs map[string]interface{}) *pql.Call {
-	args := copyArgs(attrs)
-	args["id"] = id
-	args["profileID"] = frame
-
-	return &pql.Call{
-		Name: "SetBitmapAttrs",
-		Args: args,
-	}
-}
-
-func SetProfileAttrs(id uint64, attrs map[string]interface{}) *pql.Call {
-	args := copyArgs(attrs)
-	args["id"] = id
-
-	return &pql.Call{
-		Name: "SetProfileAttrs",
-		Args: args,
-	}
-}
-
-func TopN(frame string, n int, src *pql.Call, bmids []uint64, field string, filters []interface{}) *pql.Call {
-	return &pql.Call{
-		Name:     "TopN",
-		Children: []*pql.Call{src},
-		Args: map[string]interface{}{
-			"frame":   frame,
-			"n":       n,
-			"ids":     bmids,
-			"field":   field,
-			"filters": filters,
-		},
-	}
-}
-
-func Difference(bms ...*pql.Call) *pql.Call {
-	// TODO does this need to be limited to two inputs?
-	return &pql.Call{
-		Name:     "Difference",
-		Children: bms,
-	}
-}
-
-func Intersect(bms ...*pql.Call) *pql.Call {
-	return &pql.Call{
-		Name:     "Intersect",
-		Children: bms,
-	}
-}
-
-func Union(bms ...*pql.Call) *pql.Call {
-	return &pql.Call{
-		Name:     "Union",
-		Children: bms,
-	}
-}
-
-func Bitmap(id uint64, frame string) *pql.Call {
-	return &pql.Call{
-		Name: "Bitmap",
-		Args: map[string]interface{}{
-			"id":    id,
-			"frame": frame,
-		},
-	}
-}
-
-// copyArgs returns a shallow copy of m.
-func copyArgs(m map[string]interface{}) map[string]interface{} {
-	other := make(map[string]interface{}, len(m))
-	for k, v := range m {
-		other[k] = v
-	}
-	return other
-}
diff --git a/bench/randquery.go b/bench/randquery.go
deleted file mode 100644
index 8bdc98e74..000000000
--- a/bench/randquery.go
+++ /dev/null
@@ -1,119 +0,0 @@
-package bench
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"strings"
-	"time"
-)
-
-// RandomQuery queries randomly and deterministically based on a seed.
-type RandomQuery struct {
-	HasClient
-	Name          string   `json:"name"`
-	MaxDepth      int      `json:"max-depth"`
-	MaxArgs       int      `json:"max-args"`
-	MaxN          int      `json:"max-n"`
-	BaseBitmapID  int64    `json:"base-bitmap-id"`
-	BitmapIDRange int64    `json:"bitmap-id-range"`
-	Iterations    int      `json:"iterations"`
-	Seed          int64    `json:"seed"`
-	DBs           []string `json:"dbs"`
-}
-
-// Init adds the agent num to the random seed and initializes the client.
-func (b *RandomQuery) Init(hosts []string, agentNum int) error {
-	b.Name = "random-query"
-	b.Seed = b.Seed + int64(agentNum)
-	return b.HasClient.Init(hosts, agentNum)
-}
-
-// Usage returns the usage message to be printed.
-func (b *RandomQuery) Usage() string {
-	return `
-random-query constructs random queries
-
-Agent number modifies the random seed.
-
-Usage: random-query [arguments]
-
-The following arguments are available:
-
-	-max-depth int
-		Maximum nesting depth of queries
-
-	-max-args int
-		Maximum number of args for Union/Intersect/Difference Queries
-
-	-max-n int
-		Maximum N value for TopN queries.
-
-	-base-bitmap-id int
-		bitmap id to start from
-
-	-bitmap-id-range int
-		number of possible bitmap ids that can be set
-
-	-iterations int
-		number of bits to set
-
-	-seed int
-		Seed for RNG
-
-	-dbs string
-		Comma separated list of DBs to query against
-
-	-client-type string
-		Can be 'single' (all agents hitting one host) or 'round_robin'
-
-	-content-type string
-		protobuf or pql
-`[1:]
-}
-
-// ConsumeFlags parses all flags up to the next non flag argument (argument does
-// not start with "-" and isn't the value of a flag). It returns the remaining
-// args.
-func (b *RandomQuery) ConsumeFlags(args []string) ([]string, error) {
-	fs := flag.NewFlagSet("RandomQuery", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	fs.IntVar(&b.MaxDepth, "max-depth", 4, "")
-	fs.IntVar(&b.MaxArgs, "max-args", 4, "")
-	fs.IntVar(&b.MaxN, "max-n", 4, "")
-	fs.Int64Var(&b.BaseBitmapID, "base-bitmap-id", 0, "")
-	fs.Int64Var(&b.BitmapIDRange, "bitmap-id-range", 100000, "")
-	fs.Int64Var(&b.Seed, "seed", 1, "")
-	fs.IntVar(&b.Iterations, "iterations", 100, "")
-	var dbs string
-	fs.StringVar(&dbs, "dbs", "benchdb", "")
-	fs.StringVar(&b.ClientType, "client-type", "single", "")
-	fs.StringVar(&b.ContentType, "content-type", "protobuf", "")
-
-	if err := fs.Parse(args); err != nil {
-		return nil, err
-	}
-	b.DBs = strings.Split(dbs, ",")
-	return fs.Args(), nil
-}
-
-// Run runs the RandomQuery benchmark
-func (b *RandomQuery) Run(ctx context.Context) map[string]interface{} {
-	results := make(map[string]interface{})
-	if b.client == nil {
-		results["error"] = fmt.Errorf("No client set for RandomQuery")
-		return results
-	}
-	qm := NewQueryGenerator(b.Seed)
-	s := NewStats()
-	var start time.Time
-	for n := 0; n < b.Iterations; n++ {
-		call := qm.Random(b.MaxN, b.MaxDepth, b.MaxArgs, uint64(b.BaseBitmapID), uint64(b.BitmapIDRange))
-		start = time.Now()
-		b.ExecuteQuery(b.ContentType, b.DBs[n%len(b.DBs)], call.String(), ctx)
-		s.Add(time.Now().Sub(start))
-	}
-	AddToResults(s, results)
-	return results
-}

From b27a93538125be819ba18a09b119f3d3ff7b2445 Mon Sep 17 00:00:00 2001
From: Cody Soyland <cody@soyland.org>
Date: Mon, 27 Feb 2017 14:22:14 -0600
Subject: [PATCH 11/37] Using Travis addons instead of before_install to
 continue using container-based infrastructure

---
 .travis.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 56dec77f7..4d5c27fd2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,10 +2,12 @@ language: go
 go:
   - 1.7.x
   - master
-before_install:
-  - sudo add-apt-repository ppa:masterminds/glide -y
-  - sudo apt-get update -q
-  - sudo apt-get install glide -y
+addons:
+  apt:
+    sources:
+      - sourceline: 'ppa:masterminds/glide'
+    packages:
+      - glide
 script: make test
 notifications:
   slack:

From 69cfdb0df9fd8f2394e46058e7fcda84cc3e85ea Mon Sep 17 00:00:00 2001
From: Linh Vo <lvo@umbel.com>
Date: Thu, 2 Mar 2017 16:06:34 -0600
Subject: [PATCH 12/37] clean up

---
 cmd/pilosactl/main.go | 2 --
 db_test.go            | 2 +-
 fragment.go           | 6 +++---
 frame.go              | 2 +-
 frame_test.go         | 2 +-
 pilosa.go             | 3 +--
 6 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go
index 5491dbfca..bb22d14ac 100644
--- a/cmd/pilosactl/main.go
+++ b/cmd/pilosactl/main.go
@@ -303,7 +303,6 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
 	logger := log.New(cmd.Stderr, "", log.LstdFlags)
 
 	// Validate arguments.
-	fmt.Print("1213224")
 	if cmd.Database == "" {
 		return pilosa.ErrDatabaseRequired
 	} else if cmd.Frame == "" {
@@ -973,7 +972,6 @@ func (cmd *BenchCommand) Run(ctx context.Context) error {
 
 // runSetBit executes a benchmark of random SetBit() operations.
 func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) error {
-	fmt.Print("Nodb")
 	if cmd.N == 0 {
 		return errors.New("operation count required")
 	} else if cmd.Database == "" {
diff --git a/db_test.go b/db_test.go
index f0bdc60e2..96120dc1e 100644
--- a/db_test.go
+++ b/db_test.go
@@ -149,4 +149,4 @@ func TestDB_InvalidName(t *testing.T) {
 	if db != nil {
 		t.Fatalf("unexpected db name %s", db)
 	}
-}
\ No newline at end of file
+}
diff --git a/fragment.go b/fragment.go
index 9bc542f93..1181f1739 100644
--- a/fragment.go
+++ b/fragment.go
@@ -213,11 +213,11 @@ func (f *Fragment) openCache() error {
 	// Determine cache type from frame name.
 	if strings.HasSuffix(f.frame, FrameSuffixRank) {
 		c := NewRankCache()
-		c.ThresholdLength = 500000
-		c.ThresholdIndex = 450000
+		c.ThresholdLength = 50000
+		c.ThresholdIndex = 45000
 		f.cache = c
 	} else {
-		f.cache = NewLRUCache(500000)
+		f.cache = NewLRUCache(50000)
 	}
 
 	// Read cache data from disk.
diff --git a/frame.go b/frame.go
index 8ec419057..fa47517c5 100644
--- a/frame.go
+++ b/frame.go
@@ -46,7 +46,7 @@ type Frame struct {
 }
 
 // NewFrame returns a new instance of frame.
-func NewFrame(path, db, name string) (*Frame, error){
+func NewFrame(path, db, name string) (*Frame, error) {
 	err := ValidateName(name)
 	if err != nil {
 		return nil, err
diff --git a/frame_test.go b/frame_test.go
index 34f03ee15..042bd751e 100644
--- a/frame_test.go
+++ b/frame_test.go
@@ -96,7 +96,7 @@ func (f *Frame) Reopen() error {
 
 	path, db, name := f.Path(), f.DB(), f.Name()
 	f.Frame, err = pilosa.NewFrame(path, db, name)
-	if err != nil{
+	if err != nil {
 		return err
 	}
 
diff --git a/pilosa.go b/pilosa.go
index f6efb0987..14e5ef203 100644
--- a/pilosa.go
+++ b/pilosa.go
@@ -80,7 +80,6 @@ func decodeProfile(pb *internal.Profile) *Profile {
 // TimeFormat is the go-style time format used to parse string dates.
 const TimeFormat = "2006-01-02T15:04"
 
-
 // Restrict name using regex
 func ValidateName(name string) error {
 	expr := regexp.MustCompile(`^([a-z0-9._-]{1,64}$)`)
@@ -89,4 +88,4 @@ func ValidateName(name string) error {
 		return ErrName
 	}
 	return nil
-}
\ No newline at end of file
+}

From 57de42bb91cc0ec09b5045a9e940394f52eb9b09 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Thu, 2 Mar 2017 16:27:45 -0600
Subject: [PATCH 13/37] fix empty queries crashing pilosa

---
 executor.go      |  8 +++++++-
 executor_test.go | 37 +++++++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/executor.go b/executor.go
index f29fdfe9f..0ba1e7017 100644
--- a/executor.go
+++ b/executor.go
@@ -265,6 +265,9 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call,
 // executeDifferenceSlice executes a difference() call for a local slice.
 func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
 	var other *Bitmap
+	if len(c.Children) == 0 {
+		return nil, fmt.Errorf("empty Difference query is currently not supported")
+	}
 	for i, input := range c.Children {
 		bm, err := e.executeBitmapCallSlice(ctx, db, input, slice)
 		if err != nil {
@@ -308,6 +311,9 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal
 // executeIntersectSlice executes a intersect() call for a local slice.
 func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
 	var other *Bitmap
+	if len(c.Children) == 0 {
+		return nil, fmt.Errorf("empty Intersect query is currently not supported")
+	}
 	for i, input := range c.Children {
 		bm, err := e.executeBitmapCallSlice(ctx, db, input, slice)
 		if err != nil {
@@ -382,7 +388,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call
 
 // executeUnionSlice executes a union() call for a local slice.
 func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
-	var other *Bitmap
+	other := NewBitmap()
 	for i, input := range c.Children {
 		bm, err := e.executeBitmapCallSlice(ctx, db, input, slice)
 		if err != nil {
diff --git a/executor_test.go b/executor_test.go
index 7cc7af2eb..1ad1fbbbb 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -50,6 +50,18 @@ func TestExecutor_Execute_Difference(t *testing.T) {
 	}
 }
 
+// Ensure an empty difference query behaves properly.
+func TestExecutor_Execute_Empty_Difference(t *testing.T) {
+	idx := MustOpenIndex()
+	defer idx.Close()
+	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1)
+
+	e := NewExecutor(idx.Index, NewCluster(1))
+	if res, err := e.Execute(context.Background(), "d", MustParse(`Difference()`), nil, nil); err == nil {
+		t.Fatalf("Empty Difference query should give error, but got %v", res)
+	}
+}
+
 // Ensure an intersect query can be executed.
 func TestExecutor_Execute_Intersect(t *testing.T) {
 	idx := MustOpenIndex()
@@ -70,6 +82,17 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
 	}
 }
 
+// Ensure an empty intersect query behaves properly.
+func TestExecutor_Execute_Empty_Intersect(t *testing.T) {
+	idx := MustOpenIndex()
+	defer idx.Close()
+
+	e := NewExecutor(idx.Index, NewCluster(1))
+	if res, err := e.Execute(context.Background(), "d", MustParse(`Intersect()`), nil, nil); err == nil {
+		t.Fatalf("Empty Intersect query should give error, but got %v", res)
+	}
+}
+
 // Ensure a union query can be executed.
 func TestExecutor_Execute_Union(t *testing.T) {
 	idx := MustOpenIndex()
@@ -89,6 +112,20 @@ func TestExecutor_Execute_Union(t *testing.T) {
 	}
 }
 
+// Ensure an empty union query behaves properly.
+func TestExecutor_Execute_Empty_Union(t *testing.T) {
+	idx := MustOpenIndex()
+	defer idx.Close()
+	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 0)
+
+	e := NewExecutor(idx.Index, NewCluster(1))
+	if res, err := e.Execute(context.Background(), "d", MustParse(`Union()`), nil, nil); err != nil {
+		t.Fatal(err)
+	} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) {
+		t.Fatalf("unexpected bits: %+v", bits)
+	}
+}
+
 // Ensure a count query can be executed.
 func TestExecutor_Execute_Count(t *testing.T) {
 	idx := MustOpenIndex()

From 02c3c6d3e69ccc8ccdc12e0101c5d267ecba4f3d Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 3 Mar 2017 13:03:57 -0600
Subject: [PATCH 14/37] use cobra/viper and move cmd/pilosa to server
 subcommand

---
 cmd/pilosa/main.go                            | 190 +----------------
 cmd/root.go                                   |  14 ++
 cmd/server.go                                 |  90 ++++++++
 glide.lock                                    |  50 ++++-
 glide.yaml                                    |   2 +
 server/server.go                              | 197 ++++++++++++++++++
 .../main_test.go => server/server_test.go     |  10 +-
 7 files changed, 360 insertions(+), 193 deletions(-)
 create mode 100644 cmd/root.go
 create mode 100644 cmd/server.go
 create mode 100644 server/server.go
 rename cmd/pilosa/main_test.go => server/server_test.go (99%)

diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go
index 4f7474996..255aef606 100644
--- a/cmd/pilosa/main.go
+++ b/cmd/pilosa/main.go
@@ -1,197 +1,15 @@
 package main
 
 import (
-	"errors"
-	"flag"
 	"fmt"
-	"io"
-	"math/rand"
 	"os"
-	"os/signal"
-	"path/filepath"
-	"runtime/pprof"
-	"strings"
-	"time"
 
-	"github.com/BurntSushi/toml"
-	"github.com/pilosa/pilosa"
-)
-
-// Version and BuildTime hold the version/build time information passed in at compile time.
-var (
-	Version   string
-	BuildTime string
-)
-
-func init() {
-	if Version == "" {
-		Version = "v0.0.0"
-	}
-	if BuildTime == "" {
-		BuildTime = "not recorded"
-	}
-
-	rand.Seed(time.Now().UTC().UnixNano())
-}
-
-const (
-	// DefaultDataDir is the default data directory.
-	DefaultDataDir = "~/.pilosa"
+	"github.com/pilosa/pilosa/cmd"
 )
 
 func main() {
-	m := NewMain()
-	m.Server.Handler.Version = Version
-	fmt.Fprintf(m.Stderr, "Pilosa %s, build time %s\n", Version, BuildTime)
-
-	// Parse command line arguments.
-	if err := m.ParseFlags(os.Args[1:]); err != nil {
-		fmt.Fprintln(m.Stderr, err)
-		os.Exit(2)
-	}
-
-	// Start CPU profiling.
-	if m.CPUProfile != "" {
-		f, err := os.Create(m.CPUProfile)
-		if err != nil {
-			fmt.Fprintf(m.Stderr, "create cpu profile: %v", err)
-			os.Exit(1)
-		}
-		defer f.Close()
-
-		fmt.Fprintln(m.Stderr, "Starting cpu profile")
-		pprof.StartCPUProfile(f)
-		time.AfterFunc(m.CPUTime, func() {
-			fmt.Fprintln(m.Stderr, "Stopping cpu profile")
-			pprof.StopCPUProfile()
-			f.Close()
-		})
-	}
-
-	// Execute the program.
-	if err := m.Run(); err != nil {
-		fmt.Fprintln(m.Stderr, err)
-		fmt.Fprintln(m.Stderr, "stopping profile")
-		os.Exit(1)
-	}
-
-	// First SIGKILL causes server to shut down gracefully.
-	c := make(chan os.Signal, 2)
-	signal.Notify(c, os.Interrupt)
-	sig := <-c
-	fmt.Fprintf(m.Stderr, "Received %s; gracefully shutting down...\n", sig.String())
-
-	// Second signal causes a hard shutdown.
-	go func() { <-c; os.Exit(1) }()
-
-	if err := m.Close(); err != nil {
-		fmt.Fprintln(m.Stderr, err)
-		os.Exit(1)
+	if err := cmd.RootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(-1)
 	}
 }
-
-// Main represents the main program execution.
-type Main struct {
-	Server *pilosa.Server
-
-	// Configuration options.
-	ConfigPath string
-	Config     *pilosa.Config
-
-	// Profiling options.
-	CPUProfile string
-	CPUTime    time.Duration
-
-	// Standard input/output
-	Stdin  io.Reader
-	Stdout io.Writer
-	Stderr io.Writer
-}
-
-// NewMain returns a new instance of Main.
-func NewMain() *Main {
-	return &Main{
-		Server: pilosa.NewServer(),
-		Config: pilosa.NewConfig(),
-
-		Stdin:  os.Stdin,
-		Stdout: os.Stdout,
-		Stderr: os.Stderr,
-	}
-}
-
-// Run executes the main program execution.
-func (m *Main) Run(args ...string) error {
-	// Notify user of config file.
-	if m.ConfigPath != "" {
-		fmt.Fprintf(m.Stdout, "Using config: %s\n", m.ConfigPath)
-	}
-
-	// Setup logging output.
-	m.Server.LogOutput = m.Stderr
-
-	// Configure index.
-	fmt.Fprintf(m.Stderr, "Using data from: %s\n", m.Config.DataDir)
-	m.Server.Index.Path = m.Config.DataDir
-	m.Server.Index.Stats = pilosa.NewExpvarStatsClient()
-
-	// Build cluster from config file.
-	m.Server.Host = m.Config.Host
-	m.Server.Cluster = m.Config.PilosaCluster()
-
-	// Set configuration options.
-	m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval)
-
-	// Initialize server.
-	if err := m.Server.Open(); err != nil {
-		return err
-	}
-
-	fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host)
-
-	return nil
-}
-
-// Close shuts down the server.
-func (m *Main) Close() error {
-	return m.Server.Close()
-}
-
-// ParseFlags parses command line flags from args.
-func (m *Main) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosa", flag.ContinueOnError)
-	fs.StringVar(&m.CPUProfile, "cpuprofile", "", "cpu profile")
-	fs.DurationVar(&m.CPUTime, "cputime", 30*time.Second, "cpu profile duration")
-	fs.StringVar(&m.ConfigPath, "config", "", "config path")
-	fs.SetOutput(m.Stderr)
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	// Load config, if specified.
-	if m.ConfigPath != "" {
-		if _, err := toml.DecodeFile(m.ConfigPath, &m.Config); err != nil {
-			return err
-		}
-	}
-
-	// Use default data directory if one is not specified.
-	if m.Config.DataDir == "" {
-		m.Config.DataDir = DefaultDataDir
-	}
-
-	// Expand home directory.
-	prefix := "~" + string(filepath.Separator)
-	if strings.HasPrefix(m.Config.DataDir, prefix) {
-		//	u, err := user.Current()
-		HomeDir := os.Getenv("HOME")
-		/*if err != nil {
-			return err
-		} else*/if HomeDir == "" {
-			return errors.New("data directory not specified and no home dir available")
-		}
-		m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix))
-	}
-
-	return nil
-}
diff --git a/cmd/root.go b/cmd/root.go
new file mode 100644
index 000000000..91d866a18
--- /dev/null
+++ b/cmd/root.go
@@ -0,0 +1,14 @@
+package cmd
+
+import "github.com/spf13/cobra"
+
+var RootCmd = &cobra.Command{
+	Use:   "pilosa",
+	Short: "pilosa - A Distributed In-memory Binary Bitmap Index",
+	Long: `Pilosa is a fast index to turbocharge your database.
+
+This binary contains Pilosa itself, as well as common
+tools for administering pilosa, importing/exporting data,
+backing up, and more. Complete documentation is available
+at http://pilosa.com/docs`, // TODO - is documentation actually there?
+}
diff --git a/cmd/server.go b/cmd/server.go
new file mode 100644
index 000000000..cc8bbec49
--- /dev/null
+++ b/cmd/server.go
@@ -0,0 +1,90 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"runtime/pprof"
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/pilosa/pilosa/server"
+)
+
+var serve = server.NewMain()
+
+var serveCmd = &cobra.Command{
+	Use:   "server",
+	Short: "server - run the pilosa server",
+	Long: `pilosa server runs Pilosa.
+
+It will load existing data from the configured
+directory, and start listening client connections
+on the configured port.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		serve.Server.Handler.Version = server.Version
+		fmt.Fprintf(serve.Stderr, "Pilosa %s, build time %s\n", server.Version, server.BuildTime)
+
+		// Parse command line arguments.
+		if err := serve.ParseFlags(os.Args[1:]); err != nil {
+			fmt.Fprintln(serve.Stderr, err)
+			os.Exit(2)
+		}
+
+		// Start CPU profiling.
+		if serve.CPUProfile != "" {
+			f, err := os.Create(serve.CPUProfile)
+			if err != nil {
+				fmt.Fprintf(serve.Stderr, "create cpu profile: %v", err)
+				os.Exit(1)
+			}
+			defer f.Close()
+
+			fmt.Fprintln(serve.Stderr, "Starting cpu profile")
+			pprof.StartCPUProfile(f)
+			time.AfterFunc(serve.CPUTime, func() {
+				fmt.Fprintln(serve.Stderr, "Stopping cpu profile")
+				pprof.StopCPUProfile()
+				f.Close()
+			})
+		}
+
+		// Execute the program.
+		if err := serve.Run(); err != nil {
+			fmt.Fprintln(serve.Stderr, err)
+			fmt.Fprintln(serve.Stderr, "stopping profile")
+			os.Exit(1)
+		}
+
+		// First SIGKILL causes server to shut down gracefully.
+		c := make(chan os.Signal, 2)
+		signal.Notify(c, os.Interrupt)
+		sig := <-c
+		fmt.Fprintf(serve.Stderr, "Received %s; gracefully shutting down...\n", sig.String())
+
+		// Second signal causes a hard shutdown.
+		go func() { <-c; os.Exit(1) }()
+
+		if err := serve.Close(); err != nil {
+			fmt.Fprintln(serve.Stderr, err)
+			os.Exit(1)
+		}
+
+	},
+}
+
+func init() {
+	serveCmd.Flags().StringVarP(&serve.ConfigPath, "config", "c", "", "Configuration file to read from")
+	serveCmd.Flags().StringVarP(&serve.CPUProfile, "cpuprofile", "", "", "Where to store CPU profile")
+	serveCmd.Flags().DurationVarP(&serve.CPUTime, "cputime", "", 30*time.Second, "CPU profile duration")
+
+	err := viper.BindPFlags(serveCmd.Flags())
+	if err != nil {
+		log.Fatalf("Error binding server flags: %v", err)
+	}
+
+	RootCmd.AddCommand(serveCmd)
+}
diff --git a/glide.lock b/glide.lock
index af94d5502..001257bf1 100644
--- a/glide.lock
+++ b/glide.lock
@@ -1,5 +1,5 @@
-hash: 469de49a1736f34a11e9b0e490f7c1da1d8cb0219fed4bf3ad9e71344ca7f58a
-updated: 2017-02-09T17:03:01.816613507-06:00
+hash: 7de62dbaf3cc1dc4959f4f6d8213102cb182b4dd7a87b3ac29260ad6bc1b0cef
+updated: 2017-03-03T12:25:48.088390296-06:00
 imports:
 - name: github.com/boltdb/bolt
   version: 4b1ebc1869ad66568b313d0dc410e2be72670dda
@@ -13,6 +13,8 @@ imports:
   version: 346938d642f2ec3594ed81d874461961cd0faa76
   subpackages:
   - spew
+- name: github.com/fsnotify/fsnotify
+  version: 7d7316ed6e1ed2de075aab8dfc76de5d158d66e1
 - name: github.com/gogo/protobuf
   version: a9cd0c35b97daf74d0ebf3514c5254814b2703b4
   subpackages:
@@ -23,10 +25,54 @@ imports:
   - lru
 - name: github.com/golang/protobuf
   version: 888eb0692c857ec880338addf316bd662d5e630e
+  subpackages:
+  - proto
+- name: github.com/hashicorp/hcl
+  version: 630949a3c5fa3c613328e1b8256052cbc2327c9b
+  subpackages:
+  - hcl/ast
+  - hcl/parser
+  - hcl/scanner
+  - hcl/strconv
+  - hcl/token
+  - json/parser
+  - json/scanner
+  - json/token
+- name: github.com/inconshreveable/mousetrap
+  version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
+- name: github.com/magiconair/properties
+  version: b3b15ef068fd0b17ddf408a23669f20811d194d2
+- name: github.com/mitchellh/mapstructure
+  version: db1efb556f84b25a0a13a04aad883943538ad2e0
+- name: github.com/pelletier/go-buffruneio
+  version: c37440a7cf42ac63b919c752ca73a85067e05992
+- name: github.com/pelletier/go-toml
+  version: 13d49d4606eb801b8f01ae542b4afc4c6ee3d84a
 - name: github.com/satori/go.uuid
   version: 879c5887cd475cd7864858769793b2ceb0d44feb
+- name: github.com/spf13/afero
+  version: 9be650865eab0c12963d8753212f4f9c66cdcf12
+  subpackages:
+  - mem
+- name: github.com/spf13/cast
+  version: 4f1683a2242a92e62d6ff705a30e435cbf2b50a3
+- name: github.com/spf13/cobra
+  version: fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c
+- name: github.com/spf13/jwalterweatherman
+  version: fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66
+- name: github.com/spf13/pflag
+  version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7
+- name: github.com/spf13/viper
+  version: 7538d73b4eb9511d85a9f1dfef202eeb8ac260f4
 - name: golang.org/x/sys
   version: c200b10b5d5e122be351b67af224adc6128af5bf
   subpackages:
   - unix
+- name: golang.org/x/text
+  version: 5a42fa2464759cbb7ee0af9de00b54d69f09a29c
+  subpackages:
+  - transform
+  - unicode/norm
+- name: gopkg.in/yaml.v2
+  version: a3f3340b5840cee44f372bddb5880fcbc419b46a
 testImports: []
diff --git a/glide.yaml b/glide.yaml
index 10c8509d0..1c4f35dae 100644
--- a/glide.yaml
+++ b/glide.yaml
@@ -27,3 +27,5 @@ import:
 - package: github.com/golang/protobuf
 - package: github.com/satori/go.uuid
   version: ^1.1.0
+- package: github.com/spf13/cobra
+- package: github.com/spf13/viper
diff --git a/server/server.go b/server/server.go
new file mode 100644
index 000000000..bb151a37d
--- /dev/null
+++ b/server/server.go
@@ -0,0 +1,197 @@
+package server
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime/pprof"
+	"strings"
+	"time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/pilosa/pilosa"
+)
+
+// Version and BuildTime hold the version/build time information passed in at compile time.
+var (
+	Version   string
+	BuildTime string
+)
+
+func init() {
+	if Version == "" {
+		Version = "v0.0.0"
+	}
+	if BuildTime == "" {
+		BuildTime = "not recorded"
+	}
+
+	rand.Seed(time.Now().UTC().UnixNano())
+}
+
+const (
+	// DefaultDataDir is the default data directory.
+	DefaultDataDir = "~/.pilosa"
+)
+
+func mainz() {
+	serve := NewMain()
+	serve.Server.Handler.Version = Version
+	fmt.Fprintf(serve.Stderr, "Pilosa %s, build time %s\n", Version, BuildTime)
+
+	// Parse command line arguments.
+	if err := serve.ParseFlags(os.Args[1:]); err != nil {
+		fmt.Fprintln(serve.Stderr, err)
+		os.Exit(2)
+	}
+
+	// Start CPU profiling.
+	if serve.CPUProfile != "" {
+		f, err := os.Create(serve.CPUProfile)
+		if err != nil {
+			fmt.Fprintf(serve.Stderr, "create cpu profile: %v", err)
+			os.Exit(1)
+		}
+		defer f.Close()
+
+		fmt.Fprintln(serve.Stderr, "Starting cpu profile")
+		pprof.StartCPUProfile(f)
+		time.AfterFunc(serve.CPUTime, func() {
+			fmt.Fprintln(serve.Stderr, "Stopping cpu profile")
+			pprof.StopCPUProfile()
+			f.Close()
+		})
+	}
+
+	// Execute the program.
+	if err := serve.Run(); err != nil {
+		fmt.Fprintln(serve.Stderr, err)
+		fmt.Fprintln(serve.Stderr, "stopping profile")
+		os.Exit(1)
+	}
+
+	// First SIGKILL causes server to shut down gracefully.
+	c := make(chan os.Signal, 2)
+	signal.Notify(c, os.Interrupt)
+	sig := <-c
+	fmt.Fprintf(serve.Stderr, "Received %s; gracefully shutting down...\n", sig.String())
+
+	// Second signal causes a hard shutdown.
+	go func() { <-c; os.Exit(1) }()
+
+	if err := serve.Close(); err != nil {
+		fmt.Fprintln(serve.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+// Main represents the main program execution.
+type Main struct {
+	Server *pilosa.Server
+
+	// Configuration options.
+	ConfigPath string
+	Config     *pilosa.Config
+
+	// Profiling options.
+	CPUProfile string
+	CPUTime    time.Duration
+
+	// Standard input/output
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewMain returns a new instance of Main.
+func NewMain() *Main {
+	return &Main{
+		Server: pilosa.NewServer(),
+		Config: pilosa.NewConfig(),
+
+		Stdin:  os.Stdin,
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+}
+
+// Run executes the main program execution.
+func (m *Main) Run(args ...string) error {
+	// Notify user of config file.
+	if m.ConfigPath != "" {
+		fmt.Fprintf(m.Stdout, "Using config: %s\n", m.ConfigPath)
+	}
+
+	// Setup logging output.
+	m.Server.LogOutput = m.Stderr
+
+	// Configure index.
+	fmt.Fprintf(m.Stderr, "Using data from: %s\n", m.Config.DataDir)
+	m.Server.Index.Path = m.Config.DataDir
+	m.Server.Index.Stats = pilosa.NewExpvarStatsClient()
+
+	// Build cluster from config file.
+	m.Server.Host = m.Config.Host
+	m.Server.Cluster = m.Config.PilosaCluster()
+
+	// Set configuration options.
+	m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval)
+
+	// Initialize server.
+	if err := m.Server.Open(); err != nil {
+		return err
+	}
+
+	fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host)
+
+	return nil
+}
+
+// Close shuts down the server.
+func (m *Main) Close() error {
+	return m.Server.Close()
+}
+
+// ParseFlags parses command line flags from args.
+func (m *Main) ParseFlags(args []string) error {
+	fs := flag.NewFlagSet("pilosa", flag.ContinueOnError)
+	fs.StringVar(&m.CPUProfile, "cpuprofile", "", "cpu profile")
+	fs.DurationVar(&m.CPUTime, "cputime", 30*time.Second, "cpu profile duration")
+	fs.StringVar(&m.ConfigPath, "config", "", "config path")
+	fs.SetOutput(m.Stderr)
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	// Load config, if specified.
+	if m.ConfigPath != "" {
+		if _, err := toml.DecodeFile(m.ConfigPath, &m.Config); err != nil {
+			return err
+		}
+	}
+
+	// Use default data directory if one is not specified.
+	if m.Config.DataDir == "" {
+		m.Config.DataDir = DefaultDataDir
+	}
+
+	// Expand home directory.
+	prefix := "~" + string(filepath.Separator)
+	if strings.HasPrefix(m.Config.DataDir, prefix) {
+		//	u, err := user.Current()
+		HomeDir := os.Getenv("HOME")
+		/*if err != nil {
+			return err
+		} else*/if HomeDir == "" {
+			return errors.New("data directory not specified and no home dir available")
+		}
+		m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix))
+	}
+
+	return nil
+}
diff --git a/cmd/pilosa/main_test.go b/server/server_test.go
similarity index 99%
rename from cmd/pilosa/main_test.go
rename to server/server_test.go
index 184f5c334..bb772ab45 100644
--- a/cmd/pilosa/main_test.go
+++ b/server/server_test.go
@@ -1,4 +1,4 @@
-package main_test
+package server_test
 
 import (
 	"bytes"
@@ -18,7 +18,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 	"github.com/pilosa/pilosa"
-	main "github.com/pilosa/pilosa/cmd/pilosa"
+	"github.com/pilosa/pilosa/server"
 )
 
 // Ensure program can process queries and maintain consistency.
@@ -304,7 +304,7 @@ path = "/path/to/plugins"
 
 // Main represents a test wrapper for main.Main.
 type Main struct {
-	*main.Main
+	*server.Main
 
 	Stdin  bytes.Buffer
 	Stdout bytes.Buffer
@@ -318,7 +318,7 @@ func NewMain() *Main {
 		panic(err)
 	}
 
-	m := &Main{Main: main.NewMain()}
+	m := &Main{Main: server.NewMain()}
 	m.Config.DataDir = path
 	m.Config.Host = "localhost:0"
 	m.Main.Stdin = &m.Stdin
@@ -356,7 +356,7 @@ func (m *Main) Reopen() error {
 
 	// Create new main with the same config.
 	config := m.Config
-	m.Main = main.NewMain()
+	m.Main = server.NewMain()
 	m.Config = config
 
 	// Run new program.

From ed43eca004396b001124cf8955267f9db7c13a71 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 3 Mar 2017 13:35:06 -0600
Subject: [PATCH 15/37] move "config" to subcommand

---
 cmd/config.go         | 29 ++++++++++++++++++++++
 cmd/pilosactl/main.go | 57 -------------------------------------------
 ctl/config.go         | 43 ++++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+), 57 deletions(-)
 create mode 100644 cmd/config.go
 create mode 100644 ctl/config.go

diff --git a/cmd/config.go b/cmd/config.go
new file mode 100644
index 000000000..72202ffe3
--- /dev/null
+++ b/cmd/config.go
@@ -0,0 +1,29 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/pilosa/pilosa/ctl"
+)
+
+var conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr)
+
+var confCmd = &cobra.Command{
+	Use:   "config",
+	Short: "config - prints the default configuration",
+	Long: `config prints the default configuration to stdout
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := conf.Run(context.Background()); err != nil {
+			fmt.Println(err)
+		}
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(confCmd)
+}
diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go
index 88fa6fea5..8327839fe 100644
--- a/cmd/pilosactl/main.go
+++ b/cmd/pilosactl/main.go
@@ -96,7 +96,6 @@ Usage:
 
 The commands are:
 
-	config     prints the default configuration
 	import     imports data from a CSV file
 	export     exports data to a CSV file
 	sort       sorts a data file for optimal import speed
@@ -126,8 +125,6 @@ func (m *Main) ParseFlags(args []string) error {
 		fmt.Fprintln(m.Stderr, m.Usage())
 		fmt.Fprintln(m.Stderr, "")
 		return flag.ErrHelp
-	case "config":
-		m.Cmd = NewConfigCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "import":
 		m.Cmd = pilosactl.NewImportCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "export":
@@ -167,60 +164,6 @@ type Command interface {
 	Run(context.Context) error
 }
 
-// ConfigCommand represents a command for printing a default config.
-type ConfigCommand struct {
-	// Standard input/output
-	Stdin  io.Reader
-	Stdout io.Writer
-	Stderr io.Writer
-}
-
-// NewConfigCommand returns a new instance of ConfigCommand.
-func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *ConfigCommand {
-	return &ConfigCommand{
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	}
-}
-
-// ParseFlags parses command line flags from args.
-func (cmd *ConfigCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *ConfigCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl config
-
-Prints the default configuration file to standard out.
-`)
-}
-
-// Run executes the main program execution.
-func (cmd *ConfigCommand) Run(ctx context.Context) error {
-	fmt.Fprintln(cmd.Stdout, strings.TrimSpace(`
-data-dir = "~/.pilosa"
-host = "localhost:15000"
-
-[cluster]
-replicas = 1
-
-[[cluster.node]]
-host = "localhost:15000"
-
-[plugins]
-path = ""
-`)+"\n")
-	return nil
-}
-
 // ExportCommand represents a command for bulk exporting data from a server.
 type ExportCommand struct {
 	// Remote host and port.
diff --git a/ctl/config.go b/ctl/config.go
new file mode 100644
index 000000000..516069e2d
--- /dev/null
+++ b/ctl/config.go
@@ -0,0 +1,43 @@
+package ctl
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// ConfigCommand represents a command for printing a default config.
+type ConfigCommand struct {
+	// Standard input/output
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewConfigCommand returns a new instance of ConfigCommand.
+func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *ConfigCommand {
+	return &ConfigCommand{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+	}
+}
+
+// Run executes the main program execution.
+func (cmd *ConfigCommand) Run(ctx context.Context) error {
+	fmt.Fprintln(cmd.Stdout, strings.TrimSpace(`
+data-dir = "~/.pilosa"
+host = "localhost:15000"
+
+[cluster]
+replicas = 1
+
+[[cluster.node]]
+host = "localhost:15000"
+
+[plugins]
+path = ""
+`)+"\n")
+	return nil
+}

From e349dca06ac1bae7561f13f2a8407e215fdaa284 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 3 Mar 2017 14:11:12 -0600
Subject: [PATCH 16/37] move import to subcommand

---
 cmd/import.go                | 50 ++++++++++++++++++++++++++++++++++++
 cmd/pilosactl/main.go        |  3 ---
 {pilosactl => ctl}/import.go | 40 +----------------------------
 3 files changed, 51 insertions(+), 42 deletions(-)
 create mode 100644 cmd/import.go
 rename {pilosactl => ctl}/import.go (78%)

diff --git a/cmd/import.go b/cmd/import.go
new file mode 100644
index 000000000..af1a7fcd5
--- /dev/null
+++ b/cmd/import.go
@@ -0,0 +1,50 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/pilosa/pilosa/ctl"
+)
+
+var importer = ctl.NewImportCommand(os.Stdin, os.Stdout, os.Stderr)
+
+var importCmd = &cobra.Command{
+	Use:   "import",
+	Short: "import - import data to pilosa",
+	Long: `Bulk imports one or more CSV files to a host's database and frame. The bits
+of the CSV file are grouped by slice for the most efficient import.
+
+The format of the CSV file is:
+
+	BITMAPID,PROFILEID,[TIME]
+
+The file should contain no headers. The TIME column is optional and can be
+omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		importer.Paths = args
+		if err := importer.Run(context.Background()); err != nil {
+			fmt.Println(err)
+		}
+	},
+}
+
+func init() {
+	importCmd.Flags().StringVarP(&importer.Host, "host", "", "localhost:15000", "host:port of Pilosa.")
+	importCmd.Flags().StringVarP(&importer.Database, "database", "d", "", "Pilosa database to import into.")
+	importCmd.Flags().StringVarP(&importer.Frame, "frame", "f", "", "Frame to import into.")
+	importCmd.Flags().IntVarP(&importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
+
+	err := viper.BindPFlags(importCmd.Flags())
+	if err != nil {
+		log.Fatalf("Error binding import flags: %v", err)
+	}
+
+	RootCmd.AddCommand(importCmd)
+}
diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go
index 8327839fe..8b0980815 100644
--- a/cmd/pilosactl/main.go
+++ b/cmd/pilosactl/main.go
@@ -22,7 +22,6 @@ import (
 	"unsafe"
 
 	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/pilosactl"
 	"github.com/pilosa/pilosa/roaring"
 )
 
@@ -125,8 +124,6 @@ func (m *Main) ParseFlags(args []string) error {
 		fmt.Fprintln(m.Stderr, m.Usage())
 		fmt.Fprintln(m.Stderr, "")
 		return flag.ErrHelp
-	case "import":
-		m.Cmd = pilosactl.NewImportCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "export":
 		m.Cmd = NewExportCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "sort":
diff --git a/pilosactl/import.go b/ctl/import.go
similarity index 78%
rename from pilosactl/import.go
rename to ctl/import.go
index 9d4ad63f4..39e46eb3d 100644
--- a/pilosactl/import.go
+++ b/ctl/import.go
@@ -1,17 +1,14 @@
-package pilosactl
+package ctl
 
 import (
 	"context"
 	"encoding/csv"
 	"errors"
-	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/pilosa/pilosa"
@@ -56,41 +53,6 @@ func (cmd *ImportCommand) String() string {
 	return fmt.Sprint(*cmd)
 }
 
-// ParseFlags parses command line flags from args.
-func (cmd *ImportCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port")
-	fs.StringVar(&cmd.Database, "d", "", "database")
-	fs.StringVar(&cmd.Frame, "f", "", "frame")
-	fs.IntVar(&cmd.BufferSize, "buffer-size", cmd.BufferSize, "buffer size")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	// Extract the import paths.
-	cmd.Paths = fs.Args()
-
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *ImportCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl import -host HOST -d database -f frame paths
-
-Bulk imports one or more CSV files to a host's database and frame. The bits
-of the CSV file are grouped by slice for the most efficient import.
-
-The format of the CSV file is:
-
-	BITMAPID,PROFILEID,[TIME]
-
-The file should contain no headers. The TIME column is optional and can be
-omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
-`)
-}
-
 // Run executes the main program execution.
 func (cmd *ImportCommand) Run(ctx context.Context) error {
 	logger := log.New(cmd.Stderr, "", log.LstdFlags)

From 362539fe2dd4ecb906248e6375f85d87573148fc Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 3 Mar 2017 14:18:52 -0600
Subject: [PATCH 17/37] make export a subcommand

---
 cmd/export.go         |  49 ++++++++++++++++++
 cmd/pilosactl/main.go | 116 ------------------------------------------
 ctl/export.go         |  91 +++++++++++++++++++++++++++++++++
 3 files changed, 140 insertions(+), 116 deletions(-)
 create mode 100644 cmd/export.go
 create mode 100644 ctl/export.go

diff --git a/cmd/export.go b/cmd/export.go
new file mode 100644
index 000000000..16afd2475
--- /dev/null
+++ b/cmd/export.go
@@ -0,0 +1,49 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/pilosa/pilosa/ctl"
+)
+
+var exporter = ctl.NewExportCommand(os.Stdin, os.Stdout, os.Stderr)
+
+var exportCmd = &cobra.Command{
+	Use:   "export",
+	Short: "export - export data from pilosa",
+	Long: `
+Bulk exports a fragment to a CSV file. If the OUTFILE is not specified then
+the output is written to STDOUT.
+
+The format of the CSV file is:
+
+	BITMAPID,PROFILEID
+
+The file does not contain any headers.
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := exporter.Run(context.Background()); err != nil {
+			fmt.Println(err)
+		}
+	},
+}
+
+func init() {
+	exportCmd.Flags().StringVarP(&exporter.Host, "host", "", "localhost:15000", "host:port of Pilosa.")
+	exportCmd.Flags().StringVarP(&exporter.Database, "database", "d", "", "Pilosa database to export into.")
+	exportCmd.Flags().StringVarP(&exporter.Frame, "frame", "f", "", "Frame to export into.")
+	exportCmd.Flags().StringVarP(&exporter.Frame, "output-file", "o", "", "File to write export to - default stdout")
+
+	err := viper.BindPFlags(exportCmd.Flags())
+	if err != nil {
+		log.Fatalf("Error binding export flags: %v", err)
+	}
+
+	RootCmd.AddCommand(exportCmd)
+}
diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go
index 8b0980815..a1524bc35 100644
--- a/cmd/pilosactl/main.go
+++ b/cmd/pilosactl/main.go
@@ -95,8 +95,6 @@ Usage:
 
 The commands are:
 
-	import     imports data from a CSV file
-	export     exports data to a CSV file
 	sort       sorts a data file for optimal import speed
 	backup     backs up a frame to an archive file
 	restore    restores a frame from an archive file
@@ -124,8 +122,6 @@ func (m *Main) ParseFlags(args []string) error {
 		fmt.Fprintln(m.Stderr, m.Usage())
 		fmt.Fprintln(m.Stderr, "")
 		return flag.ErrHelp
-	case "export":
-		m.Cmd = NewExportCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "sort":
 		m.Cmd = NewSortCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "backup":
@@ -161,118 +157,6 @@ type Command interface {
 	Run(context.Context) error
 }
 
-// ExportCommand represents a command for bulk exporting data from a server.
-type ExportCommand struct {
-	// Remote host and port.
-	Host string
-
-	// Name of the database & frame to export from.
-	Database string
-	Frame    string
-
-	// Filename to export to.
-	Path string
-
-	// Standard input/output
-	Stdin  io.Reader
-	Stdout io.Writer
-	Stderr io.Writer
-}
-
-// NewExportCommand returns a new instance of ExportCommand.
-func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *ExportCommand {
-	return &ExportCommand{
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	}
-}
-
-// ParseFlags parses command line flags from args.
-func (cmd *ExportCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port")
-	fs.StringVar(&cmd.Database, "d", "", "database")
-	fs.StringVar(&cmd.Frame, "f", "", "frame")
-	fs.StringVar(&cmd.Path, "o", "", "output file")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *ExportCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl export -host HOST -d database -f frame -o OUTFILE
-
-Bulk exports a fragment to a CSV file. If the OUTFILE is not specified then
-the output is written to STDOUT.
-
-The format of the CSV file is:
-
-	BITMAPID,PROFILEID
-
-The file does not contain any headers.
-`)
-}
-
-// Run executes the main program execution.
-func (cmd *ExportCommand) Run(ctx context.Context) error {
-	logger := log.New(cmd.Stderr, "", log.LstdFlags)
-
-	// Validate arguments.
-	if cmd.Database == "" {
-		return pilosa.ErrDatabaseRequired
-	} else if cmd.Frame == "" {
-		return pilosa.ErrFrameRequired
-	}
-
-	// Use output file, if specified.
-	// Otherwise use STDOUT.
-	var w io.Writer = cmd.Stdout
-	if cmd.Path != "" {
-		f, err := os.Create(cmd.Path)
-		if err != nil {
-			return err
-		}
-		defer f.Close()
-
-		w = f
-	}
-
-	// Create a client to the server.
-	client, err := pilosa.NewClient(cmd.Host)
-	if err != nil {
-		return err
-	}
-
-	// Determine slice count.
-	maxSlices, err := client.MaxSliceByDatabase(ctx)
-	if err != nil {
-		return err
-	}
-
-	// Export each slice.
-	for slice := uint64(0); slice <= maxSlices[cmd.Database]; slice++ {
-		logger.Printf("exporting slice: %d", slice)
-		if err := client.ExportCSV(ctx, cmd.Database, cmd.Frame, slice, w); err != nil {
-			return err
-		}
-	}
-
-	// Close writer, if applicable.
-	if w, ok := w.(io.Closer); ok {
-		if err := w.Close(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 // SortCommand represents a command for sorting import data.
 type SortCommand struct {
 	// Filename to sort
diff --git a/ctl/export.go b/ctl/export.go
new file mode 100644
index 000000000..f19c7123f
--- /dev/null
+++ b/ctl/export.go
@@ -0,0 +1,91 @@
+package ctl
+
+import (
+	"context"
+	"io"
+	"log"
+	"os"
+
+	"github.com/pilosa/pilosa"
+)
+
+// ExportCommand represents a command for bulk exporting data from a server.
+type ExportCommand struct {
+	// Remote host and port.
+	Host string
+
+	// Name of the database & frame to export from.
+	Database string
+	Frame    string
+
+	// Filename to export to.
+	Path string
+
+	// Standard input/output
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewExportCommand returns a new instance of ExportCommand.
+func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *ExportCommand {
+	return &ExportCommand{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+	}
+}
+
+// Run executes the main program execution.
+func (cmd *ExportCommand) Run(ctx context.Context) error {
+	logger := log.New(cmd.Stderr, "", log.LstdFlags)
+
+	// Validate arguments.
+	if cmd.Database == "" {
+		return pilosa.ErrDatabaseRequired
+	} else if cmd.Frame == "" {
+		return pilosa.ErrFrameRequired
+	}
+
+	// Use output file, if specified.
+	// Otherwise use STDOUT.
+	var w io.Writer = cmd.Stdout
+	if cmd.Path != "" {
+		f, err := os.Create(cmd.Path)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		w = f
+	}
+
+	// Create a client to the server.
+	client, err := pilosa.NewClient(cmd.Host)
+	if err != nil {
+		return err
+	}
+
+	// Determine slice count.
+	maxSlices, err := client.MaxSliceByDatabase(ctx)
+	if err != nil {
+		return err
+	}
+
+	// Export each slice.
+	for slice := uint64(0); slice <= maxSlices[cmd.Database]; slice++ {
+		logger.Printf("exporting slice: %d", slice)
+		if err := client.ExportCSV(ctx, cmd.Database, cmd.Frame, slice, w); err != nil {
+			return err
+		}
+	}
+
+	// Close writer, if applicable.
+	if w, ok := w.(io.Closer); ok {
+		if err := w.Close(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

From 312a9a533430eafefddd79bcc58a919f0e09d2fe Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 3 Mar 2017 14:38:44 -0600
Subject: [PATCH 18/37] move sort to subcommand

---
 cmd/pilosactl/main.go | 164 ------------------------------------------
 cmd/sort.go           |  45 ++++++++++++
 ctl/sort.go           | 138 +++++++++++++++++++++++++++++++++++
 3 files changed, 183 insertions(+), 164 deletions(-)
 create mode 100644 cmd/sort.go
 create mode 100644 ctl/sort.go

diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go
index a1524bc35..e3052ac12 100644
--- a/cmd/pilosactl/main.go
+++ b/cmd/pilosactl/main.go
@@ -1,20 +1,15 @@
 package main
 
 import (
-	"bufio"
 	"context"
-	"encoding/csv"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"os"
 	"path/filepath"
-	"sort"
-	"strconv"
 	"strings"
 	"syscall"
 	"text/tabwriter"
@@ -95,7 +90,6 @@ Usage:
 
 The commands are:
 
-	sort       sorts a data file for optimal import speed
 	backup     backs up a frame to an archive file
 	restore    restores a frame from an archive file
 	inspect    inspects fragment data files
@@ -122,8 +116,6 @@ func (m *Main) ParseFlags(args []string) error {
 		fmt.Fprintln(m.Stderr, m.Usage())
 		fmt.Fprintln(m.Stderr, "")
 		return flag.ErrHelp
-	case "sort":
-		m.Cmd = NewSortCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "backup":
 		m.Cmd = NewBackupCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "restore":
@@ -157,120 +149,6 @@ type Command interface {
 	Run(context.Context) error
 }
 
-// SortCommand represents a command for sorting import data.
-type SortCommand struct {
-	// Filename to sort
-	Path string
-
-	// Standard input/output
-	Stdin  io.Reader
-	Stdout io.Writer
-	Stderr io.Writer
-}
-
-// NewSortCommand returns a new instance of SortCommand.
-func NewSortCommand(stdin io.Reader, stdout, stderr io.Writer) *SortCommand {
-	return &SortCommand{
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	}
-}
-
-// ParseFlags parses command line flags from args.
-func (cmd *SortCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	// Extract the data path.
-	if fs.NArg() == 0 {
-		return errors.New("path required")
-	} else if fs.NArg() > 1 {
-		return errors.New("only one path allowed")
-	}
-	cmd.Path = fs.Arg(0)
-
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *SortCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl sort PATH
-
-Sorts the import data at PATH into the optimal sort order for importing.
-
-The format of the CSV file is:
-
-	BITMAPID,PROFILEID
-
-The file should contain no headers.
-`)
-}
-
-// Run executes the main program execution.
-func (cmd *SortCommand) Run(ctx context.Context) error {
-	// Open file for reading.
-	f, err := os.Open(cmd.Path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	// Read rows as bits.
-	r := csv.NewReader(f)
-	r.FieldsPerRecord = -1
-	a := make([]pilosa.Bit, 0, 1000000)
-	for {
-		bitmapID, profileID, timestamp, err := readCSVRow(r)
-		if err == io.EOF {
-			break
-		} else if err == errBlank {
-			continue
-		} else if err != nil {
-			return err
-		}
-		a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID, Timestamp: timestamp})
-	}
-
-	// Sort bits by position.
-	sort.Sort(pilosa.BitsByPos(a))
-
-	// Rewrite to STDOUT.
-	w := bufio.NewWriter(cmd.Stdout)
-	buf := make([]byte, 0, 1024)
-	for _, bit := range a {
-		// Write CSV to buffer.
-		buf = buf[:0]
-		buf = strconv.AppendUint(buf, bit.BitmapID, 10)
-
-		buf = append(buf, ',')
-		buf = strconv.AppendUint(buf, bit.ProfileID, 10)
-
-		if bit.Timestamp != 0 {
-			buf = append(buf, ',')
-			buf = append(buf, time.Unix(0, bit.Timestamp).UTC().Format(pilosa.TimeFormat)...)
-		}
-
-		buf = append(buf, '\n')
-
-		// Write to output.
-		if _, err := w.Write(buf); err != nil {
-			return err
-		}
-	}
-
-	// Ensure buffer is flushed before exiting.
-	if err := w.Flush(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // BackupCommand represents a command for backing up a frame.
 type BackupCommand struct {
 	// Destination host and port.
@@ -808,45 +686,3 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) e
 
 	return nil
 }
-
-// readCSVRow reads a bitmap/profile pair from a CSV row.
-func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err error) {
-	// Read CSV row.
-	record, err := r.Read()
-	if err != nil {
-		return 0, 0, 0, err
-	}
-
-	// Ignore blank rows.
-	if record[0] == "" {
-		return 0, 0, 0, errBlank
-	} else if len(record) < 2 {
-		return 0, 0, 0, fmt.Errorf("bad column count: %d", len(record))
-	}
-
-	// Parse bitmap id.
-	bitmapID, err = strconv.ParseUint(record[0], 10, 64)
-	if err != nil {
-		return 0, 0, 0, fmt.Errorf("invalid bitmap id: %q", record[0])
-	}
-
-	// Parse bitmap id.
-	profileID, err = strconv.ParseUint(record[1], 10, 64)
-	if err != nil {
-		return 0, 0, 0, fmt.Errorf("invalid profile id: %q", record[1])
-	}
-
-	// Parse timestamp, if available.
-	if len(record) > 2 && record[2] != "" {
-		t, err := time.Parse(pilosa.TimeFormat, record[2])
-		if err != nil {
-			return 0, 0, 0, fmt.Errorf("invalid timestamp: %q", record[2])
-		}
-		timestamp = t.UnixNano()
-	}
-
-	return bitmapID, profileID, timestamp, nil
-}
-
-// errBlank indicates a blank row in a CSV file.
-var errBlank = errors.New("blank row")
diff --git a/cmd/sort.go b/cmd/sort.go
new file mode 100644
index 000000000..813624ddc
--- /dev/null
+++ b/cmd/sort.go
@@ -0,0 +1,45 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/pilosa/pilosa/ctl"
+)
+
+var sorter = ctl.NewSortCommand(os.Stdin, os.Stdout, os.Stderr)
+
+var sortCmd = &cobra.Command{
+	Use:   "sort <path>",
+	Short: "sort - sort import data for optimal import performance",
+	Long: `
+Sorts the import data at PATH into the optimal sort order for importing.
+
+The format of the CSV file is:
+
+	BITMAPID,PROFILEID
+
+The file should contain no headers.
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println(cmd.Flags())
+		if len(args) == 0 {
+			fmt.Println("path required")
+			return
+		} else if len(args) > 1 {
+			fmt.Println("only one path supported")
+			return
+		}
+		sorter.Path = args[0]
+		if err := sorter.Run(context.Background()); err != nil {
+			fmt.Println(err)
+		}
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(sortCmd)
+}
diff --git a/ctl/sort.go b/ctl/sort.go
new file mode 100644
index 000000000..eac68b7ab
--- /dev/null
+++ b/ctl/sort.go
@@ -0,0 +1,138 @@
+package ctl
+
+import (
+	"bufio"
+	"context"
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"strconv"
+	"time"
+
+	"github.com/pilosa/pilosa"
+)
+
+// SortCommand represents a command for sorting import data.
+type SortCommand struct {
+	// Filename to sort
+	Path string
+
+	// Standard input/output
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewSortCommand returns a new instance of SortCommand.
+func NewSortCommand(stdin io.Reader, stdout, stderr io.Writer) *SortCommand {
+	return &SortCommand{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+	}
+}
+
+// Run executes the main program execution.
+func (cmd *SortCommand) Run(ctx context.Context) error {
+	// Open file for reading.
+	f, err := os.Open(cmd.Path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	// Read rows as bits.
+	r := csv.NewReader(f)
+	r.FieldsPerRecord = -1
+	a := make([]pilosa.Bit, 0, 1000000)
+	for {
+		bitmapID, profileID, timestamp, err := readCSVRow(r)
+		if err == io.EOF {
+			break
+		} else if err == errBlank {
+			continue
+		} else if err != nil {
+			return err
+		}
+		a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID, Timestamp: timestamp})
+	}
+
+	// Sort bits by position.
+	sort.Sort(pilosa.BitsByPos(a))
+
+	// Rewrite to STDOUT.
+	w := bufio.NewWriter(cmd.Stdout)
+	buf := make([]byte, 0, 1024)
+	for _, bit := range a {
+		// Write CSV to buffer.
+		buf = buf[:0]
+		buf = strconv.AppendUint(buf, bit.BitmapID, 10)
+
+		buf = append(buf, ',')
+		buf = strconv.AppendUint(buf, bit.ProfileID, 10)
+
+		if bit.Timestamp != 0 {
+			buf = append(buf, ',')
+			buf = append(buf, time.Unix(0, bit.Timestamp).UTC().Format(pilosa.TimeFormat)...)
+		}
+
+		buf = append(buf, '\n')
+
+		// Write to output.
+		if _, err := w.Write(buf); err != nil {
+			return err
+		}
+	}
+
+	// Ensure buffer is flushed before exiting.
+	if err := w.Flush(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// readCSVRow reads a bitmap/profile pair from a CSV row.
+func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err error) {
+	// Read CSV row.
+	record, err := r.Read()
+	if err != nil {
+		return 0, 0, 0, err
+	}
+
+	// Ignore blank rows.
+	if record[0] == "" {
+		return 0, 0, 0, errBlank
+	} else if len(record) < 2 {
+		return 0, 0, 0, fmt.Errorf("bad column count: %d", len(record))
+	}
+
+	// Parse bitmap id.
+	bitmapID, err = strconv.ParseUint(record[0], 10, 64)
+	if err != nil {
+		return 0, 0, 0, fmt.Errorf("invalid bitmap id: %q", record[0])
+	}
+
+	// Parse bitmap id.
+	profileID, err = strconv.ParseUint(record[1], 10, 64)
+	if err != nil {
+		return 0, 0, 0, fmt.Errorf("invalid profile id: %q", record[1])
+	}
+
+	// Parse timestamp, if available.
+	if len(record) > 2 && record[2] != "" {
+		t, err := time.Parse(pilosa.TimeFormat, record[2])
+		if err != nil {
+			return 0, 0, 0, fmt.Errorf("invalid timestamp: %q", record[2])
+		}
+		timestamp = t.UnixNano()
+	}
+
+	return bitmapID, profileID, timestamp, nil
+}
+
+// errBlank indicates a blank row in a CSV file.
+var errBlank = errors.New("blank row")

From f20e6c193d01207ead2191ddd318d28a1d0ef155 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 3 Mar 2017 14:46:45 -0600
Subject: [PATCH 19/37] move backup to subcommand

---
 cmd/backup.go         | 42 ++++++++++++++++++++
 cmd/pilosactl/main.go | 89 -------------------------------------------
 ctl/backup.go         | 72 ++++++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+), 89 deletions(-)
 create mode 100644 cmd/backup.go
 create mode 100644 ctl/backup.go

diff --git a/cmd/backup.go b/cmd/backup.go
new file mode 100644
index 000000000..883ce7be8
--- /dev/null
+++ b/cmd/backup.go
@@ -0,0 +1,42 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/pilosa/pilosa/ctl"
+)
+
+var backuper = ctl.NewBackupCommand(os.Stdin, os.Stdout, os.Stderr)
+
+var backupCmd = &cobra.Command{
+	Use:   "backup",
+	Short: "backup - backup data from pilosa",
+	Long: `
+Backs up the database and frame from across the cluster into a single file.
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := backuper.Run(context.Background()); err != nil {
+			fmt.Println(err)
+		}
+	},
+}
+
+func init() {
+	backupCmd.Flags().StringVarP(&backuper.Host, "host", "", "localhost:15000", "host:port of Pilosa.")
+	backupCmd.Flags().StringVarP(&backuper.Database, "database", "d", "", "Pilosa database to backup into.")
+	backupCmd.Flags().StringVarP(&backuper.Frame, "frame", "f", "", "Frame to backup into.")
+	backupCmd.Flags().StringVarP(&backuper.Path, "output-file", "o", "", "File to write backup to - default stdout")
+
+	err := viper.BindPFlags(backupCmd.Flags())
+	if err != nil {
+		log.Fatalf("Error binding backup flags: %v", err)
+	}
+
+	RootCmd.AddCommand(backupCmd)
+}
diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go
index e3052ac12..72399e9b2 100644
--- a/cmd/pilosactl/main.go
+++ b/cmd/pilosactl/main.go
@@ -90,7 +90,6 @@ Usage:
 
 The commands are:
 
-	backup     backs up a frame to an archive file
 	restore    restores a frame from an archive file
 	inspect    inspects fragment data files
 	check      performs a consistency check of data files
@@ -116,8 +115,6 @@ func (m *Main) ParseFlags(args []string) error {
 		fmt.Fprintln(m.Stderr, m.Usage())
 		fmt.Fprintln(m.Stderr, "")
 		return flag.ErrHelp
-	case "backup":
-		m.Cmd = NewBackupCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "restore":
 		m.Cmd = NewRestoreCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "inspect":
@@ -149,92 +146,6 @@ type Command interface {
 	Run(context.Context) error
 }
 
-// BackupCommand represents a command for backing up a frame.
-type BackupCommand struct {
-	// Destination host and port.
-	Host string
-
-	// Name of the database & frame to backup.
-	Database string
-	Frame    string
-
-	// Output file to write to.
-	Path string
-
-	// Standard input/output
-	Stdin  io.Reader
-	Stdout io.Writer
-	Stderr io.Writer
-}
-
-// NewBackupCommand returns a new instance of BackupCommand.
-func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand {
-	return &BackupCommand{
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	}
-}
-
-// ParseFlags parses command line flags from args.
-func (cmd *BackupCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port")
-	fs.StringVar(&cmd.Database, "d", "", "database")
-	fs.StringVar(&cmd.Frame, "f", "", "frame")
-	fs.StringVar(&cmd.Path, "o", "", "output file")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *BackupCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl backup -host HOST -d database -f frame -o PATH
-
-Backs up the database and frame from across the cluster into a single file.
-`)
-}
-
-// Run executes the main program execution.
-func (cmd *BackupCommand) Run(ctx context.Context) error {
-	// Validate arguments.
-	if cmd.Path == "" {
-		return errors.New("output file required")
-	}
-
-	// Create a client to the server.
-	client, err := pilosa.NewClient(cmd.Host)
-	if err != nil {
-		return err
-	}
-
-	// Open output file.
-	f, err := os.Create(cmd.Path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	// Begin streaming backup.
-	if err := client.BackupTo(ctx, f, cmd.Database, cmd.Frame); err != nil {
-		return err
-	}
-
-	// Sync & close file to ensure durability.
-	if err := f.Sync(); err != nil {
-		return err
-	} else if err = f.Close(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // RestoreCommand represents a command for restoring a frame from a backup.
 type RestoreCommand struct {
 	// Destination host and port.
diff --git a/ctl/backup.go b/ctl/backup.go
new file mode 100644
index 000000000..23b34cd1f
--- /dev/null
+++ b/ctl/backup.go
@@ -0,0 +1,72 @@
+package ctl
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+
+	"github.com/pilosa/pilosa"
+)
+
+// BackupCommand represents a command for backing up a frame.
+type BackupCommand struct {
+	// Destination host and port.
+	Host string
+
+	// Name of the database & frame to backup.
+	Database string
+	Frame    string
+
+	// Output file to write to.
+	Path string
+
+	// Standard input/output
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewBackupCommand returns a new instance of BackupCommand.
+func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand {
+	return &BackupCommand{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+	}
+}
+
+// Run executes the main program execution.
+func (cmd *BackupCommand) Run(ctx context.Context) error {
+	// Validate arguments.
+	if cmd.Path == "" {
+		return errors.New("output file required")
+	}
+
+	// Create a client to the server.
+	client, err := pilosa.NewClient(cmd.Host)
+	if err != nil {
+		return err
+	}
+
+	// Open output file.
+	f, err := os.Create(cmd.Path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	// Begin streaming backup.
+	if err := client.BackupTo(ctx, f, cmd.Database, cmd.Frame); err != nil {
+		return err
+	}
+
+	// Sync & close file to ensure durability.
+	if err := f.Sync(); err != nil {
+		return err
+	} else if err = f.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}

From 6dc164da69ed49543997245b06b06edc22aee1e6 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 3 Mar 2017 14:47:47 -0600
Subject: [PATCH 20/37] fix copy/paste bug in export subcommand

---
 cmd/export.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/export.go b/cmd/export.go
index 16afd2475..a5cd0f62d 100644
--- a/cmd/export.go
+++ b/cmd/export.go
@@ -38,7 +38,7 @@ func init() {
 	exportCmd.Flags().StringVarP(&exporter.Host, "host", "", "localhost:15000", "host:port of Pilosa.")
 	exportCmd.Flags().StringVarP(&exporter.Database, "database", "d", "", "Pilosa database to export into.")
 	exportCmd.Flags().StringVarP(&exporter.Frame, "frame", "f", "", "Frame to export into.")
-	exportCmd.Flags().StringVarP(&exporter.Frame, "output-file", "o", "", "File to write export to - default stdout")
+	exportCmd.Flags().StringVarP(&exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
 
 	err := viper.BindPFlags(exportCmd.Flags())
 	if err != nil {

From 21478cb85270052e4817d9f26a1c5431757c377a Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 3 Mar 2017 14:53:35 -0600
Subject: [PATCH 21/37] move restore to subcommand

---
 cmd/pilosactl/main.go | 89 -------------------------------------------
 cmd/restore.go        | 42 ++++++++++++++++++++
 ctl/restore.go        | 65 +++++++++++++++++++++++++++++++
 3 files changed, 107 insertions(+), 89 deletions(-)
 create mode 100644 cmd/restore.go
 create mode 100644 ctl/restore.go

diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go
index 72399e9b2..ee77ba2a8 100644
--- a/cmd/pilosactl/main.go
+++ b/cmd/pilosactl/main.go
@@ -90,7 +90,6 @@ Usage:
 
 The commands are:
 
-	restore    restores a frame from an archive file
 	inspect    inspects fragment data files
 	check      performs a consistency check of data files
 	bench      benchmarks operations
@@ -115,8 +114,6 @@ func (m *Main) ParseFlags(args []string) error {
 		fmt.Fprintln(m.Stderr, m.Usage())
 		fmt.Fprintln(m.Stderr, "")
 		return flag.ErrHelp
-	case "restore":
-		m.Cmd = NewRestoreCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "inspect":
 		m.Cmd = NewInspectCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "check":
@@ -146,92 +143,6 @@ type Command interface {
 	Run(context.Context) error
 }
 
-// RestoreCommand represents a command for restoring a frame from a backup.
-type RestoreCommand struct {
-	// Destination host and port.
-	Host string
-
-	// Name of the database & frame to backup.
-	Database string
-	Frame    string
-
-	// Import file to read from.
-	Path string
-
-	// Standard input/output
-	Stdin  io.Reader
-	Stdout io.Writer
-	Stderr io.Writer
-}
-
-// NewRestoreCommand returns a new instance of RestoreCommand.
-func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand {
-	return &RestoreCommand{
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	}
-}
-
-// ParseFlags parses command line flags from args.
-func (cmd *RestoreCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port")
-	fs.StringVar(&cmd.Database, "d", "", "database")
-	fs.StringVar(&cmd.Frame, "f", "", "frame")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	// Read input path from the args.
-	if fs.NArg() == 0 {
-		return errors.New("path required")
-	} else if fs.NArg() > 1 {
-		return errors.New("too many paths specified")
-	}
-	cmd.Path = fs.Arg(0)
-
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *RestoreCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl restore -host HOST -d database -f frame PATH
-
-Restores a frame to the cluster from a backup file.
-`)
-}
-
-// Run executes the main program execution.
-func (cmd *RestoreCommand) Run(ctx context.Context) error {
-	// Validate arguments.
-	if cmd.Path == "" {
-		return errors.New("backup file required")
-	}
-
-	// Create a client to the server.
-	client, err := pilosa.NewClient(cmd.Host)
-	if err != nil {
-		return err
-	}
-
-	// Open backup file.
-	f, err := os.Open(cmd.Path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	// Restore backup file to the cluster.
-	if err := client.RestoreFrom(ctx, f, cmd.Database, cmd.Frame); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // InspectCommand represents a command for inspecting fragment data files.
 type InspectCommand struct {
 	// Path to data file
diff --git a/cmd/restore.go b/cmd/restore.go
new file mode 100644
index 000000000..420cd9c5f
--- /dev/null
+++ b/cmd/restore.go
@@ -0,0 +1,42 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/pilosa/pilosa/ctl"
+)
+
+var restorer = ctl.NewRestoreCommand(os.Stdin, os.Stdout, os.Stderr)
+
+var restoreCmd = &cobra.Command{
+	Use:   "restore",
+	Short: "restore - restore data to pilosa",
+	Long: `
+Restores a frame to the cluster from a backup file.
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := restorer.Run(context.Background()); err != nil {
+			fmt.Println(err)
+		}
+	},
+}
+
+func init() {
+	restoreCmd.Flags().StringVarP(&restorer.Host, "host", "", "localhost:15000", "host:port of Pilosa.")
+	restoreCmd.Flags().StringVarP(&restorer.Database, "database", "d", "", "Pilosa database to restore into.")
+	restoreCmd.Flags().StringVarP(&restorer.Frame, "frame", "f", "", "Frame to restore into.")
+	restoreCmd.Flags().StringVarP(&restorer.Path, "input-file", "i", "", "File to write restore from")
+
+	err := viper.BindPFlags(restoreCmd.Flags())
+	if err != nil {
+		log.Fatalf("Error binding restore flags: %v", err)
+	}
+
+	RootCmd.AddCommand(restoreCmd)
+}
diff --git a/ctl/restore.go b/ctl/restore.go
new file mode 100644
index 000000000..b9a573704
--- /dev/null
+++ b/ctl/restore.go
@@ -0,0 +1,65 @@
+package ctl
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+
+	"github.com/pilosa/pilosa"
+)
+
+// RestoreCommand represents a command for restoring a frame from a backup.
+type RestoreCommand struct {
+	// Destination host and port.
+	Host string
+
+	// Name of the database & frame to backup.
+	Database string
+	Frame    string
+
+	// Import file to read from.
+	Path string
+
+	// Standard input/output
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewRestoreCommand returns a new instance of RestoreCommand.
+func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand {
+	return &RestoreCommand{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+	}
+}
+
+// Run executes the main program execution.
+func (cmd *RestoreCommand) Run(ctx context.Context) error {
+	// Validate arguments.
+	if cmd.Path == "" {
+		return errors.New("backup file required")
+	}
+
+	// Create a client to the server.
+	client, err := pilosa.NewClient(cmd.Host)
+	if err != nil {
+		return err
+	}
+
+	// Open backup file.
+	f, err := os.Open(cmd.Path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	// Restore backup file to the cluster.
+	if err := client.RestoreFrom(ctx, f, cmd.Database, cmd.Frame); err != nil {
+		return err
+	}
+
+	return nil
+}

From 2c1929a063f7560c9ea95ba644c89ceeeff5768d Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 3 Mar 2017 15:00:54 -0600
Subject: [PATCH 22/37] move inspect to subcommand

---
 cmd/inspect.go        |  39 +++++++++++++
 cmd/pilosactl/main.go | 114 -------------------------------------
 ctl/inspect.go        | 127 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 166 insertions(+), 114 deletions(-)
 create mode 100644 cmd/inspect.go
 create mode 100644 ctl/inspect.go

diff --git a/cmd/inspect.go b/cmd/inspect.go
new file mode 100644
index 000000000..48bbf8308
--- /dev/null
+++ b/cmd/inspect.go
@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/pilosa/pilosa/ctl"
+)
+
+var inspecter = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr)
+
+var inspectCmd = &cobra.Command{
+	Use:   "inspect",
+	Short: "inspect - inspect a pilosa data file",
+	Long: `
+Inspects a data file and provides stats.
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := inspecter.Run(context.Background()); err != nil {
+			fmt.Println(err)
+		}
+	},
+}
+
+func init() {
+	inspectCmd.Flags().StringVarP(&inspecter.Path, "file", "i", "", "File to inspect")
+
+	err := viper.BindPFlags(inspectCmd.Flags())
+	if err != nil {
+		log.Fatalf("Error binding inspect flags: %v", err)
+	}
+
+	RootCmd.AddCommand(inspectCmd)
+}
diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go
index ee77ba2a8..a04be0b02 100644
--- a/cmd/pilosactl/main.go
+++ b/cmd/pilosactl/main.go
@@ -12,9 +12,7 @@ import (
 	"path/filepath"
 	"strings"
 	"syscall"
-	"text/tabwriter"
 	"time"
-	"unsafe"
 
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/roaring"
@@ -90,7 +88,6 @@ Usage:
 
 The commands are:
 
-	inspect    inspects fragment data files
 	check      performs a consistency check of data files
 	bench      benchmarks operations
 
@@ -114,8 +111,6 @@ func (m *Main) ParseFlags(args []string) error {
 		fmt.Fprintln(m.Stderr, m.Usage())
 		fmt.Fprintln(m.Stderr, "")
 		return flag.ErrHelp
-	case "inspect":
-		m.Cmd = NewInspectCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "check":
 		m.Cmd = NewCheckCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "bench":
@@ -143,115 +138,6 @@ type Command interface {
 	Run(context.Context) error
 }
 
-// InspectCommand represents a command for inspecting fragment data files.
-type InspectCommand struct {
-	// Path to data file
-	Path string
-
-	// Standard input/output
-	Stdin  io.Reader
-	Stdout io.Writer
-	Stderr io.Writer
-}
-
-// NewInspectCommand returns a new instance of InspectCommand.
-func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectCommand {
-	return &InspectCommand{
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	}
-}
-
-// ParseFlags parses command line flags from args.
-func (cmd *InspectCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	// Parse path.
-	if fs.NArg() == 0 {
-		return errors.New("path required")
-	} else if fs.NArg() > 1 {
-		return errors.New("only one path allowed")
-	}
-	cmd.Path = fs.Arg(0)
-
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *InspectCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl inspect PATH
-
-Inspects a data file and provides stats.
-
-`)
-}
-
-// Run executes the main program execution.
-func (cmd *InspectCommand) Run(ctx context.Context) error {
-	// Open file handle.
-	f, err := os.Open(cmd.Path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	fi, err := f.Stat()
-	if err != nil {
-		return err
-	}
-
-	// Memory map the file.
-	data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
-	if err != nil {
-		return err
-	}
-	defer syscall.Munmap(data)
-
-	// Attach the mmap file to the bitmap.
-	t := time.Now()
-	fmt.Fprintf(cmd.Stderr, "unmarshaling bitmap...")
-	bm := roaring.NewBitmap()
-	if err := bm.UnmarshalBinary(data); err != nil {
-		return err
-	}
-	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
-
-	// Retrieve stats.
-	t = time.Now()
-	fmt.Fprintf(cmd.Stderr, "calculating stats...")
-	info := bm.Info()
-	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
-
-	// Print top-level info.
-	fmt.Fprintf(cmd.Stdout, "== Bitmap Info ==\n")
-	fmt.Fprintf(cmd.Stdout, "Containers: %d\n", len(info.Containers))
-	fmt.Fprintf(cmd.Stdout, "Operations: %d\n", info.OpN)
-	fmt.Fprintln(cmd.Stdout, "")
-
-	// Print info for each container.
-	fmt.Fprintln(cmd.Stdout, "== Containers ==")
-	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
-	fmt.Fprintf(tw, "%s\t%s\t% 8s \t% 8s\t%s\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET")
-	for _, ci := range info.Containers {
-		fmt.Fprintf(tw, "%d\t%s\t% 8d \t% 8d \t0x%08x\n",
-			ci.Key,
-			ci.Type,
-			ci.N,
-			ci.Alloc,
-			uintptr(ci.Pointer)-uintptr(unsafe.Pointer(&data[0])),
-		)
-	}
-	tw.Flush()
-
-	return nil
-}
-
 // CheckCommand represents a command for performing consistency checks on data files.
 type CheckCommand struct {
 	// Data file paths.
diff --git a/ctl/inspect.go b/ctl/inspect.go
new file mode 100644
index 000000000..1b00dbcee
--- /dev/null
+++ b/ctl/inspect.go
@@ -0,0 +1,127 @@
+package ctl
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"syscall"
+	"text/tabwriter"
+	"time"
+	"unsafe"
+
+	"github.com/pilosa/pilosa/roaring"
+)
+
+// InspectCommand represents a command for inspecting fragment data files.
+type InspectCommand struct {
+	// Path to data file
+	Path string
+
+	// Standard input/output
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewInspectCommand returns a new instance of InspectCommand.
+func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectCommand {
+	return &InspectCommand{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+	}
+}
+
+// ParseFlags parses command line flags from args.
+func (cmd *InspectCommand) ParseFlags(args []string) error {
+	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
+	fs.SetOutput(ioutil.Discard)
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	// Parse path.
+	if fs.NArg() == 0 {
+		return errors.New("path required")
+	} else if fs.NArg() > 1 {
+		return errors.New("only one path allowed")
+	}
+	cmd.Path = fs.Arg(0)
+
+	return nil
+}
+
+// Usage returns the usage message to be printed.
+func (cmd *InspectCommand) Usage() string {
+	return strings.TrimSpace(`
+usage: pilosactl inspect PATH
+
+Inspects a data file and provides stats.
+
+`)
+}
+
+// Run executes the main program execution.
+func (cmd *InspectCommand) Run(ctx context.Context) error {
+	// Open file handle.
+	f, err := os.Open(cmd.Path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
+	if err != nil {
+		return err
+	}
+
+	// Memory map the file.
+	data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
+	if err != nil {
+		return err
+	}
+	defer syscall.Munmap(data)
+
+	// Attach the mmap file to the bitmap.
+	t := time.Now()
+	fmt.Fprintf(cmd.Stderr, "unmarshaling bitmap...")
+	bm := roaring.NewBitmap()
+	if err := bm.UnmarshalBinary(data); err != nil {
+		return err
+	}
+	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+
+	// Retrieve stats.
+	t = time.Now()
+	fmt.Fprintf(cmd.Stderr, "calculating stats...")
+	info := bm.Info()
+	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+
+	// Print top-level info.
+	fmt.Fprintf(cmd.Stdout, "== Bitmap Info ==\n")
+	fmt.Fprintf(cmd.Stdout, "Containers: %d\n", len(info.Containers))
+	fmt.Fprintf(cmd.Stdout, "Operations: %d\n", info.OpN)
+	fmt.Fprintln(cmd.Stdout, "")
+
+	// Print info for each container.
+	fmt.Fprintln(cmd.Stdout, "== Containers ==")
+	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+	fmt.Fprintf(tw, "%s\t%s\t% 8s \t% 8s\t%s\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET")
+	for _, ci := range info.Containers {
+		fmt.Fprintf(tw, "%d\t%s\t% 8d \t% 8d \t0x%08x\n",
+			ci.Key,
+			ci.Type,
+			ci.N,
+			ci.Alloc,
+			uintptr(ci.Pointer)-uintptr(unsafe.Pointer(&data[0])),
+		)
+	}
+	tw.Flush()
+
+	return nil
+}

From ff532f2064e5ae0c7cefc9dabefcca582ee14436 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 3 Mar 2017 15:01:29 -0600
Subject: [PATCH 23/37] fix help text in restore command

---
 cmd/restore.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/restore.go b/cmd/restore.go
index 420cd9c5f..44899e4ca 100644
--- a/cmd/restore.go
+++ b/cmd/restore.go
@@ -31,7 +31,7 @@ func init() {
 	restoreCmd.Flags().StringVarP(&restorer.Host, "host", "", "localhost:15000", "host:port of Pilosa.")
 	restoreCmd.Flags().StringVarP(&restorer.Database, "database", "d", "", "Pilosa database to restore into.")
 	restoreCmd.Flags().StringVarP(&restorer.Frame, "frame", "f", "", "Frame to restore into.")
-	restoreCmd.Flags().StringVarP(&restorer.Path, "input-file", "i", "", "File to write restore from")
+	restoreCmd.Flags().StringVarP(&restorer.Path, "input-file", "i", "", "File to restore from.")
 
 	err := viper.BindPFlags(restoreCmd.Flags())
 	if err != nil {

From 01020d06a84ae4bf18fb308d6fddf1d114bb6e47 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 3 Mar 2017 15:05:59 -0600
Subject: [PATCH 24/37] mvoe check to subcommand

---
 cmd/check.go          |  35 ++++++++++
 cmd/pilosactl/main.go | 135 ---------------------------------------
 ctl/check.go          | 145 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 180 insertions(+), 135 deletions(-)
 create mode 100644 cmd/check.go
 create mode 100644 ctl/check.go

diff --git a/cmd/check.go b/cmd/check.go
new file mode 100644
index 000000000..541f33cb2
--- /dev/null
+++ b/cmd/check.go
@@ -0,0 +1,35 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/pilosa/pilosa/ctl"
+)
+
+var checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr)
+
+var checkCmd = &cobra.Command{
+	Use:   "check <path> [path2]...",
+	Short: "check - check a pilosa data file",
+	Long: `
+Performs a consistency check on data files.
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		if len(args) == 0 {
+			fmt.Println("path required")
+			return
+		}
+		checker.Paths = args
+		if err := checker.Run(context.Background()); err != nil {
+			fmt.Println(err)
+		}
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(checkCmd)
+}
diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go
index a04be0b02..401d1cff5 100644
--- a/cmd/pilosactl/main.go
+++ b/cmd/pilosactl/main.go
@@ -9,13 +9,10 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"os"
-	"path/filepath"
 	"strings"
-	"syscall"
 	"time"
 
 	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/roaring"
 )
 
 var (
@@ -88,7 +85,6 @@ Usage:
 
 The commands are:
 
-	check      performs a consistency check of data files
 	bench      benchmarks operations
 
 Use the "-h" flag with any command for more information.
@@ -111,8 +107,6 @@ func (m *Main) ParseFlags(args []string) error {
 		fmt.Fprintln(m.Stderr, m.Usage())
 		fmt.Fprintln(m.Stderr, "")
 		return flag.ErrHelp
-	case "check":
-		m.Cmd = NewCheckCommand(m.Stdin, m.Stdout, m.Stderr)
 	case "bench":
 		m.Cmd = NewBenchCommand(m.Stdin, m.Stdout, m.Stderr)
 	default:
@@ -138,135 +132,6 @@ type Command interface {
 	Run(context.Context) error
 }
 
-// CheckCommand represents a command for performing consistency checks on data files.
-type CheckCommand struct {
-	// Data file paths.
-	Paths []string
-
-	// Standard input/output
-	Stdin  io.Reader
-	Stdout io.Writer
-	Stderr io.Writer
-}
-
-// NewCheckCommand returns a new instance of CheckCommand.
-func NewCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *CheckCommand {
-	return &CheckCommand{
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	}
-}
-
-// ParseFlags parses command line flags from args.
-func (cmd *CheckCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	// Parse path.
-	if fs.NArg() == 0 {
-		return errors.New("path required")
-	}
-	cmd.Paths = fs.Args()
-
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *CheckCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl check PATHS...
-
-Performs a consistency check on data files.
-
-`)
-}
-
-// Run executes the main program execution.
-func (cmd *CheckCommand) Run(ctx context.Context) error {
-	for _, path := range cmd.Paths {
-		switch filepath.Ext(path) {
-		case "":
-			if err := cmd.checkBitmapFile(path); err != nil {
-				return err
-			}
-
-		case ".cache":
-			if err := cmd.checkCacheFile(path); err != nil {
-				return err
-			}
-
-		case ".snapshotting":
-			if err := cmd.checkSnapshotFile(path); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// checkBitmapFile performs a consistency check on path for a roaring bitmap file.
-func (cmd *CheckCommand) checkBitmapFile(path string) error {
-	// Open file handle.
-	f, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	fi, err := f.Stat()
-	if err != nil {
-		return err
-	}
-
-	// Memory map the file.
-	data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
-	if err != nil {
-		return err
-	}
-	defer syscall.Munmap(data)
-
-	// Attach the mmap file to the bitmap.
-	bm := roaring.NewBitmap()
-	if err := bm.UnmarshalBinary(data); err != nil {
-		return err
-	}
-
-	// Perform consistency check.
-	if err := bm.Check(); err != nil {
-		// Print returned errors.
-		switch err := err.(type) {
-		case roaring.ErrorList:
-			for i := range err {
-				fmt.Fprintf(cmd.Stdout, "%s: %s\n", path, err[i].Error())
-			}
-		default:
-			fmt.Fprintf(cmd.Stdout, "%s: %s\n", path, err.Error())
-		}
-	}
-
-	// Print success message if no errors were found.
-	fmt.Fprintf(cmd.Stdout, "%s: ok\n", path)
-
-	return nil
-}
-
-// checkCacheFile performs a consistency check on path for a cache file.
-func (cmd *CheckCommand) checkCacheFile(path string) error {
-	fmt.Fprintf(cmd.Stderr, "%s: ignoring cache file\n", path)
-	return nil
-}
-
-// checkSnapshotFile performs a consistency check on path for a snapshot file.
-func (cmd *CheckCommand) checkSnapshotFile(path string) error {
-	fmt.Fprintf(cmd.Stderr, "%s: ignoring snapshot file\n", path)
-	return nil
-}
-
 // BenchCommand represents a command for benchmarking database operations.
 type BenchCommand struct {
 	// Destination host and port.
diff --git a/ctl/check.go b/ctl/check.go
new file mode 100644
index 000000000..616e189bf
--- /dev/null
+++ b/ctl/check.go
@@ -0,0 +1,145 @@
+package ctl
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/pilosa/pilosa/roaring"
+)
+
+// CheckCommand represents a command for performing consistency checks on data files.
+type CheckCommand struct {
+	// Data file paths.
+	Paths []string
+
+	// Standard input/output
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewCheckCommand returns a new instance of CheckCommand.
+func NewCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *CheckCommand {
+	return &CheckCommand{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+	}
+}
+
+// ParseFlags parses command line flags from args.
+func (cmd *CheckCommand) ParseFlags(args []string) error {
+	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
+	fs.SetOutput(ioutil.Discard)
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	// Parse path.
+	if fs.NArg() == 0 {
+		return errors.New("path required")
+	}
+	cmd.Paths = fs.Args()
+
+	return nil
+}
+
+// Usage returns the usage message to be printed.
+func (cmd *CheckCommand) Usage() string {
+	return strings.TrimSpace(`
+usage: pilosactl check PATHS...
+
+Performs a consistency check on data files.
+
+`)
+}
+
+// Run executes the main program execution.
+func (cmd *CheckCommand) Run(ctx context.Context) error {
+	for _, path := range cmd.Paths {
+		switch filepath.Ext(path) {
+		case "":
+			if err := cmd.checkBitmapFile(path); err != nil {
+				return err
+			}
+
+		case ".cache":
+			if err := cmd.checkCacheFile(path); err != nil {
+				return err
+			}
+
+		case ".snapshotting":
+			if err := cmd.checkSnapshotFile(path); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// checkBitmapFile performs a consistency check on path for a roaring bitmap file.
+func (cmd *CheckCommand) checkBitmapFile(path string) error {
+	// Open file handle.
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
+	if err != nil {
+		return err
+	}
+
+	// Memory map the file.
+	data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
+	if err != nil {
+		return err
+	}
+	defer syscall.Munmap(data)
+
+	// Attach the mmap file to the bitmap.
+	bm := roaring.NewBitmap()
+	if err := bm.UnmarshalBinary(data); err != nil {
+		return err
+	}
+
+	// Perform consistency check.
+	if err := bm.Check(); err != nil {
+		// Print returned errors.
+		switch err := err.(type) {
+		case roaring.ErrorList:
+			for i := range err {
+				fmt.Fprintf(cmd.Stdout, "%s: %s\n", path, err[i].Error())
+			}
+		default:
+			fmt.Fprintf(cmd.Stdout, "%s: %s\n", path, err.Error())
+		}
+	}
+
+	// Print success message if no errors were found.
+	fmt.Fprintf(cmd.Stdout, "%s: ok\n", path)
+
+	return nil
+}
+
+// checkCacheFile performs a consistency check on path for a cache file.
+func (cmd *CheckCommand) checkCacheFile(path string) error {
+	fmt.Fprintf(cmd.Stderr, "%s: ignoring cache file\n", path)
+	return nil
+}
+
+// checkSnapshotFile performs a consistency check on path for a snapshot file.
+func (cmd *CheckCommand) checkSnapshotFile(path string) error {
+	fmt.Fprintf(cmd.Stderr, "%s: ignoring snapshot file\n", path)
+	return nil
+}

From 64f007547e88be2690bb0e4da9a5be0687ee1bd6 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 3 Mar 2017 15:14:02 -0600
Subject: [PATCH 25/37] move bench to subcommand

---
 cmd/bench.go          |  43 +++++++++++++
 cmd/pilosactl/main.go |   3 -
 ctl/bench.go          | 143 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 186 insertions(+), 3 deletions(-)
 create mode 100644 cmd/bench.go
 create mode 100644 ctl/bench.go

diff --git a/cmd/bench.go b/cmd/bench.go
new file mode 100644
index 000000000..9b3e85399
--- /dev/null
+++ b/cmd/bench.go
@@ -0,0 +1,43 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/pilosa/pilosa/ctl"
+)
+
+var bencher = ctl.NewBenchCommand(os.Stdin, os.Stdout, os.Stderr)
+
+var benchCmd = &cobra.Command{
+	Use:   "bench",
+	Short: "bench - benchmark operations",
+	Long: `
+Executes a benchmark for a given operation against the database.
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := bencher.Run(context.Background()); err != nil {
+			fmt.Println(err)
+		}
+	},
+}
+
+func init() {
+	benchCmd.Flags().StringVarP(&bencher.Host, "host", "", "localhost:15000", "host:port of Pilosa.")
+	benchCmd.Flags().StringVarP(&bencher.Database, "database", "d", "", "Pilosa database to benchmark.")
+	benchCmd.Flags().StringVarP(&bencher.Frame, "frame", "f", "", "Frame to benchmark.")
+	benchCmd.Flags().StringVarP(&bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]")
+	benchCmd.Flags().IntVarP(&bencher.N, "num", "n", 0, "Number of operations to perform.")
+
+	err := viper.BindPFlags(benchCmd.Flags())
+	if err != nil {
+		log.Fatalf("Error binding bench flags: %v", err)
+	}
+
+	RootCmd.AddCommand(benchCmd)
+}
diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go
index 401d1cff5..417ebaead 100644
--- a/cmd/pilosactl/main.go
+++ b/cmd/pilosactl/main.go
@@ -85,7 +85,6 @@ Usage:
 
 The commands are:
 
-	bench      benchmarks operations
 
 Use the "-h" flag with any command for more information.
 `)
@@ -107,8 +106,6 @@ func (m *Main) ParseFlags(args []string) error {
 		fmt.Fprintln(m.Stderr, m.Usage())
 		fmt.Fprintln(m.Stderr, "")
 		return flag.ErrHelp
-	case "bench":
-		m.Cmd = NewBenchCommand(m.Stdin, m.Stdout, m.Stderr)
 	default:
 		return ErrUnknownCommand
 	}
diff --git a/ctl/bench.go b/ctl/bench.go
new file mode 100644
index 000000000..7e789b5cd
--- /dev/null
+++ b/ctl/bench.go
@@ -0,0 +1,143 @@
+package ctl
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"strings"
+	"time"
+
+	"github.com/pilosa/pilosa"
+)
+
+// BenchCommand represents a command for benchmarking database operations.
+type BenchCommand struct {
+	// Destination host and port.
+	Host string
+
+	// Name of the database & frame to execute against.
+	Database string
+	Frame    string
+
+	// Type of operation and number to execute.
+	Op string
+	N  int
+
+	// Standard input/output
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewBenchCommand returns a new instance of BenchCommand.
+func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *BenchCommand {
+	return &BenchCommand{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+	}
+}
+
+// ParseFlags parses command line flags from args.
+func (cmd *BenchCommand) ParseFlags(args []string) error {
+	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
+	fs.SetOutput(ioutil.Discard)
+	fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port")
+	fs.StringVar(&cmd.Database, "d", "", "database")
+	fs.StringVar(&cmd.Frame, "f", "", "frame")
+	fs.StringVar(&cmd.Op, "op", "", "operation")
+	fs.IntVar(&cmd.N, "n", 0, "op count")
+
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Usage returns the usage message to be printed.
+func (cmd *BenchCommand) Usage() string {
+	return strings.TrimSpace(`
+usage: pilosactl bench [args]
+
+Executes a benchmark for a given operation against the database.
+
+The following flags are allowed:
+
+	-host HOSTPORT
+		hostname and port of running pilosa server
+
+	-d DATABASE
+		database to execute operation against
+
+	-f FRAME
+		frame to execute operation against
+
+	-op OP
+		name of operation to execute
+
+	-n COUNT
+		number of iterations to execute
+
+The following operations are available:
+
+	set-bit
+		Sets a single random bit on the frame
+
+`)
+}
+
+// Run executes the main program execution.
+func (cmd *BenchCommand) Run(ctx context.Context) error {
+	// Create a client to the server.
+	client, err := pilosa.NewClient(cmd.Host)
+	if err != nil {
+		return err
+	}
+
+	switch cmd.Op {
+	case "set-bit":
+		return cmd.runSetBit(ctx, client)
+	case "":
+		return errors.New("op required")
+	default:
+		return fmt.Errorf("unknown bench op: %q", cmd.Op)
+	}
+}
+
+// runSetBit executes a benchmark of random SetBit() operations.
+func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) error {
+	if cmd.N == 0 {
+		return errors.New("operation count required")
+	} else if cmd.Database == "" {
+		return pilosa.ErrDatabaseRequired
+	} else if cmd.Frame == "" {
+		return pilosa.ErrFrameRequired
+	}
+
+	const maxBitmapID = 1000
+	const maxProfileID = 100000
+
+	startTime := time.Now()
+
+	// Execute operation continuously.
+	for i := 0; i < cmd.N; i++ {
+		bitmapID := rand.Intn(maxBitmapID)
+		profileID := rand.Intn(maxProfileID)
+
+		q := fmt.Sprintf(`SetBit(id=%d, frame="%s", profileID=%d)`, bitmapID, cmd.Frame, profileID)
+
+		if _, err := client.ExecuteQuery(ctx, cmd.Database, q, true); err != nil {
+			return err
+		}
+	}
+
+	// Print results.
+	elapsed := time.Since(startTime)
+	fmt.Fprintf(cmd.Stdout, "Executed %d operations in %s (%0.3f op/sec)\n", cmd.N, elapsed, float64(cmd.N)/elapsed.Seconds())
+
+	return nil
+}

From a47f93329ea7aa55d22904df6f6875f7e2e08c49 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 3 Mar 2017 15:24:39 -0600
Subject: [PATCH 26/37] remove pilosactl and add version/build to root cmd

---
 cmd/pilosactl/main.go      | 258 -------------------------------------
 cmd/pilosactl/main_test.go |   1 -
 cmd/root.go                |  21 ++-
 3 files changed, 20 insertions(+), 260 deletions(-)
 delete mode 100644 cmd/pilosactl/main.go
 delete mode 100644 cmd/pilosactl/main_test.go

diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go
deleted file mode 100644
index 417ebaead..000000000
--- a/cmd/pilosactl/main.go
+++ /dev/null
@@ -1,258 +0,0 @@
-package main
-
-import (
-	"context"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"strings"
-	"time"
-
-	"github.com/pilosa/pilosa"
-)
-
-var (
-	// ErrUnknownCommand is returned when specifying an unknown command.
-	ErrUnknownCommand = errors.New("unknown command")
-
-	// ErrPathRequired is returned when executing a command without a required path.
-	ErrPathRequired = errors.New("path required")
-	Version         string
-	BuildTime       string
-)
-
-func init() {
-	if Version == "" {
-		Version = "v0.0.0"
-	}
-	if BuildTime == "" {
-		BuildTime = "not recorded"
-	}
-}
-
-func main() {
-	m := NewMain()
-
-	fmt.Fprintf(m.Stderr, "Pilosactl %s, build time %s\n", Version, BuildTime)
-
-	// Parse command line arguments.
-	if err := m.ParseFlags(os.Args[1:]); err == flag.ErrHelp {
-		os.Exit(2)
-	} else if err != nil {
-		fmt.Fprintln(m.Stderr, err)
-		os.Exit(2)
-	}
-
-	// Execute the program.
-	if err := m.Run(); err != nil {
-		fmt.Fprintln(m.Stderr, err)
-		os.Exit(1)
-	}
-}
-
-// Main represents the main program execution.
-type Main struct {
-	// Subcommand to execute.
-	Cmd Command
-
-	// Standard input/output
-	Stdin  io.Reader
-	Stdout io.Writer
-	Stderr io.Writer
-}
-
-// NewMain returns a new instance of Main.
-func NewMain() *Main {
-	return &Main{
-		Stdin:  os.Stdin,
-		Stdout: os.Stdout,
-		Stderr: os.Stderr,
-	}
-}
-
-// Usage returns the usage message to be printed.
-func (m *Main) Usage() string {
-	return strings.TrimSpace(`
-Pilosactl is a tool for interacting with a pilosa server.
-
-Usage:
-
-	pilosactl command [arguments]
-
-The commands are:
-
-
-Use the "-h" flag with any command for more information.
-`)
-}
-
-// Run executes the main program execution.
-func (m *Main) Run() error { return m.Cmd.Run(context.Background()) }
-
-// ParseFlags parses command line flags from args.
-func (m *Main) ParseFlags(args []string) error {
-	var command string
-	if len(args) > 0 {
-		command = args[0]
-		args = args[1:]
-	}
-
-	switch command {
-	case "", "help", "-h":
-		fmt.Fprintln(m.Stderr, m.Usage())
-		fmt.Fprintln(m.Stderr, "")
-		return flag.ErrHelp
-	default:
-		return ErrUnknownCommand
-	}
-
-	// Parse command's flags.
-	if err := m.Cmd.ParseFlags(args); err == flag.ErrHelp {
-		fmt.Fprintln(m.Stderr, m.Cmd.Usage())
-		fmt.Fprintln(m.Stderr, "")
-		return err
-	} else if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Command represents an executable subcommand.
-type Command interface {
-	Usage() string
-	ParseFlags(args []string) error
-	Run(context.Context) error
-}
-
-// BenchCommand represents a command for benchmarking database operations.
-type BenchCommand struct {
-	// Destination host and port.
-	Host string
-
-	// Name of the database & frame to execute against.
-	Database string
-	Frame    string
-
-	// Type of operation and number to execute.
-	Op string
-	N  int
-
-	// Standard input/output
-	Stdin  io.Reader
-	Stdout io.Writer
-	Stderr io.Writer
-}
-
-// NewBenchCommand returns a new instance of BenchCommand.
-func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *BenchCommand {
-	return &BenchCommand{
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	}
-}
-
-// ParseFlags parses command line flags from args.
-func (cmd *BenchCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port")
-	fs.StringVar(&cmd.Database, "d", "", "database")
-	fs.StringVar(&cmd.Frame, "f", "", "frame")
-	fs.StringVar(&cmd.Op, "op", "", "operation")
-	fs.IntVar(&cmd.N, "n", 0, "op count")
-
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *BenchCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl bench [args]
-
-Executes a benchmark for a given operation against the database.
-
-The following flags are allowed:
-
-	-host HOSTPORT
-		hostname and port of running pilosa server
-
-	-d DATABASE
-		database to execute operation against
-
-	-f FRAME
-		frame to execute operation against
-
-	-op OP
-		name of operation to execute
-
-	-n COUNT
-		number of iterations to execute
-
-The following operations are available:
-
-	set-bit
-		Sets a single random bit on the frame
-
-`)
-}
-
-// Run executes the main program execution.
-func (cmd *BenchCommand) Run(ctx context.Context) error {
-	// Create a client to the server.
-	client, err := pilosa.NewClient(cmd.Host)
-	if err != nil {
-		return err
-	}
-
-	switch cmd.Op {
-	case "set-bit":
-		return cmd.runSetBit(ctx, client)
-	case "":
-		return errors.New("op required")
-	default:
-		return fmt.Errorf("unknown bench op: %q", cmd.Op)
-	}
-}
-
-// runSetBit executes a benchmark of random SetBit() operations.
-func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) error {
-	if cmd.N == 0 {
-		return errors.New("operation count required")
-	} else if cmd.Database == "" {
-		return pilosa.ErrDatabaseRequired
-	} else if cmd.Frame == "" {
-		return pilosa.ErrFrameRequired
-	}
-
-	const maxBitmapID = 1000
-	const maxProfileID = 100000
-
-	startTime := time.Now()
-
-	// Execute operation continuously.
-	for i := 0; i < cmd.N; i++ {
-		bitmapID := rand.Intn(maxBitmapID)
-		profileID := rand.Intn(maxProfileID)
-
-		q := fmt.Sprintf(`SetBit(id=%d, frame="%s", profileID=%d)`, bitmapID, cmd.Frame, profileID)
-
-		if _, err := client.ExecuteQuery(ctx, cmd.Database, q, true); err != nil {
-			return err
-		}
-	}
-
-	// Print results.
-	elapsed := time.Since(startTime)
-	fmt.Fprintf(cmd.Stdout, "Executed %d operations in %s (%0.3f op/sec)\n", cmd.N, elapsed, float64(cmd.N)/elapsed.Seconds())
-
-	return nil
-}
diff --git a/cmd/pilosactl/main_test.go b/cmd/pilosactl/main_test.go
deleted file mode 100644
index 0fee6f5dc..000000000
--- a/cmd/pilosactl/main_test.go
+++ /dev/null
@@ -1 +0,0 @@
-package main_test
diff --git a/cmd/root.go b/cmd/root.go
index 91d866a18..f7517dc73 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -2,13 +2,32 @@ package cmd
 
 import "github.com/spf13/cobra"
 
+var (
+	Version   string
+	BuildTime string
+)
+
 var RootCmd = &cobra.Command{
 	Use:   "pilosa",
 	Short: "pilosa - A Distributed In-memory Binary Bitmap Index",
+	// TODO - is documentation actually there?
 	Long: `Pilosa is a fast index to turbocharge your database.
 
 This binary contains Pilosa itself, as well as common
 tools for administering pilosa, importing/exporting data,
 backing up, and more. Complete documentation is available
-at http://pilosa.com/docs`, // TODO - is documentation actually there?
+at http://pilosa.com/docs
+
+`,
+}
+
+func init() {
+	if Version == "" {
+		Version = "v0.0.0"
+	}
+	if BuildTime == "" {
+		BuildTime = "not recorded"
+	}
+
+	RootCmd.Long = RootCmd.Long + "Version: " + Version + "\nBuild Time: " + BuildTime + "\n"
 }

From 84c8484bb031e43cbd3b396832426349ed35972b Mon Sep 17 00:00:00 2001
From: Travis <travis@pilosa.com>
Date: Sat, 4 Mar 2017 12:37:02 -0600
Subject: [PATCH 27/37] have roaring clone() return the clone (as opposed to
 the original)

---
 roaring/roaring.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 1cf271e89..2a0dc57d1 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1057,7 +1057,7 @@ func (c *container) clone() *container {
 		copy(other.bitmap, c.bitmap)
 	}
 
-	return c
+	return other
 }
 
 // WriteTo writes c to w.

From 756c44c17d0e775bc795ab91af195315eed2b083 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Mon, 6 Mar 2017 10:19:50 -0600
Subject: [PATCH 28/37] fix bugs with pilosa server -config and remove dead
 code

---
 cmd/server.go    |  2 +-
 server/server.go | 65 +-----------------------------------------------
 2 files changed, 2 insertions(+), 65 deletions(-)

diff --git a/cmd/server.go b/cmd/server.go
index cc8bbec49..d31806eea 100644
--- a/cmd/server.go
+++ b/cmd/server.go
@@ -29,7 +29,7 @@ on the configured port.`,
 		fmt.Fprintf(serve.Stderr, "Pilosa %s, build time %s\n", server.Version, server.BuildTime)
 
 		// Parse command line arguments.
-		if err := serve.ParseFlags(os.Args[1:]); err != nil {
+		if err := serve.SetupConfig(args); err != nil {
 			fmt.Fprintln(serve.Stderr, err)
 			os.Exit(2)
 		}
diff --git a/server/server.go b/server/server.go
index bb151a37d..247d0efa9 100644
--- a/server/server.go
+++ b/server/server.go
@@ -2,14 +2,11 @@ package server
 
 import (
 	"errors"
-	"flag"
 	"fmt"
 	"io"
 	"math/rand"
 	"os"
-	"os/signal"
 	"path/filepath"
-	"runtime/pprof"
 	"strings"
 	"time"
 
@@ -39,57 +36,6 @@ const (
 	DefaultDataDir = "~/.pilosa"
 )
 
-func mainz() {
-	serve := NewMain()
-	serve.Server.Handler.Version = Version
-	fmt.Fprintf(serve.Stderr, "Pilosa %s, build time %s\n", Version, BuildTime)
-
-	// Parse command line arguments.
-	if err := serve.ParseFlags(os.Args[1:]); err != nil {
-		fmt.Fprintln(serve.Stderr, err)
-		os.Exit(2)
-	}
-
-	// Start CPU profiling.
-	if serve.CPUProfile != "" {
-		f, err := os.Create(serve.CPUProfile)
-		if err != nil {
-			fmt.Fprintf(serve.Stderr, "create cpu profile: %v", err)
-			os.Exit(1)
-		}
-		defer f.Close()
-
-		fmt.Fprintln(serve.Stderr, "Starting cpu profile")
-		pprof.StartCPUProfile(f)
-		time.AfterFunc(serve.CPUTime, func() {
-			fmt.Fprintln(serve.Stderr, "Stopping cpu profile")
-			pprof.StopCPUProfile()
-			f.Close()
-		})
-	}
-
-	// Execute the program.
-	if err := serve.Run(); err != nil {
-		fmt.Fprintln(serve.Stderr, err)
-		fmt.Fprintln(serve.Stderr, "stopping profile")
-		os.Exit(1)
-	}
-
-	// First SIGKILL causes server to shut down gracefully.
-	c := make(chan os.Signal, 2)
-	signal.Notify(c, os.Interrupt)
-	sig := <-c
-	fmt.Fprintf(serve.Stderr, "Received %s; gracefully shutting down...\n", sig.String())
-
-	// Second signal causes a hard shutdown.
-	go func() { <-c; os.Exit(1) }()
-
-	if err := serve.Close(); err != nil {
-		fmt.Fprintln(serve.Stderr, err)
-		os.Exit(1)
-	}
-}
-
 // Main represents the main program execution.
 type Main struct {
 	Server *pilosa.Server
@@ -158,16 +104,7 @@ func (m *Main) Close() error {
 }
 
 // ParseFlags parses command line flags from args.
-func (m *Main) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosa", flag.ContinueOnError)
-	fs.StringVar(&m.CPUProfile, "cpuprofile", "", "cpu profile")
-	fs.DurationVar(&m.CPUTime, "cputime", 30*time.Second, "cpu profile duration")
-	fs.StringVar(&m.ConfigPath, "config", "", "config path")
-	fs.SetOutput(m.Stderr)
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
+func (m *Main) SetupConfig(args []string) error {
 	// Load config, if specified.
 	if m.ConfigPath != "" {
 		if _, err := toml.DecodeFile(m.ConfigPath, &m.Config); err != nil {

From c6a502efed6f089306f821b4452c6d8ea31efd79 Mon Sep 17 00:00:00 2001
From: Linh Vo <lvo@umbel.com>
Date: Mon, 6 Mar 2017 11:03:17 -0600
Subject: [PATCH 29/37] fix review

---
 client.go | 10 ----------
 db.go     |  6 +++---
 frame.go  |  6 +++---
 pilosa.go | 15 ++++-----------
 4 files changed, 10 insertions(+), 27 deletions(-)

diff --git a/client.go b/client.go
index bfe4dc57e..e7d0eaf85 100644
--- a/client.go
+++ b/client.go
@@ -194,11 +194,6 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire
 		return nil, ErrQueryRequired
 	}
 
-	er := ValidateName(db)
-	if er != nil {
-		return nil, ErrName
-	}
-
 	// Encode query request.
 	buf, err := proto.Marshal(&internal.QueryRequest{
 		DB:     db,
@@ -255,11 +250,6 @@ func (c *Client) ExecutePQL(ctx context.Context, db, query string) (interface{},
 		}.Encode(),
 	}
 
-	er := ValidateName(db)
-	if er != nil {
-		return nil, ErrName
-	}
-
 	req, err := http.NewRequest("POST", u.String(), bytes.NewReader([]byte(query)))
 	if err != nil {
 		return nil, err
diff --git a/db.go b/db.go
index 9348c97b6..6ad81e16b 100644
--- a/db.go
+++ b/db.go
@@ -49,9 +49,9 @@ type DB struct {
 
 // NewDB returns a new instance of DB.
 func NewDB(path, name string) (*DB, error) {
-	err := ValidateName(name)
-	if err != nil {
-		return nil, err
+	validName := Exp.FindStringSubmatchIndex(name)
+	if len(validName) == 0 {
+		return nil, ErrName
 	}
 
 	return &DB{
diff --git a/frame.go b/frame.go
index fa47517c5..368cde821 100644
--- a/frame.go
+++ b/frame.go
@@ -47,9 +47,9 @@ type Frame struct {
 
 // NewFrame returns a new instance of frame.
 func NewFrame(path, db, name string) (*Frame, error) {
-	err := ValidateName(name)
-	if err != nil {
-		return nil, err
+	validName := Exp.FindStringSubmatchIndex(name)
+	if len(validName) == 0 {
+		return nil, ErrName
 	}
 
 	return &Frame{
diff --git a/pilosa.go b/pilosa.go
index 14e5ef203..b3cee7450 100644
--- a/pilosa.go
+++ b/pilosa.go
@@ -20,13 +20,16 @@ var (
 	ErrFrameNotFound = errors.New("frame not found")
 
 	// ErrFrameRequired is returned when no frame is specified.
-	ErrName = errors.New("name restricted to [a-z0-9_-.]")
+	ErrName = errors.New("name restricted to [a-z0-9_-]")
 
 	// ErrFragmentNotFound is returned when a fragment does not exist.
 	ErrFragmentNotFound = errors.New("fragment not found")
 	ErrQueryRequired    = errors.New("query required")
 )
 
+// Regular expression to valuate db and frame's name
+var Exp = regexp.MustCompile(`^([a-z0-9._-]{1,64}$)`)
+
 // Profile represents vertical column in a database.
 // A profile can have a set of attributes attached to it.
 type Profile struct {
@@ -79,13 +82,3 @@ func decodeProfile(pb *internal.Profile) *Profile {
 
 // TimeFormat is the go-style time format used to parse string dates.
 const TimeFormat = "2006-01-02T15:04"
-
-// Restrict name using regex
-func ValidateName(name string) error {
-	expr := regexp.MustCompile(`^([a-z0-9._-]{1,64}$)`)
-	validName := expr.FindStringSubmatchIndex(name)
-	if len(validName) == 0 {
-		return ErrName
-	}
-	return nil
-}

From 28580fd3540f3f793c71124ec26c09d479c0940e Mon Sep 17 00:00:00 2001
From: Linh Vo <lvo@umbel.com>
Date: Mon, 6 Mar 2017 11:10:23 -0600
Subject: [PATCH 30/37] remove validate name from client

---
 pilosactl/import.go | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/pilosactl/import.go b/pilosactl/import.go
index 80e9a2973..313c5da99 100644
--- a/pilosactl/import.go
+++ b/pilosactl/import.go
@@ -104,16 +104,6 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 	} else if len(cmd.Paths) == 0 {
 		return errors.New("path required")
 	}
-	// Restrict frame name and database name with regex
-	dbError := pilosa.ValidateName(cmd.Database)
-	if dbError != nil {
-		return dbError
-	}
-
-	frameError := pilosa.ValidateName(cmd.Frame)
-	if frameError != nil {
-		return frameError
-	}
 	// Create a client to the server.
 	client, err := pilosa.NewClient(cmd.Host)
 	if err != nil {

From c419c082da0a13ac0ed285853fe316f58f42baa0 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Mon, 6 Mar 2017 11:18:16 -0600
Subject: [PATCH 31/37] update readme to reflect subcommands and pilosactl gone

---
 README.md | 92 +++++--------------------------------------------------
 1 file changed, 8 insertions(+), 84 deletions(-)

diff --git a/README.md b/README.md
index bd6aacbe2..148a7a139 100644
--- a/README.md
+++ b/README.md
@@ -23,18 +23,16 @@ $ go install github.com/pilosa/pilosa/cmd/...
 Now run a single pilosa node with the default configuration:
 
 ```sh
-pilosa
+pilosa server
 ```
 
-If you would like to quickly create a multi-node pilosa cluster, see the `pilosactl create` documentation.
-
 ## Configuration
 
 You can specify a configuration by setting the `-config` flag when running `pilosa`.
 
 
 ```sh
-pilosa -config custom-config-file.cfg
+pilosa server --config custom-config-file.cfg
 ```
 
 The config file uses the [TOML](https://github.com/toml-lang/toml) configuration file format,
@@ -54,6 +52,12 @@ host = "127.0.0.1:15000"
 host = "127.0.0.1:15001"
 ```
 
+You can generate a template config file with default values with:
+
+```sh
+pilosa config
+```
+
 The first two configuration options will be unique to each node in the cluster:
 
 `data-dir`: directory in which data is stored to disk
@@ -242,83 +246,3 @@ $ go install --ldflags="-X main.Version=1.0.0"
 ```
 
 [Glide]: http://glide.sh/
-
-## Pilosactl
-
-Pilosactl contains a suite of tools for interacting with pilosa. Run `pilosactl` for an overview of commands, and `pilosactl <command> -h` for specific information on that command.
-
-### Create
-
-`pilosactl create` is used to create pilosa clusters. It has a number of options for controlling how the cluster is configured, what hosts it is on, and even the ability to build the pilosa binary locally and copy it to each cluster node automatically. To start pilosa on remote hosts, you only need `ssh` access to those hosts. See `pilosactl create -h` for a full list of options.
-
-Examples:
-
-Create a 5 node cluster locally (using 5 different ports), with a replication factor of 2.
-```
-pilosactl create \
-  -serverN 5 \
-  -replicaN 2
-```
-
-Create a cluster on 3 remote hosts - all logs will come to local stderr, pilosa binary must be available on remote hosts. The ssh user on the remote hosts needs to be the same as your local user. Otherwise use the `ssh-user` option.
-```
-pilosactl create \
-  -hosts="node1.example.com:15000,node2.example.com:15000,node3.example.com:15000"
-```
-
-Create a cluster on 3 remote hosts running OSX, but build the binary locally and copy it up. Stream the stderr of each node to a separate local log file.
-```
-pilosactl create \
-  -hosts="mac1.example.com:15000,mac2.example.com:15000,mac3.example.com:15000" \
-  -copy-binary \
-  -goos=darwin \
-  -goarch=amd64 \
-  -log-file-prefix=clusterlogs
-```
-
-### Bagent
-
-`pilosactl bagent` is what you want if you just want to run a simple benchmark against an existing cluster. Running it with no arguments will print some help, including the set of subcommands that it may be passed. Calling a subcommand with `-h'` will print the options for that subcommand. The `agent-num` flag can be passed an integer which can change the behavior the benchmarks that are run. This is useful when multiple invocations of the same benchmark are made by the `bspawn` command - they can each (for example) set different bits even though they all have the same arguments.
-
-E.G.
-```
-pilosactl bagent \
-  -hosts="localhost:15000,localhost:15001" \
-  import -h
-```
-
-Multiple subcommands and their arguments may be concatenated at the command line and they will be run serially. This is useful (i.e.) for importing a bunch of data, and then executing queries against it.
-
-This will generate and import a bunch of data, and then execute random queries against it.
-
-```
-pilosactl bagent \
-  -hosts="localhost:15000,localhost:15001" \
-  import -max-bits-per-map=10000 \
-  random-query -iterations 100
-```
-
-### Bspawn
-`pilosactl bspawn` allows you to automate the creation of clusters and the running of complex benchmarks which span multiple benchmark agents against them. It has a number of options which are described by `pilosactl bspawn` with no arguments, and also takes a config file which describes the Benchmark itself - this file is described below.
-
-#### Configuration Format
-
-The configuration file is a json object with the top level key `benchmarks`. This contains a list of objects each of which represents a `bagent` command (the `args` key) that will be run some number of times concurrently (the `num` key), and a `name` which should describe the overall effect that command. An example is below.
-```json
-{
-    "benchmarks": [
-        {
-            "num": 3,
-            "name": "set-diags",
-            "args": ["diagonal-set-bits", "-iterations", "30000", "-client-type", "round_robin"]
-        },
-        {
-            "num": 2,
-            "name": "rand-plus-zipf",
-            "args": ["random-set-bits", "-iterations", "20000", "zipf", "-iterations", "100"]
-        }
-    ]
-}
-```
-
-All of the benchmarks, and agents are run concurrently. Each agent will be passed an `agent-num` which can modify the behavior in a way that is benchmark specific. See the documentation for each benchmark to see how `agent-num` changes its behavior.

From 4e31d02979d9ece4099adcfaf37f0d510473e7b0 Mon Sep 17 00:00:00 2001
From: Linh Vo <lvo@umbel.com>
Date: Mon, 6 Mar 2017 12:31:51 -0600
Subject: [PATCH 32/37] wrap regexp in function

---
 db.go     |  6 +++---
 frame.go  |  6 +++---
 pilosa.go | 13 ++++++++++++-
 3 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/db.go b/db.go
index 6ad81e16b..9348c97b6 100644
--- a/db.go
+++ b/db.go
@@ -49,9 +49,9 @@ type DB struct {
 
 // NewDB returns a new instance of DB.
 func NewDB(path, name string) (*DB, error) {
-	validName := Exp.FindStringSubmatchIndex(name)
-	if len(validName) == 0 {
-		return nil, ErrName
+	err := ValidateName(name)
+	if err != nil {
+		return nil, err
 	}
 
 	return &DB{
diff --git a/frame.go b/frame.go
index 368cde821..fa47517c5 100644
--- a/frame.go
+++ b/frame.go
@@ -47,9 +47,9 @@ type Frame struct {
 
 // NewFrame returns a new instance of frame.
 func NewFrame(path, db, name string) (*Frame, error) {
-	validName := Exp.FindStringSubmatchIndex(name)
-	if len(validName) == 0 {
-		return nil, ErrName
+	err := ValidateName(name)
+	if err != nil {
+		return nil, err
 	}
 
 	return &Frame{
diff --git a/pilosa.go b/pilosa.go
index b3cee7450..b3620a900 100644
--- a/pilosa.go
+++ b/pilosa.go
@@ -28,7 +28,8 @@ var (
 )
 
 // Regular expression to valuate db and frame's name
-var Exp = regexp.MustCompile(`^([a-z0-9._-]{1,64}$)`)
+// Todo: remove . when frame doesn't require . for topN
+var nameRegexp = regexp.MustCompile(`^([a-z0-9._-]{1,64}$)`)
 
 // Profile represents vertical column in a database.
 // A profile can have a set of attributes attached to it.
@@ -82,3 +83,13 @@ func decodeProfile(pb *internal.Profile) *Profile {
 
 // TimeFormat is the go-style time format used to parse string dates.
 const TimeFormat = "2006-01-02T15:04"
+
+
+// Restrict name using regex
+func ValidateName(name string) error {
+	validName := nameRegexp.Match([]byte(name))
+	if validName == false{
+		return ErrName
+	}
+	return nil
+}
\ No newline at end of file

From f463ab9d9b19177e5c8e9bc4df32d0b68d1fbdab Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Mon, 6 Mar 2017 16:37:41 -0600
Subject: [PATCH 33/37] make inspect behave like pilosactl version

---
 cmd/inspect.go | 19 +++++++++----------
 cmd/sort.go    |  1 -
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/cmd/inspect.go b/cmd/inspect.go
index 48bbf8308..56ded3bb7 100644
--- a/cmd/inspect.go
+++ b/cmd/inspect.go
@@ -3,11 +3,9 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"log"
 	"os"
 
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 
 	"github.com/pilosa/pilosa/ctl"
 )
@@ -16,11 +14,19 @@ var inspecter = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr)
 
 var inspectCmd = &cobra.Command{
 	Use:   "inspect",
-	Short: "inspect - inspect a pilosa data file",
+	Short: "get stats on pilosa data file",
 	Long: `
 Inspects a data file and provides stats.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
+		if len(args) == 0 {
+			fmt.Println("path required")
+			return
+		} else if len(args) > 1 {
+			fmt.Println("only one path allowed")
+			return
+		}
+		inspecter.Path = args[0]
 		if err := inspecter.Run(context.Background()); err != nil {
 			fmt.Println(err)
 		}
@@ -28,12 +34,5 @@ Inspects a data file and provides stats.
 }
 
 func init() {
-	inspectCmd.Flags().StringVarP(&inspecter.Path, "file", "i", "", "File to inspect")
-
-	err := viper.BindPFlags(inspectCmd.Flags())
-	if err != nil {
-		log.Fatalf("Error binding inspect flags: %v", err)
-	}
-
 	RootCmd.AddCommand(inspectCmd)
 }
diff --git a/cmd/sort.go b/cmd/sort.go
index 813624ddc..4df585629 100644
--- a/cmd/sort.go
+++ b/cmd/sort.go
@@ -25,7 +25,6 @@ The format of the CSV file is:
 The file should contain no headers.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println(cmd.Flags())
 		if len(args) == 0 {
 			fmt.Println("path required")
 			return

From c7caea6b30dacd853d7c6482ef295e983ab1a408 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Mon, 6 Mar 2017 16:45:37 -0600
Subject: [PATCH 34/37] fix short help strings on commands

---
 cmd/backup.go  | 2 +-
 cmd/bench.go   | 2 +-
 cmd/check.go   | 2 +-
 cmd/config.go  | 2 +-
 cmd/export.go  | 2 +-
 cmd/import.go  | 2 +-
 cmd/inspect.go | 2 +-
 cmd/restore.go | 2 +-
 cmd/root.go    | 2 +-
 cmd/server.go  | 2 +-
 cmd/sort.go    | 2 +-
 11 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/cmd/backup.go b/cmd/backup.go
index 883ce7be8..418f6a720 100644
--- a/cmd/backup.go
+++ b/cmd/backup.go
@@ -16,7 +16,7 @@ var backuper = ctl.NewBackupCommand(os.Stdin, os.Stdout, os.Stderr)
 
 var backupCmd = &cobra.Command{
 	Use:   "backup",
-	Short: "backup - backup data from pilosa",
+	Short: "Backup data from pilosa.",
 	Long: `
 Backs up the database and frame from across the cluster into a single file.
 `,
diff --git a/cmd/bench.go b/cmd/bench.go
index 9b3e85399..3e093d68e 100644
--- a/cmd/bench.go
+++ b/cmd/bench.go
@@ -16,7 +16,7 @@ var bencher = ctl.NewBenchCommand(os.Stdin, os.Stdout, os.Stderr)
 
 var benchCmd = &cobra.Command{
 	Use:   "bench",
-	Short: "bench - benchmark operations",
+	Short: "Benchmark operations.",
 	Long: `
 Executes a benchmark for a given operation against the database.
 `,
diff --git a/cmd/check.go b/cmd/check.go
index 541f33cb2..560130187 100644
--- a/cmd/check.go
+++ b/cmd/check.go
@@ -14,7 +14,7 @@ var checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr)
 
 var checkCmd = &cobra.Command{
 	Use:   "check <path> [path2]...",
-	Short: "check - check a pilosa data file",
+	Short: "Do a consistency check on a pilosa data file.",
 	Long: `
 Performs a consistency check on data files.
 `,
diff --git a/cmd/config.go b/cmd/config.go
index 72202ffe3..5fccbfc7d 100644
--- a/cmd/config.go
+++ b/cmd/config.go
@@ -14,7 +14,7 @@ var conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr)
 
 var confCmd = &cobra.Command{
 	Use:   "config",
-	Short: "config - prints the default configuration",
+	Short: "Print the default configuration.",
 	Long: `config prints the default configuration to stdout
 `,
 	Run: func(cmd *cobra.Command, args []string) {
diff --git a/cmd/export.go b/cmd/export.go
index a5cd0f62d..06a76118d 100644
--- a/cmd/export.go
+++ b/cmd/export.go
@@ -16,7 +16,7 @@ var exporter = ctl.NewExportCommand(os.Stdin, os.Stdout, os.Stderr)
 
 var exportCmd = &cobra.Command{
 	Use:   "export",
-	Short: "export - export data from pilosa",
+	Short: "Export data from pilosa.",
 	Long: `
 Bulk exports a fragment to a CSV file. If the OUTFILE is not specified then
 the output is written to STDOUT.
diff --git a/cmd/import.go b/cmd/import.go
index af1a7fcd5..5f878c431 100644
--- a/cmd/import.go
+++ b/cmd/import.go
@@ -16,7 +16,7 @@ var importer = ctl.NewImportCommand(os.Stdin, os.Stdout, os.Stderr)
 
 var importCmd = &cobra.Command{
 	Use:   "import",
-	Short: "import - import data to pilosa",
+	Short: "Bulk load data into pilosa.",
 	Long: `Bulk imports one or more CSV files to a host's database and frame. The bits
 of the CSV file are grouped by slice for the most efficient import.
 
diff --git a/cmd/inspect.go b/cmd/inspect.go
index 56ded3bb7..666c59d5c 100644
--- a/cmd/inspect.go
+++ b/cmd/inspect.go
@@ -14,7 +14,7 @@ var inspecter = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr)
 
 var inspectCmd = &cobra.Command{
 	Use:   "inspect",
-	Short: "get stats on pilosa data file",
+	Short: "Get stats on a pilosa data file.",
 	Long: `
 Inspects a data file and provides stats.
 `,
diff --git a/cmd/restore.go b/cmd/restore.go
index 44899e4ca..399122ff3 100644
--- a/cmd/restore.go
+++ b/cmd/restore.go
@@ -16,7 +16,7 @@ var restorer = ctl.NewRestoreCommand(os.Stdin, os.Stdout, os.Stderr)
 
 var restoreCmd = &cobra.Command{
 	Use:   "restore",
-	Short: "restore - restore data to pilosa",
+	Short: "Restore data to pilosa from a backup file.",
 	Long: `
 Restores a frame to the cluster from a backup file.
 `,
diff --git a/cmd/root.go b/cmd/root.go
index f7517dc73..139214302 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -9,7 +9,7 @@ var (
 
 var RootCmd = &cobra.Command{
 	Use:   "pilosa",
-	Short: "pilosa - A Distributed In-memory Binary Bitmap Index",
+	Short: "Pilosa - A Distributed In-memory Binary Bitmap Index.",
 	// TODO - is documentation actually there?
 	Long: `Pilosa is a fast index to turbocharge your database.
 
diff --git a/cmd/server.go b/cmd/server.go
index d31806eea..6603a0989 100644
--- a/cmd/server.go
+++ b/cmd/server.go
@@ -18,7 +18,7 @@ var serve = server.NewMain()
 
 var serveCmd = &cobra.Command{
 	Use:   "server",
-	Short: "server - run the pilosa server",
+	Short: "Run Pilosa.",
 	Long: `pilosa server runs Pilosa.
 
 It will load existing data from the configured
diff --git a/cmd/sort.go b/cmd/sort.go
index 4df585629..f9a1d682e 100644
--- a/cmd/sort.go
+++ b/cmd/sort.go
@@ -14,7 +14,7 @@ var sorter = ctl.NewSortCommand(os.Stdin, os.Stdout, os.Stderr)
 
 var sortCmd = &cobra.Command{
 	Use:   "sort <path>",
-	Short: "sort - sort import data for optimal import performance",
+	Short: "Sort import data for optimal import performance.",
 	Long: `
 Sorts the import data at PATH into the optimal sort order for importing.
 

From 555a514e37526ec6624a4b1e632f927f0c96ecb7 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Tue, 7 Mar 2017 11:30:31 -0600
Subject: [PATCH 35/37] code review tweaks

---
 cmd/pilosa/main.go    |  2 +-
 cmd/server.go         |  2 +-
 ctl/backup.go         |  2 +-
 ctl/config.go         |  2 +-
 ctl/export.go         |  2 +-
 ctl/sort.go           |  2 +-
 server/server.go      | 21 +++++++++------------
 server/server_test.go | 20 ++++++++++----------
 8 files changed, 25 insertions(+), 28 deletions(-)

diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go
index 255aef606..c3bef0622 100644
--- a/cmd/pilosa/main.go
+++ b/cmd/pilosa/main.go
@@ -10,6 +10,6 @@ import (
 func main() {
 	if err := cmd.RootCmd.Execute(); err != nil {
 		fmt.Println(err)
-		os.Exit(-1)
+		os.Exit(1)
 	}
 }
diff --git a/cmd/server.go b/cmd/server.go
index 6603a0989..64e45eed4 100644
--- a/cmd/server.go
+++ b/cmd/server.go
@@ -14,7 +14,7 @@ import (
 	"github.com/pilosa/pilosa/server"
 )
 
-var serve = server.NewMain()
+var serve = server.NewCommand()
 
 var serveCmd = &cobra.Command{
 	Use:   "server",
diff --git a/ctl/backup.go b/ctl/backup.go
index 23b34cd1f..f0adf5766 100644
--- a/ctl/backup.go
+++ b/ctl/backup.go
@@ -36,7 +36,7 @@ func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand
 	}
 }
 
-// Run executes the main program execution.
+// Run executes the backup.
 func (cmd *BackupCommand) Run(ctx context.Context) error {
 	// Validate arguments.
 	if cmd.Path == "" {
diff --git a/ctl/config.go b/ctl/config.go
index 516069e2d..998c17b47 100644
--- a/ctl/config.go
+++ b/ctl/config.go
@@ -24,7 +24,7 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *ConfigCommand
 	}
 }
 
-// Run executes the main program execution.
+// Run prints out the default config.
 func (cmd *ConfigCommand) Run(ctx context.Context) error {
 	fmt.Fprintln(cmd.Stdout, strings.TrimSpace(`
 data-dir = "~/.pilosa"
diff --git a/ctl/export.go b/ctl/export.go
index f19c7123f..4eead3239 100644
--- a/ctl/export.go
+++ b/ctl/export.go
@@ -36,7 +36,7 @@ func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *ExportCommand
 	}
 }
 
-// Run executes the main program execution.
+// Run executes the export.
 func (cmd *ExportCommand) Run(ctx context.Context) error {
 	logger := log.New(cmd.Stderr, "", log.LstdFlags)
 
diff --git a/ctl/sort.go b/ctl/sort.go
index eac68b7ab..66bd16372 100644
--- a/ctl/sort.go
+++ b/ctl/sort.go
@@ -35,7 +35,7 @@ func NewSortCommand(stdin io.Reader, stdout, stderr io.Writer) *SortCommand {
 	}
 }
 
-// Run executes the main program execution.
+// Run executes the sort command.
 func (cmd *SortCommand) Run(ctx context.Context) error {
 	// Open file for reading.
 	f, err := os.Open(cmd.Path)
diff --git a/server/server.go b/server/server.go
index 247d0efa9..b7b8ca2f9 100644
--- a/server/server.go
+++ b/server/server.go
@@ -36,8 +36,8 @@ const (
 	DefaultDataDir = "~/.pilosa"
 )
 
-// Main represents the main program execution.
-type Main struct {
+// Command represents the state of the pilosa server command.
+type Command struct {
 	Server *pilosa.Server
 
 	// Configuration options.
@@ -55,8 +55,8 @@ type Main struct {
 }
 
 // NewMain returns a new instance of Main.
-func NewMain() *Main {
-	return &Main{
+func NewCommand() *Command {
+	return &Command{
 		Server: pilosa.NewServer(),
 		Config: pilosa.NewConfig(),
 
@@ -66,8 +66,8 @@ func NewMain() *Main {
 	}
 }
 
-// Run executes the main program execution.
-func (m *Main) Run(args ...string) error {
+// Run executes the pilosa server.
+func (m *Command) Run(args ...string) error {
 	// Notify user of config file.
 	if m.ConfigPath != "" {
 		fmt.Fprintf(m.Stdout, "Using config: %s\n", m.ConfigPath)
@@ -99,12 +99,12 @@ func (m *Main) Run(args ...string) error {
 }
 
 // Close shuts down the server.
-func (m *Main) Close() error {
+func (m *Command) Close() error {
 	return m.Server.Close()
 }
 
 // ParseFlags parses command line flags from args.
-func (m *Main) SetupConfig(args []string) error {
+func (m *Command) SetupConfig(args []string) error {
 	// Load config, if specified.
 	if m.ConfigPath != "" {
 		if _, err := toml.DecodeFile(m.ConfigPath, &m.Config); err != nil {
@@ -120,11 +120,8 @@ func (m *Main) SetupConfig(args []string) error {
 	// Expand home directory.
 	prefix := "~" + string(filepath.Separator)
 	if strings.HasPrefix(m.Config.DataDir, prefix) {
-		//	u, err := user.Current()
 		HomeDir := os.Getenv("HOME")
-		/*if err != nil {
-			return err
-		} else*/if HomeDir == "" {
+		if HomeDir == "" {
 			return errors.New("data directory not specified and no home dir available")
 		}
 		m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix))
diff --git a/server/server_test.go b/server/server_test.go
index bb772ab45..9e907522a 100644
--- a/server/server_test.go
+++ b/server/server_test.go
@@ -304,7 +304,7 @@ path = "/path/to/plugins"
 
 // Main represents a test wrapper for main.Main.
 type Main struct {
-	*server.Main
+	*server.Command
 
 	Stdin  bytes.Buffer
 	Stdout bytes.Buffer
@@ -318,16 +318,16 @@ func NewMain() *Main {
 		panic(err)
 	}
 
-	m := &Main{Main: server.NewMain()}
+	m := &Main{Command: server.NewCommand()}
 	m.Config.DataDir = path
 	m.Config.Host = "localhost:0"
-	m.Main.Stdin = &m.Stdin
-	m.Main.Stdout = &m.Stdout
-	m.Main.Stderr = &m.Stderr
+	m.Command.Stdin = &m.Stdin
+	m.Command.Stdout = &m.Stdout
+	m.Command.Stderr = &m.Stderr
 
 	if testing.Verbose() {
-		m.Main.Stdout = io.MultiWriter(os.Stdout, m.Main.Stdout)
-		m.Main.Stderr = io.MultiWriter(os.Stderr, m.Main.Stderr)
+		m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout)
+		m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr)
 	}
 
 	return m
@@ -345,18 +345,18 @@ func MustRunMain() *Main {
 // Close closes the program and removes the underlying data directory.
 func (m *Main) Close() error {
 	defer os.RemoveAll(m.Config.DataDir)
-	return m.Main.Close()
+	return m.Command.Close()
 }
 
 // Reopen closes the program and reopens it.
 func (m *Main) Reopen() error {
-	if err := m.Main.Close(); err != nil {
+	if err := m.Command.Close(); err != nil {
 		return err
 	}
 
 	// Create new main with the same config.
 	config := m.Config
-	m.Main = server.NewMain()
+	m.Command = server.NewCommand()
 	m.Config = config
 
 	// Run new program.

From 320110f0116c94a6194cb8a8fa276fbfde6afe9e Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Tue, 7 Mar 2017 11:35:58 -0600
Subject: [PATCH 36/37] tweak comments and remove dead code

---
 ctl/bench.go     | 53 +-----------------------------------------------
 ctl/check.go     | 33 +-----------------------------
 ctl/inspect.go   | 35 +-------------------------------
 ctl/restore.go   |  2 +-
 server/server.go |  2 +-
 5 files changed, 5 insertions(+), 120 deletions(-)

diff --git a/ctl/bench.go b/ctl/bench.go
index 7e789b5cd..2afe7bb27 100644
--- a/ctl/bench.go
+++ b/ctl/bench.go
@@ -3,12 +3,9 @@ package ctl
 import (
 	"context"
 	"errors"
-	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
-	"strings"
 	"time"
 
 	"github.com/pilosa/pilosa"
@@ -42,55 +39,7 @@ func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *BenchCommand {
 	}
 }
 
-// ParseFlags parses command line flags from args.
-func (cmd *BenchCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port")
-	fs.StringVar(&cmd.Database, "d", "", "database")
-	fs.StringVar(&cmd.Frame, "f", "", "frame")
-	fs.StringVar(&cmd.Op, "op", "", "operation")
-	fs.IntVar(&cmd.N, "n", 0, "op count")
-
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *BenchCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl bench [args]
-
-Executes a benchmark for a given operation against the database.
-
-The following flags are allowed:
-
-	-host HOSTPORT
-		hostname and port of running pilosa server
-
-	-d DATABASE
-		database to execute operation against
-
-	-f FRAME
-		frame to execute operation against
-
-	-op OP
-		name of operation to execute
-
-	-n COUNT
-		number of iterations to execute
-
-The following operations are available:
-
-	set-bit
-		Sets a single random bit on the frame
-
-`)
-}
-
-// Run executes the main program execution.
+// Run executes the bench command.
 func (cmd *BenchCommand) Run(ctx context.Context) error {
 	// Create a client to the server.
 	client, err := pilosa.NewClient(cmd.Host)
diff --git a/ctl/check.go b/ctl/check.go
index 616e189bf..0893790f2 100644
--- a/ctl/check.go
+++ b/ctl/check.go
@@ -2,14 +2,10 @@ package ctl
 
 import (
 	"context"
-	"errors"
-	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 	"syscall"
 
 	"github.com/pilosa/pilosa/roaring"
@@ -35,34 +31,7 @@ func NewCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *CheckCommand {
 	}
 }
 
-// ParseFlags parses command line flags from args.
-func (cmd *CheckCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	// Parse path.
-	if fs.NArg() == 0 {
-		return errors.New("path required")
-	}
-	cmd.Paths = fs.Args()
-
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *CheckCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl check PATHS...
-
-Performs a consistency check on data files.
-
-`)
-}
-
-// Run executes the main program execution.
+// Run executes the check command.
 func (cmd *CheckCommand) Run(ctx context.Context) error {
 	for _, path := range cmd.Paths {
 		switch filepath.Ext(path) {
diff --git a/ctl/inspect.go b/ctl/inspect.go
index 1b00dbcee..86434b131 100644
--- a/ctl/inspect.go
+++ b/ctl/inspect.go
@@ -2,13 +2,9 @@ package ctl
 
 import (
 	"context"
-	"errors"
-	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
-	"strings"
 	"syscall"
 	"text/tabwriter"
 	"time"
@@ -37,36 +33,7 @@ func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectComman
 	}
 }
 
-// ParseFlags parses command line flags from args.
-func (cmd *InspectCommand) ParseFlags(args []string) error {
-	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
-	fs.SetOutput(ioutil.Discard)
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	// Parse path.
-	if fs.NArg() == 0 {
-		return errors.New("path required")
-	} else if fs.NArg() > 1 {
-		return errors.New("only one path allowed")
-	}
-	cmd.Path = fs.Arg(0)
-
-	return nil
-}
-
-// Usage returns the usage message to be printed.
-func (cmd *InspectCommand) Usage() string {
-	return strings.TrimSpace(`
-usage: pilosactl inspect PATH
-
-Inspects a data file and provides stats.
-
-`)
-}
-
-// Run executes the main program execution.
+// Run executes the inspect command.
 func (cmd *InspectCommand) Run(ctx context.Context) error {
 	// Open file handle.
 	f, err := os.Open(cmd.Path)
diff --git a/ctl/restore.go b/ctl/restore.go
index b9a573704..a650df495 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -36,7 +36,7 @@ func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreComman
 	}
 }
 
-// Run executes the main program execution.
+// Run executes the restore command.
 func (cmd *RestoreCommand) Run(ctx context.Context) error {
 	// Validate arguments.
 	if cmd.Path == "" {
diff --git a/server/server.go b/server/server.go
index b7b8ca2f9..845340485 100644
--- a/server/server.go
+++ b/server/server.go
@@ -103,7 +103,7 @@ func (m *Command) Close() error {
 	return m.Server.Close()
 }
 
-// ParseFlags parses command line flags from args.
+// SetupConfig loads the config file if specified and sets state on the Command.
 func (m *Command) SetupConfig(args []string) error {
 	// Load config, if specified.
 	if m.ConfigPath != "" {

From fa9586f64f4c122cf0e2fc503a8f7dc5440d610e Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Tue, 7 Mar 2017 13:18:51 -0600
Subject: [PATCH 37/37] remove useless print

---
 cmd/server.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cmd/server.go b/cmd/server.go
index 64e45eed4..efff0ac7d 100644
--- a/cmd/server.go
+++ b/cmd/server.go
@@ -55,7 +55,6 @@ on the configured port.`,
 		// Execute the program.
 		if err := serve.Run(); err != nil {
 			fmt.Fprintln(serve.Stderr, err)
-			fmt.Fprintln(serve.Stderr, "stopping profile")
 			os.Exit(1)
 		}