Merge branch 'master' into 1204-pdk-docs

.travis.yml

@@ -10,6 +10,7 @@ env:
 install:
   - make install-dep install-statik vendor generate-statik
 script:
+  - GOARCH=386 make test
   - make test
 # TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009
 after_success:

CHANGELOG.md

@@ -41,6 +41,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950))
 - Refactored HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991))
 - Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907))
+- Deprecated RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205))
 
 ### Removed
 

Makefile

@@ -80,8 +80,12 @@ generate-protoc: require-protoc require-protoc-gen-gofast
 generate-statik: require-statik
 	go generate github.com/pilosa/pilosa/statik
 
+# `go generate` stringers
+generate-stringer:
+	go generate github.com/pilosa/pilosa
+
 # `go generate` all needed packages
-generate: generate-protoc generate-statik
+generate: generate-protoc generate-statik generate-stringer
 
 # Create Docker image from Dockerfile
 docker:

api.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:generate stringer -type=apiMethod
+
 package pilosa
 
 import (
@@ -59,8 +61,39 @@ func NewAPI() *API {
 	}
 }
 
+// validAPIMethods specifies the api methods that are valid for each
+// cluster state.
+var validAPIMethods = map[string]map[apiMethod]struct{}{
+	ClusterStateStarting: methodsCommon,
+	ClusterStateNormal:   appendMap(methodsCommon, methodsNormal),
+	ClusterStateResizing: appendMap(methodsCommon, methodsResizing),
+}
+
+func appendMap(a, b map[apiMethod]struct{}) map[apiMethod]struct{} {
+	r := make(map[apiMethod]struct{})
+	for k, v := range a {
+		r[k] = v
+	}
+	for k, v := range b {
+		r[k] = v
+	}
+	return r
+}
+
+func (api *API) validate(f apiMethod) error {
+	state := api.Cluster.State()
+	if _, ok := validAPIMethods[state][f]; ok {
+		return nil
+	}
+	return ApiMethodNotAllowedError{errors.Errorf("api method %s not allowed in state %s", f, state)}
+}
+
 // Query parses a PQL query out of the request and executes it.
 func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) {
+	if err := api.validate(apiQuery); err != nil {
+		return QueryResponse{}, errors.Wrap(err, "validate api method")
+	}
+
 	resp := QueryResponse{}
 
 	q, err := pql.NewParser(strings.NewReader(req.Query)).Parse()
@@ -125,6 +158,10 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet
 
 // CreateIndex makes a new Pilosa index.
 func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) {
+	if err := api.validate(apiCreateIndex); err != nil {
+		return nil, errors.Wrap(err, "validate api method")
+	}
+
 	// Create index.
 	index, err := api.Holder.CreateIndex(indexName, options)
 	if err != nil {
@@ -146,6 +183,10 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
 
 // Index retrieves the named index.
 func (api *API) Index(ctx context.Context, indexName string) (*Index, error) {
+	if err := api.validate(apiIndex); err != nil {
+		return nil, errors.Wrap(err, "validate api method")
+	}
+
 	index := api.Holder.Index(indexName)
 	if index == nil {
 		return nil, ErrIndexNotFound
@@ -156,6 +197,10 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) {
 // DeleteIndex removes the named index. If the index is not found it does
 // nothing and returns no error.
 func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
+	if err := api.validate(apiDeleteIndex); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	// Delete index from the holder.
 	err := api.Holder.DeleteIndex(indexName)
 	if err != nil {
@@ -176,6 +221,10 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
 
 // CreateFrame makes the named frame in the named index with the given options.
 func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) {
+	if err := api.validate(apiCreateFrame); err != nil {
+		return nil, errors.Wrap(err, "validate api method")
+	}
+
 	// Find index.
 	index := api.Holder.Index(indexName)
 	if index == nil {
@@ -207,6 +256,10 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str
 // found, an error is returned. If the frame is not found, it is ignored and no
 // action is taken.
 func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error {
+	if err := api.validate(apiDeleteFrame); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	// Find index.
 	index := api.Holder.Index(indexName)
 	if index == nil {
@@ -235,6 +288,10 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str
 // ExportCSV encodes the fragment designated by the index,frame,view,slice as
 // CSV of the form <row>,<col>
 func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error {
+	if err := api.validate(apiExportCSV); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	// Validate that this handler owns the slice.
 	if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) {
 		api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice)
@@ -267,14 +324,22 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin
 }
 
 // SliceNodes returns the node and all replicas which should contain a slice's data.
-func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) []*Node {
-	return api.Cluster.SliceNodes(indexName, slice)
+func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) {
+	if err := api.validate(apiSliceNodes); err != nil {
+		return nil, errors.Wrap(err, "validate api method")
+	}
+
+	return api.Cluster.SliceNodes(indexName, slice), nil
 }
 
 // MarshalFragment returns an object which can write the specified fragment's data
 // to an io.Writer. The serialized data can be read back into a fragment with
 // the UnmarshalFragment API call.
 func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) {
+	if err := api.validate(apiMarshalFragment); err != nil {
+		return nil, errors.Wrap(err, "validate api method")
+	}
+
 	// Retrieve fragment from holder.
 	f := api.Holder.Fragment(indexName, frameName, viewName, slice)
 	if f == nil {
@@ -287,6 +352,10 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName
 // Reader which was previously written by MarshalFragment to populate the
 // fragment's data.
 func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error {
+	if err := api.validate(apiUnmarshalFragment); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	// Retrieve frame.
 	f := api.Holder.Frame(indexName, frameName)
 	if f == nil {
@@ -316,6 +385,10 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa
 // return anything useful. Currently it returns protobuf encoded row and column
 // ids from a "block" which is a subdivision of a fragment.
 func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) {
+	if err := api.validate(apiFragmentBlockData); err != nil {
+		return nil, errors.Wrap(err, "validate api method")
+	}
+
 	reqBytes, err := ioutil.ReadAll(body)
 	if err != nil {
 		return nil, BadRequestError{errors.Wrap(err, "read body error")}
@@ -337,7 +410,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
 	// Encode response.
 	buf, err := proto.Marshal(&resp)
 	if err != nil {
-		return nil, errors.Wrap(err, "merge block response encoding error: %s")
+		return nil, errors.Wrap(err, "merge block response encoding error")
 	}
 
 	return buf, nil
@@ -345,6 +418,10 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
 
 // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment.
 func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) {
+	if err := api.validate(apiFragmentBlocks); err != nil {
+		return nil, errors.Wrap(err, "validate api method")
+	}
+
 	// Retrieve fragment from holder.
 	f := api.Holder.Fragment(indexName, frameName, viewName, slice)
 	if f == nil {
@@ -359,6 +436,10 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName
 // RestoreFrame reads all the data that this host should have for a given frame
 // from replicas in the cluster and restores that data to it.
 func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error {
+	if err := api.validate(apiRestoreFrame); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	// Create a client for the remote cluster.
 	client := NewInternalHTTPClientFromURI(host, api.RemoteClient)
 
@@ -433,6 +514,10 @@ func (api *API) Hosts(ctx context.Context) []*Node {
 
 // CreateInputDefinition is deprecated and will be removed. Do not use it.
 func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error {
+	if err := api.validate(apiCreateInputDefinition); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed.
 Please open an issue if you need to continue using it.`)
 	// Find index.
@@ -467,6 +552,10 @@ Please open an issue if you need to continue using it.`)
 
 // InputDefinition is deprecated and will be removed.
 func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) {
+	if err := api.validate(apiInputDefinition); err != nil {
+		return nil, errors.Wrap(err, "validate api method")
+	}
+
 	api.Logger.Printf(`InputDefinition is deprecated and will be removed.`)
 	// Find index.
 	index := api.Holder.Index(indexName)
@@ -483,6 +572,10 @@ func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefN
 
 // DeleteInputDefinition is deprecated and will be removed.
 func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error {
+	if err := api.validate(apiDeleteInputDefinition); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.")
 	// Find index.
 	index := api.Holder.Index(indexName)
@@ -508,6 +601,10 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp
 
 // WriteInput is deprecated and will be removed.
 func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error {
+	if err := api.validate(apiWriteInput); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	api.Logger.Printf("WriteInput is deprecated and will be removed.")
 	// Find index.
 	index := api.Holder.Index(indexName)
@@ -532,6 +629,10 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s
 
 // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests.
 func (api *API) RecalculateCaches(ctx context.Context) error {
+	if err := api.validate(apiRecalculateCaches); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	err := api.Broadcaster.SendSync(&internal.RecalculateCaches{})
 	if err != nil {
 		return errors.Wrap(err, "broacasting message")
@@ -542,7 +643,11 @@ func (api *API) RecalculateCaches(ctx context.Context) error {
 
 // PostClusterMessage is for internal use. It decodes a protobuf message out of
 // the body and forwards it to the BroadcastHandler.
-func (api *API) PostClusterMessage(ctx context.Context, reqBody io.Reader) error {
+func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error {
+	if err := api.validate(apiClusterMessage); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	// Read entire body.
 	body, err := ioutil.ReadAll(reqBody)
 	if err != nil {
@@ -575,6 +680,10 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo {
 
 // CreateField creates a new BSI field in the given index and frame.
 func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error {
+	if err := api.validate(apiCreateField); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	// Retrieve frame by name.
 	f := api.Holder.Frame(indexName, frameName)
 	if f == nil {
@@ -601,6 +710,10 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str
 
 // DeleteField deletes the given field.
 func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error {
+	if err := api.validate(apiDeleteField); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	// Retrieve frame by name.
 	f := api.Holder.Frame(indexName, frameName)
 	if f == nil {
@@ -627,6 +740,10 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str
 
 // Fields returns the fields in the given frame.
 func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) {
+	if err := api.validate(apiFields); err != nil {
+		return nil, errors.Wrap(err, "validate api method")
+	}
+
 	index := api.Holder.index(indexName)
 	if index == nil {
 		return nil, ErrIndexNotFound
@@ -642,6 +759,10 @@ func (api *API) Fields(ctx context.Context, indexName string, frameName string)
 
 // Views returns the views in the given frame.
 func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) {
+	if err := api.validate(apiViews); err != nil {
+		return nil, errors.Wrap(err, "validate api method")
+	}
+
 	// Retrieve views.
 	f := api.Holder.Frame(indexName, frameName)
 	if f == nil {
@@ -655,6 +776,10 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) (
 
 // DeleteView removes the given view.
 func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error {
+	if err := api.validate(apiDeleteView); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	// Retrieve frame.
 	f := api.Holder.Frame(indexName, frameName)
 	if f == nil {
@@ -685,6 +810,10 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri
 
 // IndexAttrDiff
 func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) {
+	if err := api.validate(apiIndexAttrDiff); err != nil {
+		return nil, errors.Wrap(err, "validate api method")
+	}
+
 	// Retrieve index from holder.
 	index := api.Holder.Index(indexName)
 	if index == nil {
@@ -715,6 +844,10 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At
 }
 
 func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) {
+	if err := api.validate(apiFrameAttrDiff); err != nil {
+		return nil, errors.Wrap(err, "validate api method")
+	}
+
 	// Retrieve index from holder.
 	f := api.Holder.Frame(indexName, frameName)
 	if f == nil {
@@ -746,6 +879,10 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s
 
 // Import bulk imports data into a particular index,frame,slice.
 func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
+	if err := api.validate(apiImport); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	_, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice)
 	if err != nil {
 		return err
@@ -771,6 +908,10 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
 
 // ImportValue bulk imports values into a particular field.
 func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error {
+	if err := api.validate(apiImportValue); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	_, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice)
 	if err != nil {
 		return err
@@ -784,31 +925,6 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest
 	return err
 }
 
-// ModifyIndexTimeQuantum changes the default time quantum on the given index.
-func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error {
-	// Retrieve index by name.
-	index := api.Holder.Index(indexName)
-	if index == nil {
-		return ErrIndexNotFound
-	}
-
-	// Set default time quantum on index.
-	return index.SetTimeQuantum(timeQuantum)
-}
-
-// ModifyFrameTimeQuantum changes the time quantum on the given frame. TODO:
-// what happens if there is already data in the frame?
-func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error {
-	// Retrieve index by name.
-	frame := api.Holder.Frame(indexName, frameName)
-	if frame == nil {
-		return ErrFrameNotFound
-	}
-
-	// Set default time quantum on index.
-	return frame.SetTimeQuantum(timeQuantum)
-}
-
 // MaxSlices returns the maximum slice number for each index in a map.
 func (api *API) MaxSlices(ctx context.Context) map[string]uint64 {
 	return api.Holder.MaxSlices()
@@ -933,6 +1049,10 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na
 
 // SetCoordinator makes a new Node the cluster coordinator.
 func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) {
+	if err := api.validate(apiSetCoordinator); err != nil {
+		return nil, nil, errors.Wrap(err, "validate api method")
+	}
+
 	oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator)
 	newNode = api.Cluster.nodeByID(id)
 	if newNode == nil {
@@ -959,6 +1079,10 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode
 // RemoveNode puts the cluster into the "RESIZING" state and begins the job of
 // removing the given node.
 func (api *API) RemoveNode(id string) (*Node, error) {
+	if err := api.validate(apiRemoveNode); err != nil {
+		return nil, errors.Wrap(err, "validate api method")
+	}
+
 	removeNode := api.Cluster.nodeByID(id)
 	if removeNode == nil {
 		return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove")
@@ -974,6 +1098,10 @@ func (api *API) RemoveNode(id string) (*Node, error) {
 
 // ResizeAbort stops the current resize job.
 func (api *API) ResizeAbort() error {
+	if err := api.validate(apiResizeAbort); err != nil {
+		return errors.Wrap(err, "validate api method")
+	}
+
 	if !api.Cluster.IsCoordinator() {
 		return ErrNodeNotCoordinator
 	}
@@ -992,3 +1120,89 @@ func (api *API) State() string {
 func (api *API) Version() string {
 	return strings.TrimPrefix(Version, "v")
 }
+
+type apiMethod int
+
+// API validation constants.
+const (
+	apiClusterMessage apiMethod = iota
+	apiCreateField
+	apiCreateFrame
+	apiCreateIndex
+	apiCreateInputDefinition
+	apiDeleteField
+	apiDeleteFrame
+	apiDeleteIndex
+	apiDeleteInputDefinition
+	apiDeleteView
+	apiExportCSV
+	apiFields
+	apiFragmentBlockData
+	apiFragmentBlocks
+	apiFrameAttrDiff
+	//apiHosts // not implemented
+	apiImport
+	apiImportValue
+	apiIndex
+	apiIndexAttrDiff
+	apiInputDefinition
+	//apiLocalID // not implemented
+	//apiLongQueryTime // not implemented
+	apiMarshalFragment
+	//apiMaxInverseSlices // not implemented
+	//apiMaxSlices // not implemented
+	apiQuery
+	apiRecalculateCaches
+	apiRemoveNode
+	apiResizeAbort
+	apiRestoreFrame
+	//apiSchema // not implemented
+	apiSetCoordinator
+	apiSliceNodes
+	//apiState // not implemented
+	//apiStatsWithTags // not implemented
+	apiUnmarshalFragment
+	//apiVersion // not implemented
+	apiViews
+	apiWriteInput
+)
+
+var methodsCommon = map[apiMethod]struct{}{
+	apiClusterMessage:  struct{}{},
+	apiMarshalFragment: struct{}{},
+	apiSetCoordinator:  struct{}{},
+}
+
+var methodsResizing = map[apiMethod]struct{}{
+	apiResizeAbort: struct{}{},
+}
+
+var methodsNormal = map[apiMethod]struct{}{
+	apiCreateField:           struct{}{},
+	apiCreateFrame:           struct{}{},
+	apiCreateIndex:           struct{}{},
+	apiCreateInputDefinition: struct{}{},
+	apiDeleteField:           struct{}{},
+	apiDeleteFrame:           struct{}{},
+	apiDeleteIndex:           struct{}{},
+	apiDeleteInputDefinition: struct{}{},
+	apiDeleteView:            struct{}{},
+	apiExportCSV:             struct{}{},
+	apiFields:                struct{}{},
+	apiFragmentBlockData:     struct{}{},
+	apiFragmentBlocks:        struct{}{},
+	apiFrameAttrDiff:         struct{}{},
+	apiImport:                struct{}{},
+	apiImportValue:           struct{}{},
+	apiIndex:                 struct{}{},
+	apiIndexAttrDiff:         struct{}{},
+	apiInputDefinition:       struct{}{},
+	apiQuery:                 struct{}{},
+	apiRecalculateCaches:     struct{}{},
+	apiRemoveNode:            struct{}{},
+	apiRestoreFrame:          struct{}{},
+	apiSliceNodes:            struct{}{},
+	apiUnmarshalFragment:     struct{}{},
+	apiViews:                 struct{}{},
+	apiWriteInput:            struct{}{},
+}

apimethod_string.go

@@ -0,0 +1,16 @@
+// Code generated by "stringer -type=apiMethod"; DO NOT EDIT.
+
+package pilosa
+
+import "fmt"
+
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput"
+
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 327, 347, 360, 374, 389, 406, 419, 439, 447, 460}
+
+func (i apiMethod) String() string {
+	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {
+		return fmt.Sprintf("apiMethod(%d)", i)
+	}
+	return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]]
+}

broadcast_test.go

@@ -15,12 +15,16 @@
 package pilosa_test
 
 import (
+	"bytes"
 	"reflect"
 	"testing"
 
+	"io/ioutil"
+
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/internal"
+	"github.com/pilosa/pilosa/server"
 )
 
 // Ensure a message can be marshaled and unmarshaled.
@@ -52,8 +56,12 @@ func testMessageMarshal(t *testing.T, m proto.Message) {
 
 // Ensure that BroadcastReceiver can register a BroadcastHandler.
 func TestBroadcast_BroadcastReceiver(t *testing.T) {
-
-	s := pilosa.NewServer()
+	com := server.NewCommand(bytes.NewBuffer([]byte{}), ioutil.Discard, ioutil.Discard)
+	err := com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO
+	if err != nil {
+		t.Fatalf("setting up server: %v", err)
+	}
+	s := com.Server
 
 	sbr := NewSimpleBroadcastReceiver()
 	sbh := NewSimpleBroadcastHandler()

client_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/internal"
 	"github.com/pilosa/pilosa/pql"
+	"github.com/pilosa/pilosa/server"
 	"github.com/pilosa/pilosa/test"
 )
 
@@ -47,7 +48,7 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) {
 var defaultClient *http.Client
 
 func init() {
-	defaultClient = pilosa.GetHTTPClient(nil)
+	defaultClient = server.GetHTTPClient(nil)
 
 }
 
@@ -310,7 +311,7 @@ func TestClient_ImportValue(t *testing.T) {
 
 	// Load bitmap into cache to ensure cache gets updated.
 	index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-	frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true, Fields: []*pilosa.Field{&fld}})
+	frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{Fields: []*pilosa.Field{&fld}})
 	if err != nil {
 		t.Fatal(err)
 	}

cluster.go

@@ -40,9 +40,6 @@ const (
 	// DefaultPartitionN is the default number of partitions in a cluster.
 	DefaultPartitionN = 256
 
-	// DefaultReplicaN is the default number of replicas per partition.
-	DefaultReplicaN = 1
-
 	// ClusterState represents the state returned in the /status endpoint.
 	ClusterStateStarting = "STARTING"
 	ClusterStateNormal   = "NORMAL"
@@ -264,7 +261,6 @@ type Cluster struct {
 	// Close management
 	wg      sync.WaitGroup
 	closing chan struct{}
-	prefect SecurityManager
 
 	Logger Logger
 
@@ -277,7 +273,7 @@ func NewCluster() *Cluster {
 	return &Cluster{
 		Hasher:        &jmphasher{},
 		PartitionN:    DefaultPartitionN,
-		ReplicaN:      DefaultReplicaN,
+		ReplicaN:      1,
 		EventReceiver: NopEventReceiver,
 
 		joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel
@@ -285,8 +281,7 @@ func NewCluster() *Cluster {
 		closing:             make(chan struct{}),
 		joining:             make(chan struct{}),
 
-		Logger:  NopLogger,
-		prefect: &NopSecurityManager{},
+		Logger: NopLogger,
 	}
 }
 
@@ -433,19 +428,11 @@ func (c *Cluster) setState(state string) {
 	var doCleanup bool
 
 	switch state {
-	case ClusterStateResizing:
-		c.prefect.SetRestricted()
 	case ClusterStateNormal:
-		c.prefect.SetNormal()
-		// Don't change routing for these states:
-		// - ClusterStateStarting
-
 		// If state is RESIZING -> NORMAL then run cleanup.
 		if c.state == ClusterStateResizing {
 			doCleanup = true
 		}
-	default:
-		panic(fmt.Sprintf("invalid cluster state: %s", state))
 	}
 
 	c.state = state

cluster_test.go

@@ -427,7 +427,6 @@ func TestCluster_ResizeStates(t *testing.T) {
 		// Add Field Data to node0.
 		if err := tc.CreateFrame("i", "fields", pilosa.FrameOptions{
 			InverseEnabled: false,
-			RangeEnabled:   true,
 			//CacheType:      pilosa.CacheTypeNone,
 			Fields: []*pilosa.Field{
 				{

cmd/import.go

@@ -60,10 +60,9 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
 	flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
 	flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
-	flags.Var(&Importer.IndexOptions.TimeQuantum, "index-time-quantum", "Time quantum for the index (DEPRECATED. This feature will be removed in a future version. Set time quantum of each frame instead.)")
 	flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame")
 	flags.BoolVar(&Importer.FrameOptions.InverseEnabled, "frame-inverse-enabled", false, "Enable inverse frame")
-	flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "Enabled range encoded frame")
+	flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "DEPRECATED - any frame can have fields. This option will be removed.")
 	flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked")
 	flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame")
 	ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify)

cmd/server.go

@@ -15,17 +15,11 @@
 package cmd
 
 import (
-	"fmt"
 	"io"
-	"os"
-	"os/signal"
-	"runtime/pprof"
-	"syscall"
-	"time"
 
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
-	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/ctl"
 	"github.com/pilosa/pilosa/server"
 )
@@ -45,52 +39,10 @@ It will load existing data from the configured
 directory and start listening for client connections
 on the configured port.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			// Set up the logger.
-			if err := Server.SetupLogger(); err != nil {
-				return fmt.Errorf("error setting up the logger: %v", err)
+			if err := Server.Start(); err != nil {
+				return errors.Wrap(err, "running server")
 			}
-			logger := Server.Server.Logger
-			logger.Printf("Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime)
-
-			// Start CPU profiling.
-			if Server.CPUProfile != "" {
-				f, err := os.Create(Server.CPUProfile)
-				if err != nil {
-					return fmt.Errorf("create cpu profile: %v", err)
-				}
-				defer f.Close()
-
-				fmt.Fprintln(Server.Stderr, "Starting cpu profile")
-				pprof.StartCPUProfile(f)
-				time.AfterFunc(Server.CPUTime, func() {
-					fmt.Fprintln(Server.Stderr, "Stopping cpu profile")
-					pprof.StopCPUProfile()
-					f.Close()
-				})
-			}
-
-			// Execute the program.
-			if err := Server.Run(); err != nil {
-				return fmt.Errorf("error running server: %v", err)
-			}
-
-			// First SIGKILL causes server to shut down gracefully.
-			c := make(chan os.Signal, 2)
-			signal.Notify(c, os.Interrupt, syscall.SIGTERM)
-			select {
-			case sig := <-c:
-				logger.Printf("Received %s; gracefully shutting down...\n", sig.String())
-
-				// Second signal causes a hard shutdown.
-				go func() { <-c; os.Exit(1) }()
-
-				if err := Server.Close(); err != nil {
-					return err
-				}
-			case <-Server.Done:
-				logger.Printf("Server closed externally")
-			}
-			return nil
+			return errors.Wrap(Server.Wait(), "waiting on Server")
 		},
 	}
 

cmd/server_test.go

@@ -21,9 +21,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/cmd"
 	_ "github.com/pilosa/pilosa/test"
+	"github.com/pilosa/pilosa/toml"
 )
 
 func TestServerHelp(t *testing.T) {
@@ -37,8 +37,6 @@ func TestServerHelp(t *testing.T) {
 func TestServerConfig(t *testing.T) {
 	actualDataDir, err := ioutil.TempDir("", "")
 	failErr(t, err, "making data dir")
-	profFile, err := ioutil.TempFile("", "")
-	failErr(t, err, "making temp file")
 	logFile, err := ioutil.TempFile("", "")
 	failErr(t, err, "making log file")
 	tests := []commandTest{
@@ -65,7 +63,7 @@ func TestServerConfig(t *testing.T) {
 				v.Check(cmd.Server.Config.Bind, "localhost:10111")
 				v.Check(cmd.Server.Config.Cluster.ReplicaN, 2)
 				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"})
-				v.Check(cmd.Server.Config.Cluster.LongQueryTime, pilosa.Duration(time.Second*90))
+				v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90))
 				v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
 				return v.Error()
 			},
@@ -86,14 +84,14 @@ func TestServerConfig(t *testing.T) {
 			validation: func() error {
 				v := validator{}
 				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"})
-				v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*9))
+				v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9))
 				return v.Error()
 			},
 		},
 		// TEST 2
 		{
 			args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true"},
-			env:  map[string]string{"PILOSA_PROFILE_CPU_TIME": "1m"},
+			env:  map[string]string{},
 			cfgFileContent: `
 	bind = "localhost:19444"
 	data-dir = "` + actualDataDir + `"
@@ -103,9 +101,6 @@ func TestServerConfig(t *testing.T) {
 		]
 	[anti-entropy]
 		interval = "11m0s"
-	[profile]
-		cpu = "` + profFile.Name() + `"
-		cpu-time = "35s"
 	[metric]
 		service = "statsd"
 		host = "127.0.0.1:8125"
@@ -113,9 +108,7 @@ func TestServerConfig(t *testing.T) {
 			validation: func() error {
 				v := validator{}
 				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"})
-				v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*11))
-				v.Check(cmd.Server.CPUProfile, profFile.Name())
-				v.Check(cmd.Server.CPUTime, time.Minute)
+				v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*11))
 				v.Check(cmd.Server.Config.LogPath, logFile.Name())
 				v.Check(cmd.Server.Config.Metric.Service, "statsd")
 				v.Check(cmd.Server.Config.Metric.Host, "127.0.0.1:8125")
@@ -147,6 +140,9 @@ func TestServerConfig(t *testing.T) {
 		case <-cmd.Server.Started:
 		case <-executed:
 		}
+		if execErr != nil {
+			t.Fatalf("executing server command: %v", execErr)
+		}
 		err := cmd.Server.Close()
 		failErr(t, err, "closing pilosa server command")
 		<-executed

config.go

@@ -1,226 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa
-
-import (
-	"time"
-)
-
-// Cluster types.
-const (
-	ClusterNone   = ""
-	ClusterStatic = "static"
-	ClusterGossip = "gossip"
-)
-
-// TLSConfig contains TLS configuration
-type TLSConfig struct {
-	// CertificatePath contains the path to the certificate (.crt or .pem file)
-	CertificatePath string `toml:"certificate-path"`
-	// CertificateKeyPath contains the path to the certificate key (.key file)
-	CertificateKeyPath string `toml:"certificate-key-path"`
-	// SkipVerify disables verification for self-signed certificates
-	SkipVerify bool `toml:"skip-verify"`
-}
-
-// Config represents the configuration for the command.
-type Config struct {
-	// DataDir is the directory where Pilosa stores both indexed data and
-	// running state such as cluster topology information.
-	DataDir string `toml:"data-dir"`
-	// Bind is the host:port on which Pilosa will listen.
-	Bind string `toml:"bind"`
-
-	// MaxWritesPerRequest limits the number of mutating commands that can be in
-	// a single request to the server. This includes SetBit, ClearBit,
-	// SetRowAttrs & SetColumnAttrs.
-	MaxWritesPerRequest int `toml:"max-writes-per-request"`
-
-	// LogPath configures where Pilosa will write logs.
-	LogPath string `toml:"log-path"`
-
-	// Verbose toggles verbose logging which can be useful for debugging.
-	Verbose bool `toml:"verbose"`
-
-	// TLS
-	TLS TLSConfig
-
-	Cluster struct {
-		// Disabled controls whether clustering functionality is enabled.
-		Disabled      bool     `toml:"disabled"`
-		Coordinator   bool     `toml:"coordinator"`
-		ReplicaN      int      `toml:"replicas"`
-		Hosts         []string `toml:"hosts"`
-		LongQueryTime Duration `toml:"long-query-time"`
-	} `toml:"cluster"`
-
-	// Gossip config is based around memberlist.Config.
-	Gossip struct {
-		// Port indicates the port to which pilosa should bind for internal state sharing.
-		Port  string   `toml:"port"`
-		Seeds []string `toml:"seeds"`
-		Key   string   `toml:"key"`
-		// StreamTimeout is the timeout for establishing a stream connection with
-		// a remote node for a full state sync, and for stream read and write
-		// operations. Maps to memberlist TCPTimeout.
-		StreamTimeout Duration `toml:"stream-timeout"`
-		// SuspicionMult is the multiplier for determining the time an
-		// inaccessible node is considered suspect before declaring it dead.
-		// The actual timeout is calculated using the formula:
-		//
-		//   SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
-		//
-		// This allows the timeout to scale properly with expected propagation
-		// delay with a larger cluster size. The higher the multiplier, the longer
-		// an inaccessible node is considered part of the cluster before declaring
-		// it dead, giving that suspect node more time to refute if it is indeed
-		// still alive.
-		SuspicionMult int `toml:"suspicion-mult"`
-		// PushPullInterval is the interval between complete state syncs.
-		// Complete state syncs are done with a single node over TCP and are
-		// quite expensive relative to standard gossiped messages. Setting this
-		// to zero will disable state push/pull syncs completely.
-		//
-		// Setting this interval lower (more frequent) will increase convergence
-		// speeds across larger clusters at the expense of increased bandwidth
-		// usage.
-		PushPullInterval Duration `toml:"push-pull-interval"`
-		// ProbeInterval and ProbeTimeout are used to configure probing behavior
-		// for memberlist.
-		//
-		// ProbeInterval is the interval between random node probes. Setting
-		// this lower (more frequent) will cause the memberlist cluster to detect
-		// failed nodes more quickly at the expense of increased bandwidth usage.
-		//
-		// ProbeTimeout is the timeout to wait for an ack from a probed node
-		// before assuming it is unhealthy. This should be set to 99-percentile
-		// of RTT (round-trip time) on your network.
-		ProbeInterval Duration `toml:"probe-interval"`
-		ProbeTimeout  Duration `toml:"probe-timeout"`
-
-		// Interval and Nodes are used to configure the gossip
-		// behavior of memberlist.
-		//
-		// Interval is the interval between sending messages that need
-		// to be gossiped that haven't been able to piggyback on probing messages.
-		// If this is set to zero, non-piggyback gossip is disabled. By lowering
-		// this value (more frequent) gossip messages are propagated across
-		// the cluster more quickly at the expense of increased bandwidth.
-		//
-		// Nodes is the number of random nodes to send gossip messages to
-		// per Interval. Increasing this number causes the gossip messages
-		// to propagate across the cluster more quickly at the expense of
-		// increased bandwidth.
-		//
-		// ToTheDeadTime is the interval after which a node has died that
-		// we will still try to gossip to it. This gives it a chance to refute.
-		Interval      Duration `toml:"interval"`
-		Nodes         int      `toml:"nodes"`
-		ToTheDeadTime Duration `toml:"to-the-dead-time"`
-	} `toml:"gossip"`
-
-	AntiEntropy struct {
-		Interval Duration `toml:"interval"`
-	} `toml:"anti-entropy"`
-
-	Metric struct {
-		// Service can be statsd, expvar, or none.
-		Service string `toml:"service"`
-		// Host tells the statsd client where to write.
-		Host         string   `toml:"host"`
-		PollInterval Duration `toml:"poll-interval"`
-		// Diagnostics toggles sending some limited diagnostic information to
-		// Pilosa's developers.
-		Diagnostics bool `toml:"diagnostics"`
-	} `toml:"metric"`
-}
-
-// NewConfig returns an instance of Config with default options.
-func NewConfig() *Config {
-	c := &Config{
-		DataDir:             "~/.pilosa",
-		Bind:                ":10101",
-		MaxWritesPerRequest: 5000,
-		// LogPath: "",
-		// Verbose: false,
-		TLS: TLSConfig{},
-	}
-
-	// Cluster config.
-	c.Cluster.Disabled = false
-	// c.Cluster.Coordinator = false
-	c.Cluster.ReplicaN = DefaultReplicaN
-	c.Cluster.Hosts = []string{}
-	c.Cluster.LongQueryTime = Duration(time.Minute)
-
-	// Gossip config.
-	c.Gossip.Port = "14000"
-	// c.Gossip.Seeds = []string{}
-	// c.Gossip.Key = ""
-	c.Gossip.StreamTimeout = Duration(10 * time.Second)
-	c.Gossip.SuspicionMult = 4
-	c.Gossip.PushPullInterval = Duration(30 * time.Second)
-	c.Gossip.ProbeInterval = Duration(1 * time.Second)
-	c.Gossip.ProbeTimeout = Duration(500 * time.Millisecond)
-	c.Gossip.Interval = Duration(200 * time.Millisecond)
-	c.Gossip.Nodes = 3
-	c.Gossip.ToTheDeadTime = Duration(30 * time.Second)
-
-	// AntiEntropy config.
-	c.AntiEntropy.Interval = Duration(10 * time.Minute)
-
-	// Metric config.
-	c.Metric.Service = "none"
-	// c.Metric.Host = ""
-	c.Metric.PollInterval = Duration(0 * time.Minute)
-	c.Metric.Diagnostics = true
-
-	return c
-}
-
-// Validate that all configuration permutations are compatible with each other.
-func (c *Config) Validate() error {
-	if !c.Cluster.Disabled && len(c.Cluster.Hosts) > 0 {
-		return ErrConfigClusterEnabledHosts
-	}
-	return nil
-}
-
-// Duration is a TOML wrapper type for time.Duration.
-type Duration time.Duration
-
-// String returns the string representation of the duration.
-func (d Duration) String() string { return time.Duration(d).String() }
-
-// UnmarshalText parses a TOML value into a duration value.
-func (d *Duration) UnmarshalText(text []byte) error {
-	v, err := time.ParseDuration(string(text))
-	if err != nil {
-		return err
-	}
-
-	*d = Duration(v)
-	return nil
-}
-
-// MarshalText writes duration value in text format.
-func (d Duration) MarshalText() (text []byte, err error) {
-	return []byte(d.String()), nil
-}
-
-// MarshalTOML write duration into valid TOML.
-func (d Duration) MarshalTOML() ([]byte, error) {
-	return []byte(d.String()), nil
-}

ctl/backup.go

@@ -21,6 +21,7 @@ import (
 	"os"
 
 	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/server"
 )
 
 // BackupCommand represents a command for backing up a view.
@@ -39,7 +40,7 @@ type BackupCommand struct {
 	// Standard input/output
 	*pilosa.CmdIO
 
-	TLS pilosa.TLSConfig
+	TLS server.TLSConfig
 }
 
 // NewBackupCommand returns a new instance of BackupCommand.
@@ -88,6 +89,6 @@ func (cmd *BackupCommand) TLSHost() string {
 	return cmd.Host
 }
 
-func (cmd *BackupCommand) TLSConfiguration() pilosa.TLSConfig {
+func (cmd *BackupCommand) TLSConfiguration() server.TLSConfig {
 	return cmd.TLS
 }

ctl/bench.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/internal"
+	"github.com/pilosa/pilosa/server"
 )
 
 // BenchCommand represents a command for benchmarking index operations.
@@ -42,7 +43,7 @@ type BenchCommand struct {
 	// Standard input/output
 	*pilosa.CmdIO
 
-	TLS pilosa.TLSConfig
+	TLS server.TLSConfig
 }
 
 // NewBenchCommand returns a new instance of BenchCommand.
@@ -110,6 +111,6 @@ func (cmd *BenchCommand) TLSHost() string {
 	return cmd.Host
 }
 
-func (cmd *BenchCommand) TLSConfiguration() pilosa.TLSConfig {
+func (cmd *BenchCommand) TLSConfiguration() server.TLSConfig {
 	return cmd.TLS
 }

ctl/common.go

@@ -18,13 +18,14 @@ import (
 	"crypto/tls"
 
 	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/server"
 	"github.com/spf13/pflag"
 )
 
 // CommandWithTLSSupport is the interface for commands which has TLS settings
 type CommandWithTLSSupport interface {
 	TLSHost() string
-	TLSConfiguration() pilosa.TLSConfig
+	TLSConfiguration() server.TLSConfig
 }
 
 // SetTLSConfig creates common TLS flags
@@ -48,7 +49,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error
 			InsecureSkipVerify: tlsConfig.SkipVerify,
 		}
 	}
-	client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), pilosa.GetHTTPClient(TLSConfig))
+	client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig))
 	if err != nil {
 		return nil, err
 	}

ctl/config.go

@@ -21,12 +21,13 @@ import (
 
 	toml "github.com/pelletier/go-toml"
 	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/server"
 )
 
 // ConfigCommand represents a command for printing a default config.
 type ConfigCommand struct {
 	*pilosa.CmdIO
-	Config *pilosa.Config
+	Config *server.Config
 }
 
 // NewConfigCommand returns a new instance of ConfigCommand.

ctl/config_test.go

@@ -22,7 +22,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/server"
 )
 
 func TestConfigCommand_Run(t *testing.T) {
@@ -30,7 +30,7 @@ func TestConfigCommand_Run(t *testing.T) {
 	stdin := bytes.NewReader(rder)
 	r, w, _ := os.Pipe()
 	cm := NewConfigCommand(stdin, w, os.Stderr)
-	cm.Config = pilosa.NewConfig()
+	cm.Config = server.NewConfig()
 
 	err := cm.Run(context.Background())
 	w.Close()

ctl/export.go

@@ -21,6 +21,7 @@ import (
 	"os"
 
 	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/server"
 )
 
 // ExportCommand represents a command for bulk exporting data from a server.
@@ -38,7 +39,7 @@ type ExportCommand struct {
 	// Standard input/output
 	*pilosa.CmdIO
 
-	TLS pilosa.TLSConfig
+	TLS server.TLSConfig
 }
 
 // NewExportCommand returns a new instance of ExportCommand.
@@ -114,6 +115,6 @@ func (cmd *ExportCommand) TLSHost() string {
 	return cmd.Host
 }
 
-func (cmd *ExportCommand) TLSConfiguration() pilosa.TLSConfig {
+func (cmd *ExportCommand) TLSConfiguration() server.TLSConfig {
 	return cmd.TLS
 }

ctl/import.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/server"
 )
 
 // ImportCommand represents a command for bulk importing data.
@@ -66,7 +67,7 @@ type ImportCommand struct {
 	// Standard input/output
 	*pilosa.CmdIO
 
-	TLS pilosa.TLSConfig
+	TLS server.TLSConfig
 }
 
 // NewImportCommand returns a new instance of ImportCommand.
@@ -447,6 +448,6 @@ func (cmd *ImportCommand) TLSHost() string {
 	return cmd.Host
 }
 
-func (cmd *ImportCommand) TLSConfiguration() pilosa.TLSConfig {
+func (cmd *ImportCommand) TLSConfiguration() server.TLSConfig {
 	return cmd.TLS
 }

ctl/import_test.go

@@ -86,9 +86,7 @@ func TestImportCommand_Run(t *testing.T) {
 }
 
 // Ensure that the ImportValue path runs (note: we have specified a value
-// for cm.Field. Because the handler doesn't return errors (it sends them
-// to the logger), we don't get an error returned at `cm.Run()` even though
-// we haven't setup frame `f` to be RangeEnabled.
+// for cm.Field.)
 func TestImportCommand_RunValue(t *testing.T) {
 
 	buf := bytes.Buffer{}
@@ -117,7 +115,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 	cm.Host = s.Host()
 
 	http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
-	http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"rangeEnabled": true, "fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`)))
+	http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`)))
 
 	cm.Index = "i"
 	cm.Frame = "f"

ctl/restore.go

@@ -21,6 +21,7 @@ import (
 	"os"
 
 	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/server"
 )
 
 // RestoreCommand represents a command for restoring a frame from a backup.
@@ -39,7 +40,7 @@ type RestoreCommand struct {
 	// Standard input/output
 	*pilosa.CmdIO
 
-	TLS pilosa.TLSConfig
+	TLS server.TLSConfig
 }
 
 // NewRestoreCommand returns a new instance of RestoreCommand.
@@ -81,6 +82,6 @@ func (cmd *RestoreCommand) TLSHost() string {
 	return cmd.Host
 }
 
-func (cmd *RestoreCommand) TLSConfiguration() pilosa.TLSConfig {
+func (cmd *RestoreCommand) TLSConfiguration() server.TLSConfig {
 	return cmd.TLS
 }

ctl/server.go

@@ -61,8 +61,4 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "Default URI to send metrics.")
 	flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
 	flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
-
-	// CPU Profiling
-	flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.")
-	flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.")
 }

diagnostics.go

@@ -168,23 +168,23 @@ func (d *DiagnosticsCollector) logErr(err error) bool {
 
 // EnrichWithOSInfo adds OS information to the diagnostics payload.
 func (d *DiagnosticsCollector) EnrichWithOSInfo() {
-	uptime, err := d.server.SystemInfo.Uptime()
+	uptime, err := d.server.systemInfo.Uptime()
 	if !d.logErr(err) {
 		d.Set("HostUptime", uptime)
 	}
-	platform, err := d.server.SystemInfo.Platform()
+	platform, err := d.server.systemInfo.Platform()
 	if !d.logErr(err) {
 		d.Set("OSPlatform", platform)
 	}
-	family, err := d.server.SystemInfo.Family()
+	family, err := d.server.systemInfo.Family()
 	if !d.logErr(err) {
 		d.Set("OSFamily", family)
 	}
-	version, err := d.server.SystemInfo.OSVersion()
+	version, err := d.server.systemInfo.OSVersion()
 	if !d.logErr(err) {
 		d.Set("OSVersion", version)
 	}
-	kernelVersion, err := d.server.SystemInfo.KernelVersion()
+	kernelVersion, err := d.server.systemInfo.KernelVersion()
 	if !d.logErr(err) {
 		d.Set("OSKernelVersion", kernelVersion)
 	}
@@ -192,15 +192,15 @@ func (d *DiagnosticsCollector) EnrichWithOSInfo() {
 
 // EnrichWithMemoryInfo adds memory information to the diagnostics payload.
 func (d *DiagnosticsCollector) EnrichWithMemoryInfo() {
-	memFree, err := d.server.SystemInfo.MemFree()
+	memFree, err := d.server.systemInfo.MemFree()
 	if !d.logErr(err) {
 		d.Set("MemFree", memFree)
 	}
-	memTotal, err := d.server.SystemInfo.MemTotal()
+	memTotal, err := d.server.systemInfo.MemTotal()
 	if !d.logErr(err) {
 		d.Set("MemTotal", memTotal)
 	}
-	memUsed, err := d.server.SystemInfo.MemUsed()
+	memUsed, err := d.server.systemInfo.MemUsed()
 	if !d.logErr(err) {
 		d.Set("MemUsed", memUsed)
 	}
@@ -219,10 +219,8 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() {
 		numIndexes += 1
 		for _, frame := range index.Frames() {
 			numFrames += 1
-			if frame.rangeEnabled {
-				if fields, err := frame.GetFields(); err == nil {
-					bsiFieldCount += len(fields)
-				}
+			if fields, err := frame.GetFields(); err == nil {
+				bsiFieldCount += len(fields)
 			}
 			if frame.TimeQuantum() != "" {
 				timeQuantumEnabled = true

docs/administration.md

@@ -48,9 +48,9 @@ On Mac OS X, `ulimit` does not behave predictably. [This blog post](https://blog
 
 #### Importing
 
-The import API expects a csv of RowID,ColumnID's.
+The import API expects a csv of rowID,columnID's.
 
-When importing large datasets remember it is much faster to pre sort the data by RowID and then by ColumnID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results.
+When importing large datasets remember it is much faster to pre sort the data by row ID and then by column ID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results.
 
 ```
 pilosa import --sort -i project -f stargazer project-stargazer.csv
@@ -70,7 +70,7 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count
 
 #### Exporting
 
-Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format RowID,ColumnID and sorted by column ID.
+Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format rowID,columnID and sorted by columnID.
 ```
 curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0&view=standard" \
      --header "Accept: text/csv"
@@ -201,7 +201,6 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi
 - **NumCPU:** Number of Cores per Node
 - **BSIEnabled:** Bit Slice Index Frames in use.
 - **TimeQuantumEnabled:** Time Quantum Frames in use.
-- **InverseEnabled:** Inverse Frames in use.
 - **NumIndexes:** Number of Indexes in the Cluster.
 - **NumFrames:** Number of Frames in the Cluster.
 - **NumSlices:** Number of Slices in the Cluster.

docs/api-reference.md

@@ -68,7 +68,7 @@ Sends a query to the Pilosa server with the given index. The request body is UTF
 ``` request
 curl localhost:10101/index/user/query \
      -X POST \
-     -d 'Bitmap(frame="language", rowID=5)'
+     -d 'Bitmap(frame="language", row=5)'
 ```
 ``` response
 {"results":[{"attrs":{},"bits":[100]}]}
@@ -83,7 +83,7 @@ The query is executed for all [slices](../data-model/#slice) by default. To use
 ``` request
 curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \
      -X POST \
-     -d 'Bitmap(frame="language", rowID=5)'
+     -d 'Bitmap(frame="language", row=5)'
 ```
 ``` response
 {
@@ -103,10 +103,9 @@ Creates a frame in the given index with the given name.
 The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields:
 
 * `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame.
-* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`.
 * `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
 * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
-* `rangeEnabled` (boolean): Enables range-encoded fields in this frame.
+* `rangeEnabled` (boolean): DEPRECATED - has no effect, will be removed. All frames support BSI fields.
 * `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding).
 
 Each individual `field` contains the following:
@@ -119,9 +118,7 @@ Each individual `field` contains the following:
 Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
 
 ``` request
-curl localhost:10101/index/user/frame/language \
-     -X POST \
-     -d '{"options": {"inverseEnabled": true}}'
+curl localhost:10101/index/user/frame/language -X POST
 ```
 ``` response
 {}
@@ -130,7 +127,7 @@ curl localhost:10101/index/user/frame/language \
 ``` request
 curl localhost:10101/index/repository/frame/stats \
     -X POST \
-    -d '{"rangeEnabled": true, "fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
+    -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
 ```
 ``` response
 {}
@@ -149,35 +146,6 @@ curl -XDELETE localhost:10101/index/user/frame/language
 {}
 ```
 
-### Change frame time quantum
-
-`PATCH /index/<index-name>/frame/<frame-name>/time-quantum`
-
-Changes the time quantum for the given frame. This endpoint should be called at most once right after creating a frame.
-
-The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Valid time quantum values are:
-
-* (Empty string)
-* Y: year
-* M: month
-* D: day
-* H: hour
-* YM: year and month
-* MD: month and day
-* DH: day and hour
-* YMD: year, month and day
-* MDH: month, day and hour
-* YMDH: year, month, day and hour
-
-``` request
-curl localhost:10101/index/user/frame/language/time-quantum \
-     -X POST \
-     -d '{"timeQuantum": "YM"}'
-```
-``` response
-{}
-```
-
 ### Create Field
 
 `POST /index/<index-name>/frame/<frame-name>/field/<field-name>`
@@ -198,129 +166,6 @@ curl localhost:10101/index/repository/frame/stats/field/pullrequests \
 {}
 ```
 
-### Create input definition
-
-<div class="warning">
-Input definition is deprecated as of v0.9.
-</div>
-
-`POST /index/<index-name>/input-definition/<input-definition-name>`
- 
-Creates an input definition in the given index with the given name.
- 
-The request payload is JSON, and it must contain the fields `frames` and `fields`. `frames` is an array of frames used within this input definition.  Each frame must contain a `name` and may contain the following options:
- 
-* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame.
-* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`.
-* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
-* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
- 
-The `fields` array contains a series of JSON objects describing how to process each field received in the input data.  Each `field` object must contain a `name` which maps to the source JSON field name.  One field must be defined at the `primaryKey`.  The `primarykey` source field's value must be an unsigned integer which maps directly to a columnID in Pilosa.
-
-* `name`    (string): Maps the source data field to actions that process the field's corresponding value.
-* `actions` (array): List of actions that will process the field's value.
- 
-The `action` describes how the field value will be processed.  Each `action` may contain:
-
-* `frame` (string): The Frame that will contain this action's set bits.
-* `rowid` (int): The action can use this as a pre-defined SetBit rowID.  The user is required to ensure this ID does not overlap with other rows in use per frame.
-* `valueDestination` (string): The mapping rule used for this data.
-    - `value-to-row`: The value should be an integer and will map directly to a RowID.
-    - `single-row-boolean`: If the value is true set a bit using the `rowid`.
-    - `mapping`: Map the value to a RowID in the `valueMap`.
-* `valueMap` (object): string and integer pairs used to map field values to RowID's.
- 
-``` request
-curl localhost:10101/index/user/input-definition/stargazer-input \
-    -X POST \
-    -d '{
-            "frames":[
-                {
-                    "name": "language",
-                    "options": {"inverseEnabled": true}
-                }
-            ],
-            "fields":[
-                {
-                    "name": "repo_id",
-                    "primaryKey":true
-                },
-                {
-                    "name": "language_id",
-                    "actions":[
-                        {
-                            "frame": "language",
-                            "valueDestination": "mapping",
-                            "valueMap": {
-                                "Go": 5,
-                                "Python": 17,
-                                "C++": 10
-                            }
-                        }
-                    ]
-                }
-            ]
-        }'
-```
-``` response
-{}
-```
-
-### Get input definition
-
-<div class="warning">
-Input definition is deprecated as of v0.9.
-</div>
-
-`GET /index/<index-name>/input-definition/<input-definition-name>`
-
-Returns the given input definition as JSON.
-
-``` request
-curl -XGET localhost:10101/index/user/input-definition/stargazer-input
-```
-``` response
-{"frames":[{"name":"language","options":{"inverseEnabled":true}}],"fields":[{"name":"repo_id","primaryKey":true},{"name":"language_id","actions":[{"frame":"language","valueDestination":"mapping","valueMap":{"Go":5,"Python":17,"C++":10}}]}]}
-```
-
-### Remove input definition
-
-<div class="warning">
-Input definition is deprecated as of v0.9.
-</div>
-
-`DELETE /index/<index-name>/input-definition/<input-definition-name>`
-
-Removes the given input definition.
-
-``` request
-curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input
-```
-``` response
-{}
-```
-
-### Process input data
-
-<div class="warning">
-Input definition is deprecated as of v0.9.
-</div>
-
-`POST /index/<index-name>/input/<input-definition-name>`
-
-Processes the JSON payload using the given input definition.
-
-The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column, and additional fields that will be handled by corresponding actions in the input definition.
-
-``` request
-curl localhost:10101/index/user/input/stargazer-input \
-     -X POST \
-     -d '[{"language_id": "Go", "repo_id": 92274475}]'
-```
-``` response
-{}
-```
-
 ### List hosts
 
 `GET /hosts`

docs/data-model.md

@@ -77,33 +77,19 @@ Columns are sharded on a preset width, and each shard is referred to as a Slice.
 
 ### View
 
-Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. The Inverse View contains the same data with the axes inverted.Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation.
+Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation.
 
 #### Standard
 
 The standard View contains the same Row/Column format as the input data. 
 
-#### Inverse
-
-The Inverse View contains the same data with the Row and Column swapped.
-
-For example, the following `SetBit()` queries will result in the data described in the illustration below:
-```
-SetBit(frame="A", rowID=8, columnID=3)
-SetBit(frame="A", rowID=11, columnID=3)
-SetBit(frame="A", rowID=19, columnID=5)
-```
-
-![inverse frame diagram](/img/docs/frame-inverse.svg)
-*Inverse frame diagram*
-
 #### Time Quantums
 
 If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the illustration below:
 
 ```
-SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-18T00:00")
-SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00")
+SetBit(frame="A", row=8, col=3, timestamp="2017-05-18T00:00")
+SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00")
 ```
 
 ![time quantum frame diagram](/img/docs/frame-time-quantum.svg)
@@ -114,17 +100,17 @@ SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00")
 Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded
 bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Sum` and `Range` queries on these BSI integers.
 
-Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rowIDs' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
+Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The rows of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
 
 For example, the following `SetFieldValue()` queries will result in the data described in the illustration below:
 
 ```
-SetFieldValue(columnID=1, frame="A", field0=1)
-SetFieldValue(columnID=2, frame="A", field0=2)
-SetFieldValue(columnID=3, frame="A", field0=3)
-SetFieldValue(columnID=4, frame="A", field0=7)
-SetFieldValue(columnID=2, frame="A", field1=1)
-SetFieldValue(columnID=3, frame="A", field1=6)
+SetFieldValue(col=1, frame="A", field0=1)
+SetFieldValue(col=2, frame="A", field0=2)
+SetFieldValue(col=3, frame="A", field0=3)
+SetFieldValue(col=4, frame="A", field0=7)
+SetFieldValue(col=2, frame="A", field1=1)
+SetFieldValue(col=3, frame="A", field1=6)
 ```
 
 ![BSI frame diagram](/img/docs/frame-bsi.svg)

docs/examples.md

@@ -201,11 +201,15 @@ For more examples and details, see this [ipython notebook](https://github.com/pi
 
 ### Chemical similarity search
 
+<div class="warning">
+This example uses the inverse frames feature, which is deprecated as of v0.9.0. This will soon be updated to reflect the current Pilosa API.
+</div>
+
 #### Overview
 
 The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals.
 
-We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set.
+We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. Others have tried to solve this chemical similarity search problem using databases (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set.
 
 Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient.
 ```
@@ -218,7 +222,7 @@ All source code to calculate tanimoto for molecule fingerprint using Pilosa is a
 
 #### Data model
 
-We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. 
+We use the [latest ChEMBL release](ftp://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/) chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. 
 
 Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions.
 

docs/getting-started.md

@@ -4,7 +4,6 @@ weight = 3
 nav = [
      "Starting Pilosa",
      "Sample Project",
-     "Input Definition",
      "What's Next?",
 ]
 +++
@@ -111,14 +110,6 @@ docker exec -it pilosa /pilosa import -i repository -f language /language.csv
 
 Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
 
-### Input Definition
-
-<div class="warning">
-Input definition is deprecated as of v0.9.
-</div>
-
-Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data.
-
 #### Make Some Queries
 
 <div class="note">

docs/glossary.md

@@ -24,7 +24,7 @@ nav = []
 
 <strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index).
 
-<strong id="frame">[Frame](../data-model/#frame):</strong> Frames are used to group [rows](#row) into different categories. `RowID`s are namespaced by frame such that the same `RowID` in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame.
+<strong id="frame">[Frame](../data-model/#frame):</strong> Frames are used to group [rows](#row) into different categories. Row IDs are namespaced by frame such that the same row ID in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame.
 
 <strong id="index">[Index](../data-model/#index):</strong> An Index is a top level container in Pilosa, analogous to a database in an RDBMS. Queries cannot operate across multiple indexes.
 
@@ -62,6 +62,6 @@ nav = []
 
 <strong id="toml">[TOML](https://github.com/toml-lang/toml):</strong> the language used for Pilosa's [configuration file](../configuration/).
 
-<strong id="topn">[TopN](../query-language/#topn):</strong> A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame).
+<strong id="topn">[TopN](../query-language/#topn):</strong> A [PQL](#pql) query that returns a list of row IDs, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame).
 
-<strong id="view">[View](../data-model/#view):</strong> Views separate the different data layouts within a [Frame](#frame). The two primary views are standard and inverse which represent the typical [row](#row)/[column](#column) data and its inverse respectively (an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), or a matrix transpose). Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.
+<strong id="view">[View](../data-model/#view):</strong> Views separate the different data layouts within a [Frame](#frame). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.

docs/input-definition.md

@@ -1,11 +0,0 @@
-+++
-title = "Input Definition"
-weight = 8
-+++
-
-## Input Definition
-<div class="warning">
-Input definition is deprecated as of Pilosa v0.9.<br/>
-<br/>
-The previous version of this page is still available <a href="https://www.pilosa.com/docs/v0.8/input-definition">here</a>.
-</div>

docs/pdk.md

@@ -70,3 +70,4 @@ control over the way data is indexed, and ingestion performance.
 ### Library
 
 For now, the [Godocs](https://godoc.org/github.com/pilosa/pdk) have the most up to date library documentation.
+

docs/query-language.md

@@ -25,17 +25,17 @@ There will be one item in the `results` array for each PQL query in the request.
 
 * Angle Brackets `<>` denote required arguments
 * Square Brackets `[]` denote optional arguments
-* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ROW_LABEL`, `STRING`)
+* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ATTR_NAME`, `STRING`)
 
 ##### Examples
 
 Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index, frames, and populate them with some data.
 
-The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", columnID=10, rowID=1)` against a server using curl, you would:
+The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", col=10, row=1)` against a server using curl, you would:
 ``` request
 curl localhost:10101/index/repository/query \
      -X POST \
-     -d 'SetBit(frame="stargazer", columnID=10, rowID=1)'
+     -d 'SetBit(frame="stargazer", col=10, row=1)'
 ```
 ``` response
 {"results":[true]}
@@ -58,7 +58,7 @@ curl localhost:10101/index/repository/query \
 **Spec:**
 
 ```
-SetBit(<frame=STRING>, <ROW_LABEL=UINT>, <COL_LABEL=UINT>, 
+SetBit(<frame=STRING>, <row=UINT>, <col=UINT>, 
        [timestamp=TIMESTAMP])
 ```
 
@@ -76,26 +76,26 @@ A return value of `false` indicates that the bit was already set to 1 and nothin
 **Examples:**
 
 ```
-SetBit(frame="stargazer", columnID=10, rowID=1)
+SetBit(frame="stargazer", col=10, row=1)
 ```
 
 This query illustrates setting a bit in the stargazer frame. User with id=1 has starred repository with id=10.
 
 SetBit also supports providing a timestamp. To write the date that a user starred a repository.
 ```
-SetBit(frame="stargazer", columnID=10, rowID=1, timestamp="2016-01-01T00:00")
+SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00")
 ```
 
 Setting multiple bits in a single request:
 ```
-SetBit(frame="stargazer", columnID=10, rowID=1) SetBit(frame="stargazer", columnID=10, rowID=2) SetBit(frame="stargazer", columnID=20, rowID=1) SetBit(frame="stargazer", columnID=30, rowID=2)
+SetBit(frame="stargazer", col=10, row=1) SetBit(frame="stargazer", col=10, row=2) SetBit(frame="stargazer", col=20, row=1) SetBit(frame="stargazer", col=30, row=2)
 ```
 
 #### SetRowAttrs
 **Spec:**
 
 ```
-SetRowAttrs(<frame=STRING>, <ROW_LABEL=UINT>, 
+SetRowAttrs(<frame=STRING>, <row=UINT>, 
             <ATTR_NAME=ATTR_VALUE>, 
             [ATTR_NAME=ATTR_VALUE ...])
 ```
@@ -111,13 +111,13 @@ SetRowAttrs queries always return `null` upon success.
 **Examples:**
 
 ```
-SetRowAttrs(frame="stargazer", rowID=10, username="mrpi", active=true)
+SetRowAttrs(frame="stargazer", row=10, username="mrpi", active=true)
 ```
 
 Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", stargazer_id=10)`.
 
 ```
-SetRowAttrs(frame="stargazer", rowID=10, username=null)
+SetRowAttrs(frame="stargazer", row=10, username=null)
 ```
 
 Delete username value for user 10.
@@ -127,7 +127,7 @@ Delete username value for user 10.
 **Spec:**
 
 ```
-SetColumnAttrs(<frame=STRING>, <ROW_LABEL=UINT>, 
+SetColumnAttrs(<frame=STRING>, <row=UINT>, 
                <ATTR_NAME=ATTR_VALUE>, 
                [ATTR_NAME=ATTR_VALUE ...])
 ```
@@ -143,13 +143,13 @@ SetColumnAttrs queries always return `null` upon success. Setting a value of `nu
 **Examples:**
 
 ```
-SetColumnAttrs(columnID=10, stars=123, url="http://projects.pilosa.com/10", active=true)
+SetColumnAttrs(col=10, stars=123, url="http://projects.pilosa.com/10", active=true)
 ```
 
-Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", columnID=10)`.
+Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", col=10)`.
 
 ```
-SetColumnAttrs(columnID=10, url=null)
+SetColumnAttrs(col=10, url=null)
 ```
 
 Delete url value for repo 10.
@@ -160,7 +160,7 @@ Delete url value for repo 10.
 **Spec:**
 
 ```
-SetBit(<frame=STRING>, <ROW_LABEL=UINT>, <COL_LABEL=UINT>, 
+SetBit(<frame=STRING>, <row=UINT>, <col=UINT>, 
        [timestamp=TIMESTAMP])
 ```
 
@@ -177,7 +177,7 @@ A return value of `false` indicates that the bit was already set to 0 and nothin
 **Examples:**
 
 ```
-ClearBit(frame="stargazer", columnID=10, rowID=1)
+ClearBit(frame="stargazer", col=10, row=1)
 ```
 
 Remove relationship between the stargazer in row 1 and the repository in column 10 from the stargazer frame.
@@ -188,12 +188,12 @@ Remove relationship between the stargazer in row 1 and the repository in column
 **Spec:**
 
 ```
-SetFieldValue(<COL_LABEL=UINT>, <frame=STRING>, <FIELD_NAME=INT>)
+SetFieldValue(<col=UINT>, <frame=STRING>, <FIELD_NAME=INT>)
 ```
 
 **Description:**
 
-`SetFieldValue` assigns an integer value with the specified field name to the `columnID` in the given `frame`.
+`SetFieldValue` assigns an integer value with the specified field name to the `col` in the given `frame`.
 
 **Result Type:** null
 
@@ -203,7 +203,7 @@ SetFieldValue returns `null` upon success.
 
 Set the number of pull requests of repository 10.
 ```
-SetFieldValue(columnID=10, frame="stats", pullrequests=2)
+SetFieldValue(col=10, frame="stats", pullrequests=2)
 ```
 
 
@@ -214,7 +214,7 @@ SetFieldValue(columnID=10, frame="stats", pullrequests=2)
 **Spec:**
 
 ```
-Bitmap(<frame=STRING>, (<ROW_LABEL=UINT> | <COL_LABEL>=UINT))
+Bitmap(<frame=STRING>, (<rowL=UINT> | <col>=UINT))
 ```
 
 **Description:**
@@ -229,7 +229,7 @@ e.g. `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}`
 
 Query all repositories that user 1 has starred.
 ```
-Bitmap(frame="stargazer", rowID=1)
+Bitmap(frame="stargazer", row=1)
 ```
 
 Returns `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}`
@@ -286,7 +286,7 @@ attrs will always be empty
 Query repositories which have been starred by two users.
 
 ```
-Intersect(Bitmap(frame="stargazer", rowID=1), Bitmap(frame="stargazer", rowID=2))
+Intersect(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2))
 ```
 
 Returns `{"attrs":{},"bits":[10]}`.
@@ -313,7 +313,7 @@ attrs will always be empty
 
 Query repositories which have been starred by one user and not another.
 ```
-Difference(Bitmap(frame="stargazer", rowID=1), Bitmap( frame="stargazer", rowID=2))
+Difference(Bitmap(frame="stargazer", row=1), Bitmap( frame="stargazer", row=2))
 ```
 
 Return `{"results":[{"attrs":{},"bits":[20]}]}`
@@ -321,7 +321,7 @@ Return `{"results":[{"attrs":{},"bits":[20]}]}`
 * bits are repositories that were starred by user 1 BUT NOT user 2
 
 ```
-Difference(Bitmap(frame="stargazer", rowID=2), Bitmap( frame="stargazer", rowID=1))
+Difference(Bitmap(frame="stargazer", row=2), Bitmap( frame="stargazer", row=1))
 ```
 
 Return `{"attrs":{},"bits":[30]}`
@@ -349,7 +349,7 @@ attrs will always be empty
 Query repositories which have been starred by two users.
 
 ```
-Xor(Bitmap(frame="stargazer", rowID=1), Bitmap(frame="stargazer", rowID=2))
+Xor(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2))
 ```
 
 Returns `{"attrs":{},"bits":[30]}`.
@@ -373,7 +373,7 @@ Returns the number of set bits in the `BITMAP_CALL` passed in.
 
 Query the number of repositories to which a user has contributed.
 ```
-Count(Bitmap(frame="stargazer", rowID=1))
+Count(Bitmap(frame="stargazer", row=1))
 ```
 
 Return `2`
@@ -386,13 +386,12 @@ Return `2`
 
 ```
 TopN([BITMAP_CALL], <frame=STRING>, [n=UINT],
-     [inverse=true], [<field=ATTR_NAME>, <filters=[]ATTR_VALUE>])
+     [<field=ATTR_NAME>, <filters=[]ATTR_VALUE>])
 ```
 
 **Description:**
 
 Return the id and count of the top `n` bitmaps (by count of bits) in the frame.
-`inverse=true` specifies that the call should operate on the [inverse view ](../data-model/#inverse).
 The `field` and `filters` arguments work together to only return Bitmaps which
 have the attribute specified by `field` with one of the values specified in
 `filters`.
@@ -419,16 +418,6 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1}
 * count is amount of repositories
 * Results are the number of repositories that each user starred in descending order for all users in the stargazer frame, for example user 1 starred two repositories, user 2 starred two repositories, user 3 starred one repository.
 
-```
-TopN(frame="stargazer", inverse=true)
-```
-
-Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1}]`
-
-* key is a repository ID
-* count is amount of users
-* Results are the number of users that starred each repository in descending order for all respositories in the stargazer frame.
-
 ```
 TopN(frame="stargazer", n=2)
 ```
@@ -438,7 +427,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}]`
 * Results are the top two users sorted by number of repositories they've starred in descending order.
 
 ```
-TopN(Bitmap(frame="language", rowID=1), frame="stargazer", n=2)
+TopN(Bitmap(frame="language", row=1), frame="stargazer", n=2)
 ```
 
 Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]`
@@ -450,7 +439,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]`
 **Spec:**
 
 ```
-Range(<frame=STRING>, <ROW_LABEL=UINT>, 
+Range(<frame=STRING>, <row=UINT>, 
       <start=TIMESTAMP>, <end=TIMESTAMP>)
 ```
 
@@ -466,7 +455,7 @@ between the given `start` and `end` timestamps.
 
 When you set timestamp using SetBit, you will able to query all repositories that a user has starred within a date range.
 ```
-Range(frame="stargazer", rowID=1, start="2010-01-01T00:00", end="2017-03-02T03:00")
+Range(frame="stargazer", row=1, start="2010-01-01T00:00", end="2017-03-02T03:00")
 ```
 
 Returns `{{"attrs":{},"bits":[10]}`

docs/tutorials.md

@@ -259,7 +259,6 @@ In addition to storing rows of bits, a frame can also contain fields that store
 curl localhost:10101/index/patients/frame/measurements \
      -X POST \
      -d '{"options":{
-              "rangeEnabled": true,
               "fields": [
                   {"name": "age", "type": "int", "min": 0, "max": 120},
                   {"name": "weight", "type": "int", "min": 0, "max": 500},

docs/webui.md

@@ -32,9 +32,9 @@ In addition to standard PQL, the console supports a few special commands, prefix
 - `:create frame <framename>`
 - `:delete frame <framename>`
 
-Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame).
+Frame creation also supports options like `timeQuantum`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame).
 
-- `:create frame <framename> inverseEnabled=true cacheSize=10000`
+- `:create frame <framename> cacheSize=10000`
 
 
 ### Cluster Admin

executor_test.go

@@ -280,7 +280,6 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) {
 		// Create frames.
 		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
 		if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{
-			RangeEnabled: true,
 			Fields: []*pilosa.Field{
 				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 50},
 				{Name: "field1", Type: pilosa.FieldTypeInt, Min: 1, Max: 2},
@@ -330,7 +329,6 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) {
 		defer hldr.Close()
 		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
 		if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{
-			RangeEnabled: true,
 			Fields: []*pilosa.Field{
 				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 100},
 			},
@@ -611,7 +609,6 @@ func TestExecutor_Execute_Sum(t *testing.T) {
 	}
 
 	if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
-		RangeEnabled: true,
 		Fields: []*pilosa.Field{
 			{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
 			{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
@@ -621,7 +618,6 @@ func TestExecutor_Execute_Sum(t *testing.T) {
 	}
 
 	if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
-		RangeEnabled: true,
 		Fields: []*pilosa.Field{
 			{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
 		},
@@ -723,7 +719,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
 	}
 
 	if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
-		RangeEnabled: true,
 		Fields: []*pilosa.Field{
 			{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
 			{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
@@ -733,7 +728,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
 	}
 
 	if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
-		RangeEnabled: true,
 		Fields: []*pilosa.Field{
 			{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
 		},
@@ -742,7 +736,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
 	}
 
 	if _, err := idx.CreateFrame("edge", pilosa.FrameOptions{
-		RangeEnabled: true,
 		Fields: []*pilosa.Field{
 			{Name: "foo", Type: pilosa.FieldTypeInt, Min: -100, Max: 100},
 		},

frame.go

@@ -33,7 +33,6 @@ import (
 const (
 	DefaultCacheType      = CacheTypeRanked
 	DefaultInverseEnabled = false
-	DefaultRangeEnabled   = false
 
 	// Default ranked frame cache
 	DefaultCacheSize = 50000
@@ -59,7 +58,6 @@ type Frame struct {
 	cacheType      string
 	cacheSize      uint32
 	timeQuantum    TimeQuantum
-	rangeEnabled   bool
 	fields         []*Field
 
 	Logger Logger
@@ -88,7 +86,6 @@ func NewFrame(path, index, name string) (*Frame, error) {
 		cacheType:      DefaultCacheType,
 		cacheSize:      DefaultCacheSize,
 		//timeQuantum
-		rangeEnabled: DefaultRangeEnabled,
 		//fields
 
 		Logger: NopLogger,
@@ -145,11 +142,6 @@ func (f *Frame) InverseEnabled() bool {
 	return f.inverseEnabled
 }
 
-// RangeEnabled returns true if range fields can be stored on this frame.
-func (f *Frame) RangeEnabled() bool {
-	return f.rangeEnabled
-}
-
 // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update.
 // defaults to DefaultCacheSize 50000
 func (f *Frame) SetCacheSize(v uint32) error {
@@ -188,7 +180,6 @@ func (f *Frame) Options() FrameOptions {
 func (f *Frame) options() FrameOptions {
 	return FrameOptions{
 		InverseEnabled: f.inverseEnabled,
-		RangeEnabled:   f.rangeEnabled,
 		CacheType:      f.cacheType,
 		CacheSize:      f.cacheSize,
 		TimeQuantum:    f.timeQuantum,
@@ -268,7 +259,6 @@ func (f *Frame) loadMeta() error {
 		f.cacheType = DefaultCacheType
 		f.cacheSize = DefaultCacheSize
 		f.timeQuantum = ""
-		f.rangeEnabled = DefaultRangeEnabled
 		//f.fields
 		return nil
 	} else if err != nil {
@@ -287,7 +277,6 @@ func (f *Frame) loadMeta() error {
 	}
 	f.cacheSize = pb.CacheSize
 	f.timeQuantum = TimeQuantum(pb.TimeQuantum)
-	f.rangeEnabled = pb.RangeEnabled
 	f.fields = decodeFields(pb.Fields)
 
 	return nil
@@ -365,11 +354,6 @@ func (f *Frame) CreateField(field *Field) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 
-	// Ensure frame supports fields.
-	if !f.RangeEnabled() {
-		return ErrFrameFieldsNotAllowed
-	}
-
 	// Append field.
 	if err := f.addField(field); err != nil {
 		return err
@@ -402,11 +386,6 @@ func (f *Frame) GetFields() ([]*Field, error) {
 	f.mu.RLock()
 	defer f.mu.RUnlock()
 
-	// Ensure the frame supports fields.
-	if !f.RangeEnabled() {
-		return nil, ErrFrameFieldsNotAllowed
-	}
-
 	err := f.loadMeta()
 	if err != nil {
 		return nil, err
@@ -420,11 +399,6 @@ func (f *Frame) DeleteField(name string) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 
-	// Ensure frame supports fields.
-	if !f.RangeEnabled() {
-		return ErrFrameFieldsNotAllowed
-	}
-
 	// Remove field.
 	if err := f.deleteField(name); err != nil {
 		return err
@@ -890,11 +864,6 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
 
 // ImportValue bulk imports range-encoded value data.
 func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error {
-	// Verify that this frame is range-encoded.
-	if !f.RangeEnabled() {
-		return fmt.Errorf("Frame not RangeEnabled: %s", f.name)
-	}
-
 	viewName := ViewFieldPrefix + fieldName
 	// Get the field so we know bitDepth.
 	field := f.Field(fieldName)
@@ -991,7 +960,7 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
 // FrameOptions represents options to set when initializing a frame.
 type FrameOptions struct {
 	InverseEnabled bool        `json:"inverseEnabled,omitempty"`
-	RangeEnabled   bool        `json:"rangeEnabled,omitempty"`
+	RangeEnabled   bool        `json:"rangeEnabled,omitempty"` // deprecated, will be removed
 	CacheType      string      `json:"cacheType,omitempty"`
 	CacheSize      uint32      `json:"cacheSize,omitempty"`
 	TimeQuantum    TimeQuantum `json:"timeQuantum,omitempty"`
@@ -1009,7 +978,6 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta {
 	}
 	return &internal.FrameMeta{
 		InverseEnabled: o.InverseEnabled,
-		RangeEnabled:   o.RangeEnabled,
 		CacheType:      o.CacheType,
 		CacheSize:      o.CacheSize,
 		TimeQuantum:    string(o.TimeQuantum),
@@ -1023,7 +991,6 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions {
 	}
 	return &FrameOptions{
 		InverseEnabled: options.InverseEnabled,
-		RangeEnabled:   options.RangeEnabled,
 		CacheType:      options.CacheType,
 		CacheSize:      options.CacheSize,
 		TimeQuantum:    TimeQuantum(options.TimeQuantum),

frame_test.go

@@ -77,7 +77,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
 		defer idx.Close()
 
 		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
-			RangeEnabled: true,
 			Fields: []*pilosa.Field{
 				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
 				{Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25},
@@ -123,7 +122,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
 		defer idx.Close()
 
 		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
-			RangeEnabled: true,
 			Fields: []*pilosa.Field{
 				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
 			},
@@ -161,7 +159,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
 		defer idx.Close()
 
 		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
-			RangeEnabled: true,
 			Fields: []*pilosa.Field{
 				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
 			},
@@ -181,7 +178,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
 		defer idx.Close()
 
 		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
-			RangeEnabled: true,
 			Fields: []*pilosa.Field{
 				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
 			},
@@ -201,7 +197,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
 		defer idx.Close()
 
 		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
-			RangeEnabled: true,
 			Fields: []*pilosa.Field{
 				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
 			},

gossip/gossip.go

@@ -29,6 +29,7 @@ import (
 	"github.com/hashicorp/memberlist"
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/internal"
+	"github.com/pilosa/pilosa/toml"
 	"github.com/pkg/errors"
 )
 
@@ -151,7 +152,7 @@ type gossipConfig struct {
 type GossipMemberSetOption func(*GossipMemberSet) error
 
 // WithTransport is a functional option for providing a transport to NewGossipMemberSet.
-func WithTransport(transport *Transport) func(*GossipMemberSet) error {
+func WithTransport(transport *Transport) GossipMemberSetOption {
 	return func(g *GossipMemberSet) error {
 		g.transport = transport
 		return nil
@@ -159,7 +160,7 @@ func WithTransport(transport *Transport) func(*GossipMemberSet) error {
 }
 
 // WithLogger is a functional option for providing a logger to NewGossipMemberSet.
-func WithLogger(logger *log.Logger) func(*GossipMemberSet) error {
+func WithLogger(logger *log.Logger) GossipMemberSetOption {
 	return func(g *GossipMemberSet) error {
 		g.logger = logger
 		return nil
@@ -167,11 +168,8 @@ func WithLogger(logger *log.Logger) func(*GossipMemberSet) error {
 }
 
 // NewGossipMemberSet returns a new instance of GossipMemberSet based on options.
-func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
-
-	g := &GossipMemberSet{
-		Logger: server.Logger,
-	}
+func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventReceiver, sh pilosa.StatusHandler, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
+	g := &GossipMemberSet{}
 
 	// options
 	for _, opt := range options {
@@ -181,17 +179,11 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server,
 	}
 
 	if g.transport == nil {
-		port, err := strconv.Atoi(cfg.Gossip.Port)
+		port, err := strconv.Atoi(cfg.Port)
 		if err != nil {
 			return nil, fmt.Errorf("convert port: %s", err)
 		}
 
-		bindURI, err := pilosa.NewURIFromAddress(cfg.Bind)
-		if err != nil {
-			return nil, fmt.Errorf("getting uri from bind address: %s", err)
-		}
-		host := bindURI.Host()
-
 		// Set up the transport.
 		transport, err := NewTransport(host, port, g.logger)
 		if err != nil {
@@ -203,15 +195,10 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server,
 
 	port := g.transport.Net.GetAutoBindPort()
 
-	bindURI, err := pilosa.NewURIFromAddress(cfg.Bind)
-	if err != nil {
-		return nil, fmt.Errorf("getting uri from bind address (with transport): %s", err)
-	}
-	host := bindURI.Host()
-
 	var gossipKey []byte
-	if cfg.Gossip.Key != "" {
-		gossipKey, err = ioutil.ReadFile(cfg.Gossip.Key)
+	var err error
+	if cfg.Key != "" {
+		gossipKey, err = ioutil.ReadFile(cfg.Key)
 		if err != nil {
 			return nil, fmt.Errorf("reading gossip key: %s", err)
 		}
@@ -226,26 +213,26 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server,
 	conf.AdvertisePort = port
 	conf.AdvertiseAddr = pilosa.HostToIP(host)
 	//
-	conf.TCPTimeout = time.Duration(cfg.Gossip.StreamTimeout)
-	conf.SuspicionMult = cfg.Gossip.SuspicionMult
-	conf.PushPullInterval = time.Duration(cfg.Gossip.PushPullInterval)
-	conf.ProbeTimeout = time.Duration(cfg.Gossip.ProbeTimeout)
-	conf.ProbeInterval = time.Duration(cfg.Gossip.ProbeInterval)
-	conf.GossipNodes = cfg.Gossip.Nodes
-	conf.GossipInterval = time.Duration(cfg.Gossip.Interval)
-	conf.GossipToTheDeadTime = time.Duration(cfg.Gossip.ToTheDeadTime)
+	conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
+	conf.SuspicionMult = cfg.SuspicionMult
+	conf.PushPullInterval = time.Duration(cfg.PushPullInterval)
+	conf.ProbeTimeout = time.Duration(cfg.ProbeTimeout)
+	conf.ProbeInterval = time.Duration(cfg.ProbeInterval)
+	conf.GossipNodes = cfg.Nodes
+	conf.GossipInterval = time.Duration(cfg.Interval)
+	conf.GossipToTheDeadTime = time.Duration(cfg.ToTheDeadTime)
 	//
 	conf.Delegate = g
 	conf.SecretKey = gossipKey
-	conf.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate)
+	conf.Events = ger
 	conf.Logger = g.logger
 
 	g.config = &gossipConfig{
 		memberlistConfig: conf,
-		gossipSeeds:      cfg.Gossip.Seeds,
+		gossipSeeds:      cfg.Seeds,
 	}
 
-	g.statusHandler = server
+	g.statusHandler = sh
 
 	return g, nil
 }
@@ -526,3 +513,68 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
 
 	return nt, nil
 }
+
+// Config holds toml-friendly memberlist configuration.
+type Config struct {
+	// Port indicates the port to which pilosa should bind for internal state sharing.
+	Port  string   `toml:"port"`
+	Seeds []string `toml:"seeds"`
+	Key   string   `toml:"key"`
+	// StreamTimeout is the timeout for establishing a stream connection with
+	// a remote node for a full state sync, and for stream read and write
+	// operations. Maps to memberlist TCPTimeout.
+	StreamTimeout toml.Duration `toml:"stream-timeout"`
+	// SuspicionMult is the multiplier for determining the time an
+	// inaccessible node is considered suspect before declaring it dead.
+	// The actual timeout is calculated using the formula:
+	//
+	//   SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
+	//
+	// This allows the timeout to scale properly with expected propagation
+	// delay with a larger cluster size. The higher the multiplier, the longer
+	// an inaccessible node is considered part of the cluster before declaring
+	// it dead, giving that suspect node more time to refute if it is indeed
+	// still alive.
+	SuspicionMult int `toml:"suspicion-mult"`
+	// PushPullInterval is the interval between complete state syncs.
+	// Complete state syncs are done with a single node over TCP and are
+	// quite expensive relative to standard gossiped messages. Setting this
+	// to zero will disable state push/pull syncs completely.
+	//
+	// Setting this interval lower (more frequent) will increase convergence
+	// speeds across larger clusters at the expense of increased bandwidth
+	// usage.
+	PushPullInterval toml.Duration `toml:"push-pull-interval"`
+	// ProbeInterval and ProbeTimeout are used to configure probing behavior
+	// for memberlist.
+	//
+	// ProbeInterval is the interval between random node probes. Setting
+	// this lower (more frequent) will cause the memberlist cluster to detect
+	// failed nodes more quickly at the expense of increased bandwidth usage.
+	//
+	// ProbeTimeout is the timeout to wait for an ack from a probed node
+	// before assuming it is unhealthy. This should be set to 99-percentile
+	// of RTT (round-trip time) on your network.
+	ProbeInterval toml.Duration `toml:"probe-interval"`
+	ProbeTimeout  toml.Duration `toml:"probe-timeout"`
+
+	// Interval and Nodes are used to configure the gossip
+	// behavior of memberlist.
+	//
+	// Interval is the interval between sending messages that need
+	// to be gossiped that haven't been able to piggyback on probing messages.
+	// If this is set to zero, non-piggyback gossip is disabled. By lowering
+	// this value (more frequent) gossip messages are propagated across
+	// the cluster more quickly at the expense of increased bandwidth.
+	//
+	// Nodes is the number of random nodes to send gossip messages to
+	// per Interval. Increasing this number causes the gossip messages
+	// to propagate across the cluster more quickly at the expense of
+	// increased bandwidth.
+	//
+	// ToTheDeadTime is the interval after which a node has died that
+	// we will still try to gossip to it. This gives it a chance to refute.
+	Interval      toml.Duration `toml:"interval"`
+	Nodes         int           `toml:"nodes"`
+	ToTheDeadTime toml.Duration `toml:"to-the-dead-time"`
+}

handler.go

@@ -15,7 +15,6 @@
 package pilosa
 
 import (
-	"context"
 	"encoding/json"
 	"expvar"
 	"fmt"
@@ -35,7 +34,6 @@ import (
 	"github.com/gogo/protobuf/proto"
 	"github.com/gorilla/mux"
 	"github.com/pilosa/pilosa/internal"
-	"github.com/pilosa/pilosa/pql"
 	"github.com/pkg/errors"
 )
 
@@ -43,14 +41,7 @@ import (
 type Handler struct {
 	Router *mux.Router
 
-	FileSystem       FileSystem
-	NormalRouter     *mux.Router
-	RestrictedRouter *mux.Router
-
-	// The execution engine for running queries.
-	Executor interface {
-		Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error)
-	}
+	FileSystem FileSystem
 
 	Logger Logger
 
@@ -83,29 +74,11 @@ func NewHandler() *Handler {
 		FileSystem: NopFileSystem,
 		Logger:     NopLogger,
 	}
-	BuildRouters(handler)
+	handler.Router = NewRouter(handler)
 	handler.populateValidators()
 	return handler
 }
 
-// BuildRouters creates Gorilla Mux http routers for both normal and restricted endpoints.
-func BuildRouters(handler *Handler) {
-	router := mux.NewRouter()
-	loadCommon(router, handler)
-	loadNormal(router, handler)
-	handler.NormalRouter = router
-	router.Use(handler.queryArgValidator)
-
-	// Restricted router.
-	router = mux.NewRouter()
-	loadCommon(router, handler)
-	loadRestricted(router, handler)
-	handler.RestrictedRouter = router
-	router.Use(handler.queryArgValidator)
-
-	handler.SetRestricted()
-}
-
 func (h *Handler) populateValidators() {
 	h.validators = map[string]*queryValidationSpec{}
 	h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index")
@@ -138,17 +111,9 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
 	})
 }
 
-// SetNormal is a method of the SecurityManager interface which provides normal URI routing.
-func (h *Handler) SetNormal() {
-	h.Router = h.NormalRouter
-}
-
-// SetRestricted is a method of the SecurityManager interface which provides restricted URI routing.
-func (h *Handler) SetRestricted() {
-	h.Router = h.RestrictedRouter
-}
-
-func loadCommon(router *mux.Router, handler *Handler) {
+// NewRouter creates a new mux http router.
+func NewRouter(handler *Handler) *mux.Router {
+	router := mux.NewRouter()
 	router.HandleFunc("/", handler.handleWebUI).Methods("GET")
 	router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET")
 	router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST")
@@ -162,16 +127,9 @@ func loadCommon(router *mux.Router, handler *Handler) {
 	router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups)
 	router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
 	router.HandleFunc("/version", handler.handleGetVersion).Methods("GET")
-	router.Use(handler.queryArgValidator)
-}
 
-func loadRestricted(router *mux.Router, handler *Handler) {
 	router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST")
-	router.NotFoundHandler = http.HandlerFunc(handler.reportRestricted)
-	router.Use(handler.queryArgValidator)
-}
 
-func loadNormal(router *mux.Router, handler *Handler) {
 	router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST")
 	router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
 	router.Handle("/debug/vars", expvar.Handler()).Methods("GET")
@@ -192,7 +150,6 @@ func loadNormal(router *mux.Router, handler *Handler) {
 	router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE")
 	router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST")
 	router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST").Name("PostFrameRestore")
-	router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH")
 	router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST")
 	router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET")
 	router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE")
@@ -203,7 +160,6 @@ func loadNormal(router *mux.Router, handler *Handler) {
 	router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostInputDefinition).Methods("POST")
 	router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteInputDefinition).Methods("DELETE")
 	router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery")
-	router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH")
 	router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST")
 
 	// TODO: Apply MethodNotAllowed statuses to all endpoints.
@@ -212,10 +168,8 @@ func loadNormal(router *mux.Router, handler *Handler) {
 	// For now we just do it for the most commonly used handler, /query
 	router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET")
 
-}
-
-func (h *Handler) reportRestricted(w http.ResponseWriter, r *http.Request) {
-	http.Error(w, fmt.Sprintf("not allowed in cluster state %s", h.API.State()), http.StatusMethodNotAllowed)
+	router.Use(handler.queryArgValidator)
+	return router
 }
 
 func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) {
@@ -494,45 +448,6 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// handlePatchIndexTimeQuantum handles PATCH /index/time_quantum request.
-func (h *Handler) handlePatchIndexTimeQuantum(w http.ResponseWriter, r *http.Request) {
-	indexName := mux.Vars(r)["index"]
-
-	// Decode request.
-	var req patchIndexTimeQuantumRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	// Validate quantum.
-	tq, err := ParseTimeQuantum(req.TimeQuantum)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	if err = h.API.ModifyIndexTimeQuantum(r.Context(), indexName, tq); err != nil {
-		if err == ErrIndexNotFound {
-			http.Error(w, err.Error(), http.StatusNotFound)
-		} else {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-		}
-		return
-	}
-
-	// Encode response.
-	if err := json.NewEncoder(w).Encode(patchIndexTimeQuantumResponse{}); err != nil {
-		h.Logger.Printf("response encoding error: %s", err)
-	}
-}
-
-type patchIndexTimeQuantumRequest struct {
-	TimeQuantum string `json:"timeQuantum"`
-}
-
-type patchIndexTimeQuantumResponse struct{}
-
 // handlePostIndexAttrDiff handles POST /index/attr/diff requests.
 func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) {
 	indexName := mux.Vars(r)["index"]
@@ -673,46 +588,6 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) {
 
 type deleteFrameResponse struct{}
 
-// handlePatchFrameTimeQuantum handles PATCH /frame/time_quantum request.
-func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Request) {
-	indexName := mux.Vars(r)["index"]
-	frameName := mux.Vars(r)["frame"]
-
-	// Decode request.
-	var req patchFrameTimeQuantumRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	// Validate quantum.
-	tq, err := ParseTimeQuantum(req.TimeQuantum)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	if err := h.API.ModifyFrameTimeQuantum(r.Context(), indexName, frameName, tq); err != nil {
-		if err == ErrFragmentNotFound {
-			http.Error(w, err.Error(), http.StatusNotFound)
-		} else {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-		}
-		return
-	}
-
-	// Encode response.
-	if err := json.NewEncoder(w).Encode(patchFrameTimeQuantumResponse{}); err != nil {
-		h.Logger.Printf("response encoding error: %s", err)
-	}
-}
-
-type patchFrameTimeQuantumRequest struct {
-	TimeQuantum string `json:"timeQuantum"`
-}
-
-type patchFrameTimeQuantumResponse struct{}
-
 // handlePostFrameField handles POST /frame/field request.
 func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) {
 	indexName := mux.Vars(r)["index"]
@@ -788,8 +663,6 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) {
 			fallthrough
 		case ErrFrameNotFound:
 			http.Error(w, err.Error(), http.StatusNotFound)
-		case ErrFrameFieldsNotAllowed:
-			http.Error(w, err.Error(), http.StatusBadRequest)
 		default:
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 		}
@@ -1169,7 +1042,11 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
 	}
 
 	// Retrieve fragment owner nodes.
-	nodes := h.API.SliceNodes(r.Context(), index, slice)
+	nodes, err := h.API.SliceNodes(r.Context(), index, slice)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
 
 	// Write to response.
 	if err := json.NewEncoder(w).Encode(nodes); err != nil {
@@ -1727,7 +1604,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	err := h.API.PostClusterMessage(r.Context(), r.Body)
+	err := h.API.ClusterMessage(r.Context(), r.Body)
 	if err != nil {
 		// TODO this was the previous behavior, but perhaps not everything is a bad request
 		http.Error(w, err.Error(), http.StatusBadRequest)

handler_test.go

@@ -160,7 +160,7 @@ func TestHandler_ClusterResizeAbort(t *testing.T) {
 	t.Run("No resize job", func(t *testing.T) {
 		h := test.NewHandler()
 		h.API.Cluster = test.NewCluster(1)
-		h.SetRestricted()
+		h.API.Cluster.SetState(pilosa.ClusterStateResizing)
 
 		w := httptest.NewRecorder()
 		h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil))
@@ -748,50 +748,6 @@ func TestHandler_DeleteFrame(t *testing.T) {
 	}
 }
 
-// Ensure handler can set the Index time quantum.
-func TestHandler_SetIndexTimeQuantum(t *testing.T) {
-	hldr := test.MustOpenHolder()
-	defer hldr.Close()
-	hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
-
-	h := test.NewHandler()
-	h.API.Holder = hldr.Holder
-	h.API.Cluster = test.NewCluster(1)
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
-	if w.Code != http.StatusOK {
-		t.Fatalf("unexpected status code: %d", w.Code)
-	} else if body := w.Body.String(); body != `{}`+"\n" {
-		t.Fatalf("unexpected body: %s", body)
-	} else if q := hldr.Index("i0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
-		t.Fatalf("unexpected time quantum: %s", q)
-	}
-}
-
-// Ensure handler can set the frame time quantum.
-func TestHandler_SetFrameTimeQuantum(t *testing.T) {
-	hldr := test.MustOpenHolder()
-	defer hldr.Close()
-
-	// Create frame.
-	if _, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil {
-		t.Fatal(err)
-	}
-
-	h := test.NewHandler()
-	h.API.Holder = hldr.Holder
-	h.API.Cluster = test.NewCluster(1)
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
-	if w.Code != http.StatusOK {
-		t.Fatalf("unexpected status code: %d", w.Code)
-	} else if body := w.Body.String(); body != `{}`+"\n" {
-		t.Fatalf("unexpected body: %s", body)
-	} else if q := hldr.Index("i0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
-		t.Fatalf("unexpected time quantum: %s", q)
-	}
-}
-
 // Ensure the handler can return data in differing blocks for an index.
 func TestHandler_Index_AttrStore_Diff(t *testing.T) {
 	hldr := test.MustOpenHolder()
@@ -902,7 +858,7 @@ func TestHandler_Frame_AddField(t *testing.T) {
 
 	t.Run("OK", func(t *testing.T) {
 		idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-		f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true})
+		f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -927,7 +883,7 @@ func TestHandler_Frame_AddField(t *testing.T) {
 
 	t.Run("ErrInvalidFieldType", func(t *testing.T) {
 		idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-		if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}); err != nil {
+		if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
 			t.Fatal(err)
 		}
 
@@ -949,7 +905,7 @@ func TestHandler_Frame_AddField(t *testing.T) {
 
 	t.Run("ErrInvalidFieldRange", func(t *testing.T) {
 		idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-		if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}); err != nil {
+		if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
 			t.Fatal(err)
 		}
 
@@ -972,8 +928,7 @@ func TestHandler_Frame_AddField(t *testing.T) {
 	t.Run("ErrFieldAlreadyExists", func(t *testing.T) {
 		idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
 		if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{
-			RangeEnabled: true,
-			Fields:       []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}},
+			Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}},
 		}); err != nil {
 			t.Fatal(err)
 		}
@@ -1006,7 +961,7 @@ func TestHandler_Frame_DeleteField(t *testing.T) {
 
 	t.Run("OK", func(t *testing.T) {
 		idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-		f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true})
+		f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
 		if err != nil {
 			t.Fatal(err)
 		} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil {
@@ -1034,7 +989,7 @@ func TestHandler_Frame_DeleteField(t *testing.T) {
 
 	t.Run("ErrFieldNotFound", func(t *testing.T) {
 		idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-		f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true})
+		f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
 		if err != nil {
 			t.Fatal(err)
 		} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil {
@@ -1071,7 +1026,7 @@ func TestHandler_Frame_GetFields(t *testing.T) {
 
 	t.Run("OK", func(t *testing.T) {
 		idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-		f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true})
+		f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
 		if err != nil {
 			t.Fatal(err)
 		} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 1, Max: 100}); err != nil {
@@ -1105,7 +1060,7 @@ func TestHandler_Frame_GetFields(t *testing.T) {
 
 	t.Run("ErrFrameFieldNotAllowed", func(t *testing.T) {
 		idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-		_, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{RangeEnabled: false})
+		_, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{})
 
 		resp, err := http.Get(s.URL + "/index/i/frame/f1/fields")
 		if err != nil {
@@ -1113,12 +1068,12 @@ func TestHandler_Frame_GetFields(t *testing.T) {
 		}
 		if err != nil {
 			t.Fatal(err)
-		} else if resp.StatusCode != http.StatusBadRequest {
+		} else if resp.StatusCode != http.StatusOK {
 			t.Fatalf("unexpected status code: %d", resp.StatusCode)
 		} else if body, err := ioutil.ReadAll(resp.Body); err != nil {
 			t.Fatal(err)
-		} else if strings.TrimSpace(string(body)) != `frame fields not allowed` {
-			t.Fatalf("unexpected body: %q", body)
+		} else if strings.TrimSpace(string(body)) == `frame fields not allowed` {
+			t.Fatalf("shouldn't get frame fields not allowed error: %q", body)
 		}
 	})
 

holder.go

@@ -359,7 +359,6 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
 	}
 
 	// Update options.
-	index.SetTimeQuantum(opt.TimeQuantum)
 
 	h.indexes[index.Name()] = index
 
@@ -531,7 +530,6 @@ func (h *Holder) setFileLimit() {
 func (h *Holder) loadNodeID() (string, error) {
 	idPath := path.Join(h.Path, "ID")
 	nodeID := ""
-
 	h.Logger.Printf("load NodeID: %s", idPath)
 	if err := os.MkdirAll(h.Path, 0777); err != nil {
 		return "", err

holder_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/pql"
+	"github.com/pilosa/pilosa/server"
 	"github.com/pilosa/pilosa/test"
 )
 
@@ -73,22 +74,6 @@ func TestHolder_Open(t *testing.T) {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
-	t.Run("ErrIndexMetaCorrupt", func(t *testing.T) {
-		h := test.MustOpenHolder()
-		defer h.Close()
-
-		if _, err := h.CreateIndex("test", pilosa.IndexOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}); err != nil {
-			t.Fatal(err)
-		} else if err := h.Holder.Close(); err != nil {
-			t.Fatal(err)
-		} else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".meta"), 2); err != nil {
-			t.Fatal(err)
-		}
-
-		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "unexpected EOF") {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	})
 	t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) {
 		h := test.MustOpenHolder()
 		defer h.Close()
@@ -383,7 +368,7 @@ func TestHolder_DeleteIndex(t *testing.T) {
 // Ensure holder can sync with a remote holder.
 func TestHolderSyncer_SyncHolder(t *testing.T) {
 	cluster := test.NewCluster(2)
-	client := pilosa.GetHTTPClient(nil)
+	client := server.GetHTTPClient(nil)
 	// Create a local holder.
 	hldr0 := test.MustOpenHolder()
 	defer hldr0.Close()
@@ -467,7 +452,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 		Holder:       hldr0.Holder,
 		Node:         cluster.Nodes[0],
 		Cluster:      cluster,
-		RemoteClient: pilosa.GetHTTPClient(nil),
+		RemoteClient: server.GetHTTPClient(nil),
 		Stats:        pilosa.NopStatsClient,
 	}
 

index.go

@@ -39,10 +39,6 @@ type Index struct {
 	path string
 	name string
 
-	// Default time quantum for all frames in index.
-	// This can be overridden by individual frames.
-	timeQuantum TimeQuantum
-
 	// Frames by name.
 	frames map[string]*Frame
 
@@ -106,9 +102,7 @@ func (i *Index) Options() IndexOptions {
 }
 
 func (i *Index) options() IndexOptions {
-	return IndexOptions{
-		TimeQuantum: i.timeQuantum,
-	}
+	return IndexOptions{}
 }
 
 // Open opens and initializes the index.
@@ -175,7 +169,6 @@ func (i *Index) loadMeta() error {
 	// Read data from meta file.
 	buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta"))
 	if os.IsNotExist(err) {
-		i.timeQuantum = ""
 		return nil
 	} else if err != nil {
 		return err
@@ -186,17 +179,18 @@ func (i *Index) loadMeta() error {
 	}
 
 	// Copy metadata fields.
-	i.timeQuantum = TimeQuantum(pb.TimeQuantum)
 
 	return nil
 }
 
+// NOTE: Until we introduce new attributes to store in the index .meta file,
+// we don't need to actually write the file. The code related to index.options
+// and the index meta file are left in place for future use.
+/*
 // saveMeta writes meta data for the index.
 func (i *Index) saveMeta() error {
 	// Marshal metadata.
-	buf, err := proto.Marshal(&internal.IndexMeta{
-		TimeQuantum: string(i.timeQuantum),
-	})
+	buf, err := proto.Marshal(&internal.IndexMeta{})
 	if err != nil {
 		return err
 	}
@@ -208,6 +202,7 @@ func (i *Index) saveMeta() error {
 
 	return nil
 }
+*/
 
 // Close closes the index and its frames.
 func (i *Index) Close() error {
@@ -278,34 +273,6 @@ func (i *Index) SetRemoteMaxInverseSlice(v uint64) {
 	i.remoteMaxInverseSlice = v
 }
 
-// TimeQuantum returns the default time quantum for the index.
-func (i *Index) TimeQuantum() TimeQuantum {
-	i.mu.RLock()
-	defer i.mu.RUnlock()
-	return i.timeQuantum
-}
-
-// SetTimeQuantum sets the default time quantum for the index.
-func (i *Index) SetTimeQuantum(q TimeQuantum) error {
-	i.mu.Lock()
-	defer i.mu.Unlock()
-
-	// Validate input.
-	if !q.Valid() {
-		return ErrInvalidTimeQuantum
-	}
-
-	// Update value on index.
-	i.timeQuantum = q
-
-	// Perist meta data to disk.
-	if err := i.saveMeta(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // FramePath returns the path to a frame in the index.
 func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) }
 
@@ -404,13 +371,7 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
 
 	// Validate mutually exclusive options if ranges are enabled.
 	if opt.RangeEnabled {
-		if opt.InverseEnabled {
-			return nil, ErrInverseRangeNotAllowed
-		}
-	} else {
-		if len(opt.Fields) > 0 {
-			return nil, ErrFrameFieldsNotAllowed
-		}
+		i.Logger.Printf("RangeEnabled is deprecated - no need to set RangeEnabled to true when creating a frame")
 	}
 
 	// Validate fields.
@@ -431,12 +392,8 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
 		return nil, err
 	}
 
-	// Default the time quantum to what is set on the Index.
-	timeQuantum := i.timeQuantum
-	if opt.TimeQuantum != "" {
-		timeQuantum = opt.TimeQuantum
-	}
-	if err := f.SetTimeQuantum(timeQuantum); err != nil {
+	// Set the time quantum.
+	if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil {
 		f.Close()
 		return nil, err
 	}
@@ -452,9 +409,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
 	}
 
 	f.inverseEnabled = opt.InverseEnabled
-	f.rangeEnabled = opt.RangeEnabled
-
-	f.rangeEnabled = opt.RangeEnabled
 
 	// Set fields.
 	f.fields = opt.Fields
@@ -527,46 +481,6 @@ func (p indexInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p indexInfoSlice) Len() int           { return len(p) }
 func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
 
-// MergeSchemas combines indexes and frames from a and b into one schema.
-func MergeSchemas(a, b []*IndexInfo) []*IndexInfo {
-	// Generate a map from both schemas.
-	m := make(map[string]map[string]map[string]struct{})
-	for _, idxs := range [][]*IndexInfo{a, b} {
-		for _, idx := range idxs {
-			if m[idx.Name] == nil {
-				m[idx.Name] = make(map[string]map[string]struct{})
-			}
-			for _, frame := range idx.Frames {
-				if m[idx.Name][frame.Name] == nil {
-					m[idx.Name][frame.Name] = make(map[string]struct{})
-				}
-				for _, view := range frame.Views {
-					m[idx.Name][frame.Name][view.Name] = struct{}{}
-				}
-			}
-		}
-	}
-
-	// Generate new schema from map.
-	idxs := make([]*IndexInfo, 0, len(m))
-	for idx, frames := range m {
-		di := &IndexInfo{Name: idx}
-		for frame, views := range frames {
-			fi := &FrameInfo{Name: frame}
-			for view := range views {
-				fi.Views = append(fi.Views, &ViewInfo{Name: view})
-			}
-			sort.Sort(viewInfoSlice(fi.Views))
-			di.Frames = append(di.Frames, fi)
-		}
-		sort.Sort(frameInfoSlice(di.Frames))
-		idxs = append(idxs, di)
-	}
-	sort.Sort(indexInfoSlice(idxs))
-
-	return idxs
-}
-
 // EncodeIndexes converts a into its internal representation.
 func EncodeIndexes(a []*Index) []*internal.Index {
 	other := make([]*internal.Index, len(a))
@@ -586,15 +500,11 @@ func encodeIndex(d *Index) *internal.Index {
 }
 
 // IndexOptions represents options to set when initializing an index.
-type IndexOptions struct {
-	TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
-}
+type IndexOptions struct{}
 
 // Encode converts i into its internal representation.
 func (i *IndexOptions) Encode() *internal.IndexMeta {
-	return &internal.IndexMeta{
-		TimeQuantum: string(i.TimeQuantum),
-	}
+	return &internal.IndexMeta{}
 }
 
 // hasTime returns true if a contains a non-nil time.

index_test.go

@@ -58,11 +58,6 @@ func TestIndex_CreateFrame(t *testing.T) {
 			index := test.MustOpenIndex()
 			defer index.Close()
 
-			// Set index time quantum.
-			if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil {
-				t.Fatal(err)
-			}
-
 			// Create frame with explicit quantum.
 			f, err := index.CreateFrame("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")})
 			if err != nil {
@@ -71,24 +66,6 @@ func TestIndex_CreateFrame(t *testing.T) {
 				t.Fatalf("unexpected frame time quantum: %s", q)
 			}
 		})
-
-		t.Run("Inherited", func(t *testing.T) {
-			index := test.MustOpenIndex()
-			defer index.Close()
-
-			// Set index time quantum.
-			if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil {
-				t.Fatal(err)
-			}
-
-			// Create frame.
-			f, err := index.CreateFrame("f", pilosa.FrameOptions{})
-			if err != nil {
-				t.Fatal(err)
-			} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") {
-				t.Fatalf("unexpected frame time quantum: %s", q)
-			}
-		})
 	})
 
 	// Ensure frame can include range columns.
@@ -99,7 +76,7 @@ func TestIndex_CreateFrame(t *testing.T) {
 
 			// Create frame with schema and verify it exists.
 			if f, err := index.CreateFrame("f", pilosa.FrameOptions{
-				RangeEnabled: true,
+				RangeEnabled: false,
 				Fields: []*pilosa.Field{
 					{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
 					{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
@@ -124,16 +101,47 @@ func TestIndex_CreateFrame(t *testing.T) {
 			}
 		})
 
-		t.Run("ErrInverseRangeNotAllowed", func(t *testing.T) {
+		t.Run("ErrInverseRangeAllowed", func(t *testing.T) {
 			index := test.MustOpenIndex()
 			defer index.Close()
 
-			if _, err := index.CreateFrame("f", pilosa.FrameOptions{
-				InverseEnabled: true,
+			frame, err := index.CreateFrame("f", pilosa.FrameOptions{
 				RangeEnabled:   true,
-			}); err != pilosa.ErrInverseRangeNotAllowed {
+				InverseEnabled: true,
+				Fields: []*pilosa.Field{
+					&pilosa.Field{
+						Name: "myfield",
+						Type: pilosa.FieldTypeInt,
+						Min:  -20,
+						Max:  100,
+					},
+				},
+			})
+			if err != nil {
 				t.Fatal(err)
 			}
+
+			ch, err := frame.SetBit(pilosa.ViewStandard, 1, 2, nil)
+			if !ch || err != nil {
+				t.Fatal(ch, err)
+			}
+			ch, err = frame.SetBit(pilosa.ViewInverse, 1, 2, nil)
+			if !ch || err != nil {
+				t.Fatal(ch, err)
+			}
+			ch, err = frame.SetFieldValue(1, "myfield", 87)
+			if !ch || err != nil {
+				t.Fatal(ch, err)
+			}
+			views := frame.Views()
+			if len(views) != 3 {
+				var names string
+				for _, v := range views {
+					names = names + v.Name() + " "
+				}
+				t.Fatalf("Unexpected views: %s", names)
+			}
+
 		})
 
 		t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
@@ -141,8 +149,7 @@ func TestIndex_CreateFrame(t *testing.T) {
 			defer index.Close()
 
 			if _, err := index.CreateFrame("f", pilosa.FrameOptions{
-				RangeEnabled: true,
-				CacheType:    pilosa.CacheTypeRanked,
+				CacheType: pilosa.CacheTypeRanked,
 			}); err != nil {
 				t.Fatal(err)
 			}
@@ -152,15 +159,14 @@ func TestIndex_CreateFrame(t *testing.T) {
 			index := test.MustOpenIndex()
 			defer index.Close()
 			if _, err := index.CreateFrame("f", pilosa.FrameOptions{
-				RangeEnabled: true,
-				CacheType:    pilosa.CacheTypeNone,
-				CacheSize:    uint32(5),
+				CacheType: pilosa.CacheTypeNone,
+				CacheSize: uint32(5),
 			}); err != nil {
 				t.Fatal(err)
 			}
 		})
 
-		t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) {
+		t.Run("ErrFrameFieldsAllowed", func(t *testing.T) {
 			index := test.MustOpenIndex()
 			defer index.Close()
 
@@ -168,7 +174,7 @@ func TestIndex_CreateFrame(t *testing.T) {
 				Fields: []*pilosa.Field{
 					{Name: "field0", Type: pilosa.FieldTypeInt},
 				},
-			}); err != pilosa.ErrFrameFieldsNotAllowed {
+			}); err != nil {
 				t.Fatal(err)
 			}
 		})
@@ -178,7 +184,6 @@ func TestIndex_CreateFrame(t *testing.T) {
 			defer index.Close()
 
 			if _, err := index.CreateFrame("f", pilosa.FrameOptions{
-				RangeEnabled: true,
 				Fields: []*pilosa.Field{
 					{Name: "", Type: pilosa.FieldTypeInt},
 				},
@@ -192,7 +197,6 @@ func TestIndex_CreateFrame(t *testing.T) {
 			defer index.Close()
 
 			if _, err := index.CreateFrame("f", pilosa.FrameOptions{
-				RangeEnabled: true,
 				Fields: []*pilosa.Field{
 					{Name: "field0", Type: "bad_type"},
 				},
@@ -206,7 +210,7 @@ func TestIndex_CreateFrame(t *testing.T) {
 			defer index.Close()
 
 			if _, err := index.CreateFrame("f", pilosa.FrameOptions{
-				RangeEnabled: true,
+				RangeEnabled: true, // make sure we can still create frames with RangeEnabled: true after deprecation
 				Fields: []*pilosa.Field{
 					{Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50},
 				},
@@ -240,26 +244,6 @@ func TestIndex_DeleteFrame(t *testing.T) {
 	}
 }
 
-// Ensure index can set the default time quantum.
-func TestIndex_SetTimeQuantum(t *testing.T) {
-	index := test.MustOpenIndex()
-	defer index.Close()
-
-	// Set & retrieve time quantum.
-	if err := index.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
-		t.Fatal(err)
-	} else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
-		t.Fatalf("unexpected quantum: %s", q)
-	}
-
-	// Reload index and verify that it is persisted.
-	if err := index.Reopen(); err != nil {
-		t.Fatal(err)
-	} else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
-		t.Fatalf("unexpected quantum (reopen): %s", q)
-	}
-}
-
 // Ensure index can delete a frame.
 func TestIndex_InvalidName(t *testing.T) {
 	path, err := ioutil.TempDir("", "pilosa-index-")
@@ -377,18 +361,13 @@ func TestIndex_InputBits(t *testing.T) {
 	index := test.MustOpenIndex()
 	defer index.Close()
 
-	// Set index time quantum.
-	if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil {
-		t.Fatal(err)
-	}
-
 	err := index.InputBits("f", bits)
 	if !strings.Contains(err.Error(), "Frame not found") {
 		t.Fatalf("Expected Frame not found error, actual error: %s", err)
 	}
 
 	// Create frame.
-	if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
+	if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YM")}); err != nil {
 		t.Fatal(err)
 	}
 

internal/private.pb.go

@@ -68,7 +68,6 @@ var _ = math.Inf
 const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
 
 type IndexMeta struct {
-	TimeQuantum string `protobuf:"bytes,2,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
 }
 
 func (m *IndexMeta) Reset()                    { *m = IndexMeta{} }
@@ -76,13 +75,6 @@ func (m *IndexMeta) String() string            { return proto.CompactTextString(
 func (*IndexMeta) ProtoMessage()               {}
 func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} }
 
-func (m *IndexMeta) GetTimeQuantum() string {
-	if m != nil {
-		return m.TimeQuantum
-	}
-	return ""
-}
-
 type FrameMeta struct {
 	InverseEnabled bool     `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"`
 	CacheType      string   `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"`
@@ -1232,12 +1224,6 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) {
 	_ = i
 	var l int
 	_ = l
-	if len(m.TimeQuantum) > 0 {
-		dAtA[i] = 0x12
-		i++
-		i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum)))
-		i += copy(dAtA[i:], m.TimeQuantum)
-	}
 	return i, nil
 }
 
@@ -2746,10 +2732,6 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int {
 func (m *IndexMeta) Size() (n int) {
 	var l int
 	_ = l
-	l = len(m.TimeQuantum)
-	if l > 0 {
-		n += 1 + l + sovPrivate(uint64(l))
-	}
 	return n
 }
 
@@ -3439,35 +3421,6 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error {
 			return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire)
 		}
 		switch fieldNum {
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPrivate
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.TimeQuantum = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipPrivate(dAtA[iNdEx:])
@@ -8573,87 +8526,87 @@ var (
 func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
 
 var fileDescriptorPrivate = []byte{
-	// 1308 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45,
-	0x14, 0x67, 0xbd, 0xb6, 0x13, 0x3f, 0xd7, 0xa9, 0x33, 0x6d, 0x83, 0x5b, 0x45, 0xae, 0x19, 0x15,
-	0x1a, 0x2a, 0x35, 0x2a, 0xa9, 0x84, 0x68, 0xa1, 0x52, 0x69, 0xec, 0xaa, 0x0b, 0xa4, 0x2a, 0xe3,
-	0xb6, 0x48, 0x48, 0x20, 0x4d, 0xed, 0x21, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0x49, 0xdc, 0x03,
-	0x47, 0x84, 0x84, 0xb8, 0x23, 0xae, 0x7c, 0x19, 0x8e, 0x7c, 0x02, 0x84, 0xc2, 0x87, 0xe0, 0x08,
-	0x9a, 0x37, 0x33, 0xbb, 0xeb, 0x7f, 0x49, 0x13, 0xb8, 0xed, 0xfb, 0xff, 0x9b, 0xf7, 0x6f, 0x66,
-	0xa1, 0x36, 0x8c, 0xfd, 0x7d, 0x2e, 0xc5, 0xe6, 0x30, 0x8e, 0x64, 0x44, 0x96, 0xfd, 0x50, 0x8a,
-	0x38, 0xe4, 0x01, 0xbd, 0x09, 0x15, 0x2f, 0xec, 0x8b, 0xc3, 0x1d, 0x21, 0x39, 0x69, 0x41, 0xf5,
-	0xa9, 0x3f, 0x10, 0x9f, 0x8f, 0x78, 0x28, 0x47, 0x83, 0x46, 0xa1, 0xe5, 0x6c, 0x54, 0x58, 0x9e,
-	0x45, 0xff, 0x70, 0xa0, 0xf2, 0x30, 0xe6, 0x03, 0x81, 0xfa, 0xef, 0xc0, 0x8a, 0x17, 0xee, 0x8b,
-	0x38, 0x11, 0x9d, 0x90, 0xbf, 0x08, 0x44, 0x1f, 0x4d, 0x96, 0xd9, 0x14, 0x97, 0xac, 0x43, 0x65,
-	0x9b, 0xf7, 0x5e, 0x8a, 0xa7, 0xe3, 0xa1, 0x68, 0xb8, 0xe8, 0x35, 0x63, 0xa4, 0xd2, 0xae, 0xff,
-	0x4a, 0x34, 0x8a, 0x2d, 0x67, 0xa3, 0xc6, 0x32, 0xc6, 0x34, 0xa6, 0xd2, 0x0c, 0x26, 0x42, 0xe1,
-	0x1c, 0xe3, 0xe1, 0x6e, 0x8a, 0xa1, 0x8c, 0x18, 0x26, 0x78, 0xe4, 0x3a, 0x94, 0x1f, 0xfa, 0x22,
-	0xe8, 0x27, 0x8d, 0xa5, 0x96, 0xbb, 0x51, 0xdd, 0x3a, 0xbf, 0x69, 0x33, 0xb0, 0x89, 0x7c, 0x66,
-	0xc4, 0x94, 0xc2, 0x8a, 0x37, 0x18, 0x46, 0xb1, 0x64, 0x22, 0x19, 0x46, 0x61, 0x22, 0x48, 0x1d,
-	0xdc, 0x4e, 0x1c, 0x37, 0x1c, 0x0c, 0xac, 0x3e, 0xe9, 0x77, 0x50, 0x7f, 0x10, 0x44, 0xbd, 0xbd,
-	0x36, 0x97, 0x9c, 0x89, 0x6f, 0x47, 0x22, 0x91, 0xe4, 0x22, 0x94, 0x30, 0x8f, 0x46, 0x4f, 0x13,
-	0x8a, 0x8b, 0xd9, 0x32, 0xa9, 0xd4, 0x84, 0xe2, 0xa2, 0x3d, 0xa6, 0xa2, 0xc8, 0x34, 0xa1, 0xb8,
-	0xdd, 0xc0, 0xef, 0xe9, 0x14, 0x14, 0x99, 0x26, 0x08, 0x81, 0xe2, 0x73, 0x5f, 0x1c, 0x98, 0x73,
-	0xe3, 0x37, 0xf5, 0x60, 0x35, 0x17, 0xdf, 0xc0, 0x5c, 0x83, 0x32, 0x8b, 0x0e, 0xbc, 0x76, 0xd2,
-	0x70, 0x5a, 0xee, 0x46, 0x91, 0x19, 0x0a, 0xb3, 0x1b, 0x05, 0xa3, 0x41, 0xa8, 0x44, 0x05, 0x14,
-	0x65, 0x0c, 0x7a, 0x19, 0x4a, 0x98, 0x6a, 0x75, 0xca, 0xcc, 0x56, 0x7d, 0xd2, 0x7f, 0x1c, 0xa8,
-	0xec, 0xf0, 0x43, 0x84, 0x91, 0x90, 0x7b, 0xb0, 0xdc, 0x95, 0x3c, 0xec, 0xf3, 0xb8, 0x8f, 0x4a,
-	0xd5, 0xad, 0xb7, 0xb2, 0x14, 0xa6, 0x6a, 0x9b, 0x56, 0xa7, 0x13, 0xca, 0x78, 0xcc, 0x52, 0x13,
-	0x72, 0x17, 0x96, 0x4c, 0x4f, 0x20, 0x86, 0xea, 0x56, 0x6b, 0x9e, 0x75, 0xda, 0x36, 0xca, 0xd8,
-	0x1a, 0x5c, 0xf9, 0x10, 0x6a, 0x13, 0x6e, 0x15, 0xd6, 0x3d, 0x31, 0xb6, 0x15, 0xd9, 0x13, 0x63,
-	0x95, 0xbb, 0x7d, 0x1e, 0x8c, 0x74, 0x9e, 0x8b, 0x4c, 0x13, 0x77, 0x0b, 0x1f, 0x38, 0x57, 0xee,
-	0xc2, 0xb9, 0xbc, 0xd7, 0xd3, 0xd8, 0xd2, 0xaf, 0x81, 0x6c, 0xc7, 0x82, 0x4b, 0x81, 0xf0, 0x76,
-	0x44, 0x92, 0xf0, 0x5d, 0xb1, 0xb8, 0xd2, 0xba, 0x7a, 0x85, 0x7c, 0xf5, 0xd6, 0xa1, 0xe2, 0x25,
-	0xf6, 0xe0, 0x2e, 0xf6, 0x65, 0xc6, 0xa0, 0x37, 0x80, 0xb4, 0x45, 0x20, 0xa4, 0x30, 0x13, 0x78,
-	0x8c, 0x7f, 0xda, 0xb5, 0x58, 0x4e, 0xd6, 0x25, 0xd7, 0xa1, 0xa8, 0xc6, 0x13, 0xa1, 0x54, 0xb7,
-	0x2e, 0x64, 0x99, 0x4e, 0x27, 0x9d, 0xa1, 0x02, 0xf5, 0xad, 0x53, 0x33, 0xd2, 0x27, 0x1c, 0x70,
-	0x4e, 0x2b, 0xdb, 0x50, 0xee, 0x74, 0xa8, 0x74, 0x49, 0x98, 0x50, 0xf7, 0xed, 0x59, 0xcf, 0x1a,
-	0x8a, 0xee, 0xa6, 0x60, 0xd5, 0xa4, 0x9e, 0x05, 0xec, 0xdb, 0x50, 0x42, 0x5b, 0x83, 0x76, 0x66,
-	0x07, 0x68, 0x29, 0x7d, 0x9e, 0x42, 0x3d, 0x6b, 0xa0, 0x8b, 0xf9, 0x40, 0x15, 0xeb, 0xf7, 0x4b,
-	0xa3, 0xab, 0x66, 0xfa, 0xb1, 0xb2, 0xd1, 0x9e, 0xf0, 0x7b, 0x71, 0xcd, 0xa6, 0x12, 0xa9, 0x7c,
-	0xab, 0x25, 0x90, 0x34, 0xdc, 0x96, 0xab, 0x7c, 0x23, 0x41, 0x6f, 0x43, 0xb9, 0xdb, 0x7b, 0x29,
-	0x06, 0x9c, 0xbc, 0xab, 0x26, 0xad, 0x2f, 0x0e, 0x45, 0x62, 0xe6, 0xf4, 0xfc, 0x54, 0xfd, 0x99,
-	0x95, 0xd3, 0x1f, 0x1d, 0x73, 0xa6, 0x05, 0x88, 0xca, 0x18, 0x3b, 0x69, 0x14, 0x67, 0x56, 0xa6,
-	0xe2, 0x33, 0x23, 0x26, 0x1d, 0xa8, 0x7b, 0xe1, 0x70, 0x24, 0xdb, 0xe2, 0x1b, 0x3f, 0xf4, 0xa5,
-	0x1f, 0x85, 0x49, 0xa3, 0x8c, 0x26, 0x97, 0xf3, 0xa1, 0x27, 0x34, 0xd8, 0x8c, 0x09, 0xfd, 0xde,
-	0x81, 0xf3, 0x53, 0xcc, 0x13, 0x70, 0x15, 0x8e, 0xc7, 0xf5, 0x7e, 0xba, 0xf3, 0x5d, 0x54, 0x6c,
-	0x2e, 0x44, 0x33, 0x79, 0x05, 0xfc, 0xea, 0xc0, 0xc5, 0x79, 0x0a, 0x73, 0xd1, 0x34, 0x01, 0x9e,
-	0xc4, 0xfe, 0x80, 0xc7, 0xe3, 0x4f, 0xc5, 0xd8, 0x5c, 0x7f, 0x39, 0x0e, 0xf9, 0x02, 0xd6, 0xa6,
-	0x7c, 0x7d, 0xdc, 0xd3, 0x29, 0xd2, 0xa0, 0xae, 0x2e, 0x04, 0xa5, 0xf5, 0xd8, 0x02, 0x73, 0xfa,
-	0xb7, 0x03, 0x97, 0xe6, 0x8a, 0xb2, 0x9e, 0x74, 0xf2, 0x3d, 0x79, 0x03, 0xea, 0xcf, 0xd5, 0x66,
-	0x6b, 0x8b, 0x44, 0xfa, 0x21, 0x57, 0x9a, 0xa6, 0x69, 0x67, 0xf8, 0xc4, 0x83, 0x65, 0xe4, 0xed,
-	0xf0, 0xa1, 0x81, 0x79, 0xf3, 0x04, 0x98, 0x9b, 0x56, 0xdf, 0x2c, 0x7e, 0x4b, 0x2a, 0x30, 0x78,
-	0x11, 0xd9, 0x5b, 0x0d, 0x09, 0xb5, 0xd2, 0x27, 0x0c, 0x4e, 0xb5, 0x96, 0x23, 0x58, 0xb7, 0xab,
-	0x70, 0x02, 0xc9, 0xf1, 0x93, 0x7a, 0x07, 0x20, 0x53, 0x35, 0x1b, 0xe0, 0x98, 0xfe, 0xcc, 0x29,
-	0xd3, 0x47, 0xb0, 0x6e, 0xf7, 0xf4, 0x29, 0x02, 0xda, 0x6e, 0x29, 0x64, 0xdd, 0x42, 0x3b, 0xe0,
-	0x3e, 0x63, 0x9e, 0xba, 0xab, 0x71, 0x5a, 0x6d, 0x89, 0x0c, 0xa5, 0x4c, 0x1e, 0x45, 0x89, 0xb4,
-	0x26, 0xea, 0x5b, 0xf1, 0x9e, 0x44, 0xb1, 0x44, 0xc4, 0x35, 0x86, 0xdf, 0xf4, 0x2b, 0x28, 0x3e,
-	0x8e, 0xfa, 0x82, 0xac, 0x40, 0xc1, 0x6b, 0x1b, 0x1f, 0x05, 0xaf, 0x4d, 0xae, 0xa2, 0x7b, 0xb3,
-	0x43, 0x6a, 0xd9, 0xe1, 0x9e, 0x31, 0x8f, 0x61, 0xe0, 0x6b, 0x50, 0xf3, 0x92, 0xed, 0x28, 0x8a,
-	0xfb, 0xaa, 0xd4, 0x51, 0x6c, 0xee, 0xa4, 0x49, 0x26, 0xbd, 0x0f, 0x75, 0xe5, 0xbe, 0x2b, 0xb9,
-	0x4c, 0x37, 0xf5, 0x1a, 0x94, 0x15, 0x2f, 0x0d, 0x67, 0x28, 0xbc, 0xf7, 0x94, 0x9e, 0x5d, 0x80,
-	0x48, 0xd0, 0xcf, 0xb4, 0x87, 0xce, 0xbe, 0x08, 0x65, 0x2e, 0x4b, 0x48, 0xa3, 0x83, 0x1a, 0xd3,
-	0x04, 0xa1, 0xfa, 0x28, 0x06, 0xf3, 0x4a, 0x86, 0x59, 0x71, 0x19, 0xca, 0xe8, 0x4f, 0x0e, 0x80,
-	0x05, 0x34, 0x4a, 0x52, 0x13, 0x67, 0xb1, 0x09, 0x79, 0x2f, 0xf7, 0x76, 0x99, 0xdd, 0xa9, 0xa9,
-	0x88, 0xe5, 0x5e, 0x38, 0x1b, 0x76, 0x85, 0x9a, 0xe6, 0xa8, 0x67, 0xfa, 0x9a, 0x6f, 0xca, 0xa4,
-	0xae, 0xcd, 0xda, 0x76, 0x30, 0x4a, 0xa4, 0x88, 0x0d, 0x22, 0xf5, 0xc6, 0xd2, 0x8c, 0x34, 0x3f,
-	0x19, 0x63, 0x7e, 0x8a, 0xc8, 0x35, 0x28, 0x29, 0xa4, 0x76, 0x0f, 0x4c, 0x1f, 0x43, 0x0b, 0x69,
-	0xd7, 0xdc, 0x24, 0x73, 0x77, 0x0f, 0x81, 0x22, 0xbe, 0xa8, 0x4d, 0xbb, 0xe0, 0x63, 0xba, 0x0e,
-	0xee, 0x8e, 0xaf, 0xfb, 0xdb, 0x65, 0xea, 0x13, 0x39, 0xfc, 0x10, 0xe7, 0x4f, 0x71, 0xb8, 0x7a,
-	0x4b, 0xac, 0xea, 0x01, 0x52, 0x77, 0xc7, 0x59, 0xee, 0x37, 0xfb, 0x28, 0x75, 0x73, 0x8f, 0xd2,
-	0x2e, 0xac, 0xea, 0x21, 0xf9, 0x3f, 0x9d, 0xfe, 0x52, 0x80, 0x55, 0x26, 0x12, 0xff, 0x95, 0xf0,
-	0xc2, 0x44, 0xc6, 0xa3, 0x74, 0xc1, 0x7d, 0x12, 0xbd, 0x30, 0xa9, 0x76, 0x99, 0x26, 0x5e, 0xa7,
-	0x93, 0xc8, 0x2d, 0xa8, 0x4e, 0x77, 0xff, 0xac, 0x6a, 0x5e, 0x85, 0xdc, 0x82, 0xa5, 0x6e, 0x34,
-	0x8a, 0x7b, 0xe9, 0x35, 0xb8, 0x96, 0x69, 0x6b, 0x64, 0x5a, 0xcc, 0xac, 0x5a, 0xae, 0x8f, 0x4a,
-	0xc7, 0xf7, 0x11, 0xb9, 0x37, 0xd5, 0x47, 0xf8, 0xe7, 0x52, 0xdd, 0x7a, 0x33, 0x33, 0x98, 0x10,
-	0xb3, 0x49, 0x6d, 0xfa, 0x83, 0x03, 0xe7, 0xf2, 0x10, 0x5e, 0x6b, 0x30, 0xd2, 0x8a, 0x14, 0xe6,
-	0x56, 0xc4, 0x9d, 0x57, 0x91, 0x62, 0x56, 0x91, 0xec, 0x9d, 0x5b, 0xca, 0xbd, 0x73, 0xe9, 0x1e,
-	0x5c, 0x9e, 0x29, 0xd3, 0x76, 0x34, 0x18, 0xaa, 0x7e, 0xf8, 0x0f, 0xe5, 0x52, 0x2b, 0x23, 0x8e,
-	0x4d, 0xa1, 0x2a, 0x4c, 0x13, 0xf4, 0x0e, 0x5c, 0xea, 0x0a, 0x99, 0x2b, 0x92, 0xed, 0xb6, 0x16,
-	0xb8, 0x8f, 0xc5, 0xc1, 0x82, 0xe3, 0x2b, 0x11, 0xfd, 0x08, 0x1a, 0xcf, 0x86, 0x7d, 0x2e, 0xc5,
-	0x99, 0xac, 0x1f, 0xc0, 0xf2, 0xd3, 0x68, 0x18, 0x05, 0xd1, 0xee, 0xf8, 0x84, 0x91, 0x6f, 0xc0,
-	0x92, 0xde, 0x8f, 0xfa, 0x91, 0x52, 0x61, 0x96, 0xa4, 0x17, 0x54, 0x43, 0xf7, 0x78, 0xd0, 0x1b,
-	0x05, 0x0a, 0x86, 0xfa, 0xf7, 0x4a, 0x1e, 0xd4, 0x7f, 0x3b, 0x6a, 0x3a, 0xbf, 0x1f, 0x35, 0x9d,
-	0x3f, 0x8f, 0x9a, 0xce, 0xcf, 0x7f, 0x35, 0xdf, 0x78, 0x51, 0xc6, 0xff, 0xf4, 0xdb, 0xff, 0x06,
-	0x00, 0x00, 0xff, 0xff, 0xda, 0x68, 0xc4, 0x54, 0xb8, 0x0f, 0x00, 0x00,
+	// 1307 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4d, 0x6f, 0x1b, 0xc5,
+	0x1b, 0xff, 0xaf, 0xd7, 0x76, 0xe2, 0xc7, 0x71, 0xe2, 0x4c, 0xd3, 0xfc, 0x37, 0x55, 0xe4, 0x9a,
+	0x51, 0xa1, 0xa6, 0x12, 0x51, 0x49, 0x25, 0x44, 0x03, 0x95, 0x4a, 0x62, 0x57, 0x5d, 0x20, 0x51,
+	0x19, 0x27, 0x41, 0x42, 0x02, 0x69, 0x62, 0x0f, 0xe9, 0x2a, 0xeb, 0x5d, 0xb3, 0x3b, 0x4e, 0xe2,
+	0x1e, 0x38, 0x22, 0x24, 0xc4, 0x1d, 0x71, 0xe5, 0xcb, 0x70, 0xe4, 0x13, 0x20, 0x14, 0x3e, 0x04,
+	0x47, 0xd0, 0xbc, 0xed, 0xae, 0xdf, 0x92, 0x26, 0x70, 0xdb, 0xe7, 0xfd, 0x37, 0xcf, 0xdb, 0xcc,
+	0x42, 0xa5, 0x1f, 0x79, 0xa7, 0x94, 0xb3, 0x8d, 0x7e, 0x14, 0xf2, 0x10, 0xcd, 0x7b, 0x01, 0x67,
+	0x51, 0x40, 0x7d, 0x5c, 0x86, 0x92, 0x1b, 0x74, 0xd9, 0xf9, 0x2e, 0xe3, 0x14, 0xff, 0x6e, 0x41,
+	0xe9, 0x59, 0x44, 0x7b, 0x4c, 0x50, 0xe8, 0x2d, 0x58, 0x74, 0x83, 0x53, 0x16, 0xc5, 0xac, 0x15,
+	0xd0, 0x23, 0x9f, 0x75, 0x9d, 0x5c, 0xdd, 0x6a, 0xcc, 0x93, 0x31, 0x2e, 0x5a, 0x87, 0xd2, 0x0e,
+	0xed, 0xbc, 0x64, 0xfb, 0xc3, 0x3e, 0x73, 0xec, 0xba, 0xd5, 0x28, 0x91, 0x94, 0x91, 0x48, 0xdb,
+	0xde, 0x2b, 0xe6, 0xe4, 0xeb, 0x56, 0xa3, 0x42, 0x52, 0x06, 0xaa, 0x43, 0x79, 0xdf, 0xeb, 0xb1,
+	0xcf, 0x06, 0x34, 0xe0, 0x83, 0x9e, 0x53, 0x90, 0xd6, 0x59, 0x16, 0xc2, 0xb0, 0x40, 0x68, 0x70,
+	0x9c, 0x60, 0x28, 0x4a, 0x0c, 0x23, 0x3c, 0x74, 0x1f, 0x8a, 0xcf, 0x3c, 0xe6, 0x77, 0x63, 0x67,
+	0xae, 0x6e, 0x37, 0xca, 0x9b, 0x4b, 0x1b, 0xe6, 0x7c, 0x1b, 0x92, 0x4f, 0xb4, 0x18, 0x63, 0x58,
+	0x74, 0x7b, 0xfd, 0x30, 0xe2, 0x84, 0xc5, 0xfd, 0x30, 0x88, 0x19, 0xaa, 0x82, 0xdd, 0x8a, 0x22,
+	0xc7, 0x92, 0x81, 0xc5, 0x27, 0xfe, 0x16, 0xaa, 0xdb, 0x7e, 0xd8, 0x39, 0x69, 0x52, 0x4e, 0x09,
+	0xfb, 0x66, 0xc0, 0x62, 0x8e, 0x56, 0xa0, 0x20, 0xb3, 0xa4, 0xf5, 0x14, 0x21, 0xb8, 0x32, 0x5b,
+	0x32, 0x2f, 0x25, 0xa2, 0x08, 0xc1, 0x95, 0xf6, 0x32, 0x15, 0x79, 0xa2, 0x08, 0xc1, 0x6d, 0xfb,
+	0x5e, 0x47, 0xa5, 0x20, 0x4f, 0x14, 0x81, 0x10, 0xe4, 0x0f, 0x3d, 0x76, 0xa6, 0xcf, 0x2d, 0xbf,
+	0xb1, 0x0b, 0xcb, 0x99, 0xf8, 0x1a, 0xe6, 0x2a, 0x14, 0x49, 0x78, 0xe6, 0x36, 0x63, 0xc7, 0xaa,
+	0xdb, 0x8d, 0x3c, 0xd1, 0x94, 0xcc, 0x6e, 0xe8, 0x0f, 0x7a, 0x81, 0x10, 0xe5, 0xa4, 0x28, 0x65,
+	0xe0, 0x35, 0x28, 0xc8, 0x54, 0x8b, 0x53, 0xa6, 0xb6, 0xe2, 0x13, 0xff, 0x6d, 0x41, 0x69, 0x97,
+	0x9e, 0x4b, 0x18, 0x31, 0x7a, 0x02, 0xf3, 0x6d, 0x4e, 0x83, 0x2e, 0x8d, 0xba, 0x52, 0xa9, 0xbc,
+	0xf9, 0x46, 0x9a, 0xc2, 0x44, 0x6d, 0xc3, 0xe8, 0xb4, 0x02, 0x1e, 0x0d, 0x49, 0x62, 0x82, 0xb6,
+	0x60, 0x4e, 0xf7, 0x84, 0xc4, 0x50, 0xde, 0xac, 0x4f, 0xb3, 0x4e, 0xda, 0x46, 0x18, 0x1b, 0x83,
+	0x3b, 0x1f, 0x40, 0x65, 0xc4, 0xad, 0xc0, 0x7a, 0xc2, 0x86, 0xa6, 0x22, 0x27, 0x6c, 0x28, 0x72,
+	0x77, 0x4a, 0xfd, 0x81, 0xca, 0x73, 0x9e, 0x28, 0x62, 0x2b, 0xf7, 0xbe, 0x75, 0x67, 0x0b, 0x16,
+	0xb2, 0x5e, 0xaf, 0x63, 0x8b, 0xbf, 0x02, 0xb4, 0x13, 0x31, 0xca, 0x99, 0x84, 0xb7, 0xcb, 0xe2,
+	0x98, 0x1e, 0xb3, 0xd9, 0x95, 0x56, 0xd5, 0xcb, 0x65, 0xab, 0xb7, 0x0e, 0x25, 0x37, 0x36, 0x07,
+	0xb7, 0x65, 0x5f, 0xa6, 0x0c, 0xfc, 0x00, 0x50, 0x93, 0xf9, 0x8c, 0x33, 0x3d, 0x5f, 0x97, 0xf8,
+	0xc7, 0x6d, 0x83, 0xe5, 0x6a, 0x5d, 0x74, 0x1f, 0xf2, 0x62, 0x3c, 0x25, 0x94, 0xf2, 0xe6, 0xad,
+	0x34, 0xd3, 0xc9, 0x1c, 0x13, 0xa9, 0x80, 0x3d, 0xe3, 0x54, 0x8f, 0xf4, 0x15, 0x07, 0x9c, 0xd2,
+	0xca, 0x26, 0x94, 0x3d, 0x1e, 0x2a, 0x59, 0x12, 0x3a, 0xd4, 0x53, 0x73, 0xd6, 0x9b, 0x86, 0xc2,
+	0xc7, 0x09, 0x58, 0x31, 0xa9, 0x37, 0x01, 0xfb, 0x26, 0x14, 0xa4, 0xad, 0x46, 0x3b, 0xb1, 0x03,
+	0x94, 0x14, 0x1f, 0x26, 0x50, 0x6f, 0x1a, 0x68, 0x25, 0x1b, 0xa8, 0x64, 0xfc, 0x7e, 0xa1, 0x75,
+	0xc5, 0x4c, 0xef, 0x09, 0x1b, 0xe5, 0x49, 0x7e, 0xcf, 0xae, 0xd9, 0x58, 0x22, 0x85, 0x6f, 0xb1,
+	0x04, 0x62, 0xc7, 0xae, 0xdb, 0xc2, 0xb7, 0x24, 0xf0, 0x23, 0x28, 0xb6, 0x3b, 0x2f, 0x59, 0x8f,
+	0xa2, 0xb7, 0xc5, 0xa4, 0x75, 0xd9, 0x39, 0x8b, 0xf5, 0x9c, 0x2e, 0x8d, 0xd5, 0x9f, 0x18, 0x39,
+	0xfe, 0xc1, 0xd2, 0x67, 0x9a, 0x81, 0xa8, 0x28, 0x63, 0xc7, 0x4e, 0x7e, 0x62, 0x65, 0x0a, 0x3e,
+	0xd1, 0x62, 0xd4, 0x82, 0xaa, 0x1b, 0xf4, 0x07, 0xbc, 0xc9, 0xbe, 0xf6, 0x02, 0x8f, 0x7b, 0x61,
+	0x10, 0x3b, 0x45, 0x69, 0xb2, 0x96, 0x0d, 0x3d, 0xa2, 0x41, 0x26, 0x4c, 0xf0, 0x77, 0x16, 0x2c,
+	0x8d, 0x31, 0xaf, 0xc0, 0x95, 0xbb, 0x1c, 0xd7, 0x7b, 0xc9, 0xce, 0xb7, 0xa5, 0x62, 0x6d, 0x26,
+	0x9a, 0xd1, 0x2b, 0xe0, 0x17, 0x0b, 0x56, 0xa6, 0x29, 0x4c, 0x45, 0x53, 0x03, 0x78, 0x11, 0x79,
+	0x3d, 0x1a, 0x0d, 0x3f, 0x61, 0x43, 0x7d, 0xfd, 0x65, 0x38, 0xe8, 0x73, 0x58, 0x1d, 0xf3, 0xf5,
+	0x51, 0x47, 0xa5, 0x48, 0x81, 0xba, 0x3b, 0x13, 0x94, 0xd2, 0x23, 0x33, 0xcc, 0xf1, 0x5f, 0x16,
+	0xdc, 0x9e, 0x2a, 0x4a, 0x7b, 0xd2, 0xca, 0xf6, 0xe4, 0x03, 0xa8, 0x1e, 0x8a, 0xcd, 0xd6, 0x64,
+	0x31, 0xf7, 0x02, 0x2a, 0x34, 0x75, 0xd3, 0x4e, 0xf0, 0x91, 0x0b, 0xf3, 0x92, 0xb7, 0x4b, 0xfb,
+	0x1a, 0xe6, 0x3b, 0x57, 0xc0, 0xdc, 0x30, 0xfa, 0x7a, 0xf1, 0x1b, 0x52, 0x80, 0x91, 0x17, 0x91,
+	0xb9, 0xd5, 0x24, 0x21, 0x56, 0xfa, 0x88, 0xc1, 0xb5, 0xd6, 0x72, 0x08, 0xeb, 0x66, 0x15, 0x8e,
+	0x20, 0xb9, 0x7c, 0x52, 0x1f, 0x03, 0xa4, 0xaa, 0x7a, 0x03, 0x5c, 0xd2, 0x9f, 0x19, 0x65, 0xfc,
+	0x1c, 0xd6, 0xcd, 0x9e, 0xbe, 0x46, 0x40, 0xd3, 0x2d, 0xb9, 0xb4, 0x5b, 0x70, 0x0b, 0xec, 0x03,
+	0xe2, 0x8a, 0xbb, 0x5a, 0x4e, 0xab, 0x29, 0x91, 0xa6, 0x84, 0xc9, 0xf3, 0x30, 0xe6, 0xc6, 0x44,
+	0x7c, 0x0b, 0xde, 0x8b, 0x30, 0xe2, 0x12, 0x71, 0x85, 0xc8, 0x6f, 0xfc, 0x25, 0xe4, 0xf7, 0xc2,
+	0x2e, 0x43, 0x8b, 0x90, 0x73, 0x9b, 0xda, 0x47, 0xce, 0x6d, 0xa2, 0xbb, 0xd2, 0xbd, 0xde, 0x21,
+	0x95, 0xf4, 0x70, 0x07, 0xc4, 0x25, 0x32, 0xf0, 0x3d, 0xa8, 0xb8, 0xf1, 0x4e, 0x18, 0x46, 0x5d,
+	0x51, 0xea, 0x30, 0xd2, 0x77, 0xd2, 0x28, 0x13, 0x3f, 0x85, 0xaa, 0x70, 0xdf, 0xe6, 0x94, 0x27,
+	0x9b, 0x7a, 0x15, 0x8a, 0x82, 0x97, 0x84, 0xd3, 0x94, 0xbc, 0xf7, 0x84, 0x9e, 0x59, 0x80, 0x92,
+	0xc0, 0x9f, 0x2a, 0x0f, 0xad, 0x53, 0x16, 0xf0, 0x4c, 0x96, 0x24, 0x2d, 0x1d, 0x54, 0x88, 0x22,
+	0x10, 0x56, 0x47, 0xd1, 0x98, 0x17, 0x53, 0xcc, 0x82, 0x4b, 0xa4, 0x0c, 0xff, 0x68, 0x01, 0x18,
+	0x40, 0x83, 0x38, 0x31, 0xb1, 0x66, 0x9b, 0xa0, 0x77, 0x33, 0x6f, 0x97, 0xc9, 0x9d, 0x9a, 0x88,
+	0x48, 0xe6, 0x85, 0xd3, 0x30, 0x2b, 0x54, 0x37, 0x47, 0x35, 0xd5, 0x57, 0x7c, 0x5d, 0x26, 0x71,
+	0x6d, 0x56, 0x76, 0xfc, 0x41, 0xcc, 0x59, 0xa4, 0x11, 0x89, 0x37, 0x96, 0x62, 0x24, 0xf9, 0x49,
+	0x19, 0xd3, 0x53, 0x84, 0xee, 0x41, 0x41, 0x20, 0x35, 0x7b, 0x60, 0xfc, 0x18, 0x4a, 0x88, 0xdb,
+	0xfa, 0x26, 0x99, 0xba, 0x7b, 0x10, 0xe4, 0xe5, 0x8b, 0x5a, 0xb7, 0x8b, 0x7c, 0x4c, 0x57, 0xc1,
+	0xde, 0xf5, 0x54, 0x7f, 0xdb, 0x44, 0x7c, 0x4a, 0x0e, 0x3d, 0x97, 0xf3, 0x27, 0x38, 0x54, 0xbc,
+	0x25, 0x96, 0xd5, 0x00, 0x89, 0xbb, 0xe3, 0x26, 0xf7, 0x9b, 0x79, 0x94, 0xda, 0x99, 0x47, 0x69,
+	0x1b, 0x96, 0xd5, 0x90, 0xfc, 0x97, 0x4e, 0x7f, 0xce, 0xc1, 0x32, 0x61, 0xb1, 0xf7, 0x8a, 0xb9,
+	0x41, 0xcc, 0xa3, 0x41, 0xb2, 0xe0, 0x3e, 0x0e, 0x8f, 0x74, 0xaa, 0x6d, 0xa2, 0x88, 0xd7, 0xe9,
+	0x24, 0xf4, 0x10, 0xca, 0xe3, 0xdd, 0x3f, 0xa9, 0x9a, 0x55, 0x41, 0x0f, 0x61, 0xae, 0x1d, 0x0e,
+	0xa2, 0x4e, 0x72, 0x0d, 0xae, 0xa6, 0xda, 0x0a, 0x99, 0x12, 0x13, 0xa3, 0x96, 0xe9, 0xa3, 0xc2,
+	0xe5, 0x7d, 0x84, 0x9e, 0x8c, 0xf5, 0x91, 0xfc, 0x73, 0x29, 0x6f, 0xfe, 0x3f, 0x35, 0x18, 0x11,
+	0x93, 0x51, 0x6d, 0xfc, 0xbd, 0x05, 0x0b, 0x59, 0x08, 0xaf, 0x35, 0x18, 0x49, 0x45, 0x72, 0x53,
+	0x2b, 0x62, 0x4f, 0xab, 0x48, 0x3e, 0xad, 0x48, 0xfa, 0xce, 0x2d, 0x64, 0xde, 0xb9, 0xf8, 0x04,
+	0xd6, 0x26, 0xca, 0xb4, 0x13, 0xf6, 0xfa, 0xa2, 0x1f, 0xfe, 0x45, 0xb9, 0xc4, 0xca, 0x88, 0x22,
+	0x5d, 0xa8, 0x12, 0x51, 0x04, 0x7e, 0x0c, 0xb7, 0xdb, 0x8c, 0x67, 0x8a, 0x64, 0xba, 0xad, 0x0e,
+	0xf6, 0x1e, 0x3b, 0x9b, 0x71, 0x7c, 0x21, 0xc2, 0x1f, 0x82, 0x73, 0xd0, 0xef, 0x52, 0xce, 0x6e,
+	0x64, 0xbd, 0x0d, 0xf3, 0xfb, 0x61, 0x3f, 0xf4, 0xc3, 0xe3, 0xe1, 0x15, 0x23, 0xef, 0xc0, 0x9c,
+	0xda, 0x8f, 0xea, 0x91, 0x52, 0x22, 0x86, 0xc4, 0xb7, 0x44, 0x43, 0x77, 0xa8, 0xdf, 0x19, 0xf8,
+	0x02, 0x86, 0xf8, 0xf7, 0x8a, 0xb7, 0xab, 0xbf, 0x5e, 0xd4, 0xac, 0xdf, 0x2e, 0x6a, 0xd6, 0x1f,
+	0x17, 0x35, 0xeb, 0xa7, 0x3f, 0x6b, 0xff, 0x3b, 0x2a, 0xca, 0xbf, 0xf0, 0x47, 0xff, 0x04, 0x00,
+	0x00, 0xff, 0xff, 0xc3, 0xb3, 0xdc, 0xe3, 0x96, 0x0f, 0x00, 0x00,
 }

internal/private.proto

@@ -3,7 +3,6 @@ syntax = "proto3";
 package internal;
 
 message IndexMeta {
-	string TimeQuantum = 2;
 }
 
 message FrameMeta {

pilosa.go

@@ -46,19 +46,16 @@ var (
 	ErrInputDefinitionActionRequired = errors.New("field definitions require an action")
 	ErrInputDefinitionNotFound       = errors.New("input-definition not found")
 
-	ErrFieldNotFound          = errors.New("field not found")
-	ErrFieldExists            = errors.New("field already exists")
-	ErrFieldNameRequired      = errors.New("field name required")
-	ErrInvalidFieldType       = errors.New("invalid field type")
-	ErrInvalidFieldRange      = errors.New("invalid field range")
-	ErrInverseRangeNotAllowed = errors.New("inverse range not allowed")
-	ErrRangeCacheNotAllowed   = errors.New("range cache not allowed")
-	ErrFrameFieldsNotAllowed  = errors.New("frame fields not allowed")
-	ErrInvalidFieldValueType  = errors.New("invalid field value type")
-	ErrFieldValueTooLow       = errors.New("field value too low")
-	ErrFieldValueTooHigh      = errors.New("field value too high")
-	ErrInvalidRangeOperation  = errors.New("invalid range operation")
-	ErrInvalidBetweenValue    = errors.New("invalid value for between operation")
+	ErrFieldNotFound         = errors.New("field not found")
+	ErrFieldExists           = errors.New("field already exists")
+	ErrFieldNameRequired     = errors.New("field name required")
+	ErrInvalidFieldType      = errors.New("invalid field type")
+	ErrInvalidFieldRange     = errors.New("invalid field range")
+	ErrInvalidFieldValueType = errors.New("invalid field value type")
+	ErrFieldValueTooLow      = errors.New("field value too low")
+	ErrFieldValueTooHigh     = errors.New("field value too high")
+	ErrInvalidRangeOperation = errors.New("invalid range operation")
+	ErrInvalidBetweenValue   = errors.New("invalid value for between operation")
 
 	ErrInvalidView      = errors.New("invalid view")
 	ErrInvalidCacheType = errors.New("invalid cache type")
@@ -71,10 +68,6 @@ var (
 	ErrQueryRequired    = errors.New("query required")
 	ErrTooManyWrites    = errors.New("too many write commands")
 
-	ErrConfigClusterEnabledHosts = errors.New("providing hosts to a non-disabled cluster is not allowed")
-	ErrConfigClusterTypeInvalid  = errors.New("invalid cluster type")
-	ErrConfigHostsMissing        = errors.New("missing bind address in cluster hosts")
-
 	ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice")
 
 	ErrNodeIDNotExists    = errors.New("node with provided ID does not exist")
@@ -82,6 +75,12 @@ var (
 	ErrResizeNotRunning   = errors.New("no resize job currently running")
 )
 
+// ApiMethodNotAllowedError wraps an error value indicating that a particular
+// API method is not allowed in the current cluster state.
+type ApiMethodNotAllowedError struct {
+	error
+}
+
 // BadRequestError wraps an error value to signify that a request could not be
 // read, decoded, or parsed such that in an HTTP scenario, http.StatusBadRequest
 // would be returned.

roaring/roaring.go

@@ -1674,19 +1674,6 @@ func (c *container) clone() *container {
 	return other
 }
 
-// flipBitmap returns a new bitmap containter containing the inverse of all
-// bits in c.
-func (c *container) flipBitmap() *container {
-	other := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
-
-	for i, bitmap := range c.bitmap {
-		other.bitmap[i] = ^bitmap
-	}
-
-	other.n = other.count()
-	return other
-}
-
 // WriteTo writes c to w.
 func (c *container) WriteTo(w io.Writer) (n int64, err error) {
 	if c.isArray() {
@@ -1812,6 +1799,43 @@ type ContainerInfo struct {
 	Pointer unsafe.Pointer // offset within the mmap
 }
 
+// flip returns a new container containing the inverse of all
+// bits in a.
+func flip(a *container) *container {
+	if a.isArray() {
+		return flipArray(a)
+	} else if a.isRun() {
+		return flipRun(a)
+	} else {
+		return flipBitmap(a)
+	}
+}
+
+func flipArray(b *container) *container {
+	// TODO: actually implement this
+	x := b.clone()
+	x.arrayToBitmap()
+	return flipBitmap(x)
+}
+
+func flipBitmap(b *container) *container {
+	other := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
+
+	for i, bitmap := range b.bitmap {
+		other.bitmap[i] = ^bitmap
+	}
+
+	other.n = other.count()
+	return other
+}
+
+func flipRun(b *container) *container {
+	// TODO: actually implement this
+	x := b.clone()
+	x.runToBitmap()
+	return flipBitmap(x)
+}
+
 func intersectionCount(a, b *container) int {
 	if a.isArray() {
 		if b.isArray() {
@@ -2571,7 +2595,7 @@ RUNLOOP:
 func differenceRunBitmap(a, b *container) *container {
 	// If a is full, difference is the flip of b.
 	if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 {
-		return b.flipBitmap()
+		return flipBitmap(b)
 	}
 	output := &container{containerType: ContainerRun}
 	output.n = a.n

roaring/roaring_helpers_test.go

@@ -220,8 +220,11 @@ func runEvenBitsSet() []interval16 {
 
 ///////////////////////////////////////////////////////////////////////////
 
+// f is a container function taking either one or two containers as input
+// func(a *container) *container
+// func(a, b *container) *container
 type testOp struct {
-	f   func(a, b *container) *container
+	f   interface{}
 	x   string
 	y   string
 	exp string

roaring/roaring_internal_test.go

@@ -2020,38 +2020,6 @@ func TestXorRunRun(t *testing.T) {
 	}
 }
 
-func TestBitmapFlip(t *testing.T) {
-	c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
-
-	ttable := []struct {
-		original uint64
-		flipped  uint64
-	}{
-		{0x0000000000000000, 0xFFFFFFFFFFFFFFFF},
-		{0xFFFFFFFFFFFFFFFF, 0x0000000000000000},
-		{0xFFFFFFFFFFFFFFF0, 0x000000000000000F},
-		{0xFFFFFFEFFFFFFFFF, 0x0000001000000000},
-		{0x0000001000000000, 0xFFFFFFEFFFFFFFFF},
-	}
-
-	expectedN := int(65536)
-	for i, tt := range ttable {
-		c.bitmap[i] = tt.original
-		expectedN -= int(popcount(tt.original))
-	}
-
-	o := c.flipBitmap()
-
-	for i, tt := range ttable {
-		if o.bitmap[i] != tt.flipped {
-			t.Fatalf("bitmapFlip calculation. expected %v, got %v", tt.flipped, o.bitmap[i])
-		}
-	}
-	if o.n != expectedN {
-		t.Fatalf("bitmapFlip calculation. expected count %v, got %v", expectedN, o.n)
-	}
-}
-
 func TestBitmapXorRange(t *testing.T) {
 	c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
 	tests := []struct {
@@ -3185,12 +3153,24 @@ func TestContainerCombinations(t *testing.T) {
 		//{xor, "evenBitsSet", "outerBitsSet", ""},
 		{xor, "evenBitsSet", "oddBitsSet", "full"},
 		{xor, "evenBitsSet", "evenBitsSet", "empty"},
+
+		// flip
+		{flip, "empty", "", "full"},
+		{flip, "full", "", "empty"},
+		{flip, "firstBitSet", "", "firstBitUnset"},
+		{flip, "lastBitSet", "", "lastBitUnset"},
+		{flip, "firstBitUnset", "", "firstBitSet"},
+		{flip, "lastBitUnset", "", "lastBitSet"},
+		{flip, "innerBitsSet", "", "outerBitsSet"},
+		{flip, "outerBitsSet", "", "innerBitsSet"},
+		{flip, "oddBitsSet", "", "evenBitsSet"},
+		{flip, "evenBitsSet", "", "oddBitsSet"},
 	}
 	for _, testOp := range testOps {
 		for _, x := range containerTypes {
 			for _, y := range containerTypes {
 				desc := fmt.Sprintf("%s(%s/%s, %s/%s)", getFunctionName(testOp.f), cm[x], testOp.x, cm[y], testOp.y)
-				ret := testOp.f(cts[x][testOp.x], cts[y][testOp.y])
+				ret := runContainerFunc(testOp.f, cts[x][testOp.x], cts[y][testOp.y])
 				exp := testOp.exp
 
 				// Convert to all container types and check result.
@@ -3240,3 +3220,14 @@ func TestContainerCombinations(t *testing.T) {
 		}
 	}
 }
+
+//func getFunc(func(a, b *container) *container, m, n *container) *container {
+func runContainerFunc(f interface{}, c ...*container) *container {
+	switch f.(type) {
+	case func(*container) *container:
+		return f.(func(*container) *container)(c[0])
+	case func(*container, *container) *container:
+		return f.(func(a, b *container) *container)(c[0], c[1])
+	}
+	return nil
+}

security_manager.go

@@ -1,32 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa
-
-// SecurityManager provides the ability to limit access to restricted endpoints
-// during cluster configuration.
-type SecurityManager interface {
-	SetRestricted()
-	SetNormal()
-}
-
-// NopSecurityManager provides a no-op implementation of the SecurityManager interface.
-type NopSecurityManager struct {
-}
-
-// SetRestricted no-op.
-func (sdm *NopSecurityManager) SetRestricted() {}
-
-// SetNormal no-op.
-func (sdm *NopSecurityManager) SetNormal() {}

server.go

@@ -16,8 +16,6 @@ package pilosa
 
 import (
 	"context"
-	"crypto/tls"
-	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -31,6 +29,7 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/internal"
+	"github.com/pkg/errors"
 
 	"golang.org/x/sync/errgroup"
 )
@@ -47,87 +46,205 @@ var _ StatusHandler = &Server{}
 
 // Server represents a holder wrapped by a running HTTP server.
 type Server struct {
-	ln net.Listener
-
 	// Close management.
 	wg      sync.WaitGroup
 	closing chan struct{}
 
-	// Data storage and HTTP interface.
-	Holder            *Holder
-	Handler           *Handler
+	// Internal
+	Holder      *Holder
+	Cluster     *Cluster
+	diagnostics *DiagnosticsCollector
+
+	// External
+	handler           *Handler
 	Broadcaster       Broadcaster
 	BroadcastReceiver BroadcastReceiver
 	Gossiper          Gossiper
-	RemoteClient      *http.Client
+	remoteClient      *http.Client
+	systemInfo        SystemInfo
+	gcNotifier        GCNotifier
+	NewAttrStore      func(string) AttrStore
+	logger            Logger
+	ln                net.Listener
 
-	// Cluster configuration.
-	Network     string
-	NodeID      string
-	URI         URI
-	Cluster     *Cluster
-	diagnostics *DiagnosticsCollector
-	SystemInfo  SystemInfo
-
-	GCNotifier GCNotifier
-
-	NewAttrStore func(string) AttrStore
-
-	// Background monitoring intervals.
-	AntiEntropyInterval time.Duration
-	MetricInterval      time.Duration
-	DiagnosticInterval  time.Duration
-
-	// TLS configuration
-	TLS *tls.Config
-
-	// Misc options.
-	MaxWritesPerRequest int
-
-	Logger Logger
+	NodeID              string
+	URI                 URI
+	antiEntropyInterval time.Duration
+	metricInterval      time.Duration
+	diagnosticInterval  time.Duration
+	maxWritesPerRequest int
 
 	defaultClient InternalClient
 }
 
-// NewServer returns a new instance of Server.
-func NewServer() *Server {
-	s := &Server{
-		closing: make(chan struct{}),
+// ServerOption is a functional option type for pilosa.Server
+type ServerOption func(s *Server) error
 
+func OptServerLogger(l Logger) ServerOption {
+	return func(s *Server) error {
+		s.logger = l
+		return nil
+	}
+}
+
+func OptServerReplicaN(n int) ServerOption {
+	return func(s *Server) error {
+		s.Cluster.ReplicaN = n
+		return nil
+	}
+}
+
+func OptServerDataDir(dir string) ServerOption {
+	return func(s *Server) error {
+		s.Cluster.Path = dir
+		s.Holder.Path = dir
+		return nil
+	}
+}
+
+func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption {
+	return func(s *Server) error {
+		s.NewAttrStore = af
+		s.Holder.NewAttrStore = af
+		return nil
+	}
+}
+
+func OptServerAntiEntropyInterval(interval time.Duration) ServerOption {
+	return func(s *Server) error {
+		s.antiEntropyInterval = interval
+		return nil
+	}
+}
+
+func OptServerLongQueryTime(dur time.Duration) ServerOption {
+	return func(s *Server) error {
+		s.Cluster.LongQueryTime = dur
+		return nil
+	}
+}
+
+func OptServerHandler(h *Handler) ServerOption {
+	return func(s *Server) error {
+		s.handler = h
+		return nil
+	}
+}
+
+func OptServerMaxWritesPerRequest(n int) ServerOption {
+	return func(s *Server) error {
+		s.maxWritesPerRequest = n
+		return nil
+	}
+}
+
+func OptServerMetricInterval(dur time.Duration) ServerOption {
+	return func(s *Server) error {
+		s.metricInterval = dur
+		return nil
+	}
+}
+
+func OptServerSystemInfo(si SystemInfo) ServerOption {
+	return func(s *Server) error {
+		s.systemInfo = si
+		return nil
+	}
+}
+
+func OptServerGCNotifier(gcn GCNotifier) ServerOption {
+	return func(s *Server) error {
+		s.gcNotifier = gcn
+		return nil
+	}
+}
+
+func OptServerRemoteClient(c *http.Client) ServerOption {
+	return func(s *Server) error {
+		s.remoteClient = c
+		s.Cluster.RemoteClient = c
+		return nil
+	}
+}
+
+func OptServerStatsClient(sc StatsClient) ServerOption {
+	return func(s *Server) error {
+		s.Holder.Stats = sc
+		return nil
+	}
+}
+
+func OptServerDiagnosticsInterval(dur time.Duration) ServerOption {
+	return func(s *Server) error {
+		s.diagnosticInterval = dur
+		return nil
+	}
+}
+
+func OptServerListener(ln net.Listener) ServerOption {
+	return func(s *Server) error {
+		s.ln = ln
+
+		return nil
+	}
+}
+
+func OptServerURI(uri *URI) ServerOption {
+	return func(s *Server) error {
+		s.URI = *uri
+		return nil
+	}
+}
+
+// NewServer returns a new instance of Server.
+func NewServer(opts ...ServerOption) (*Server, error) {
+	s := &Server{
+		closing:           make(chan struct{}),
+		Cluster:           NewCluster(),
 		Holder:            NewHolder(),
-		Handler:           NewHandler(),
+		handler:           NewHandler(),
 		Broadcaster:       NopBroadcaster,
 		BroadcastReceiver: NopBroadcastReceiver,
 		diagnostics:       NewDiagnosticsCollector(DefaultDiagnosticServer),
-		SystemInfo:        NewNopSystemInfo(),
+		systemInfo:        NewNopSystemInfo(),
 
-		Network: "tcp",
-
-		GCNotifier: NopGCNotifier,
+		gcNotifier: NopGCNotifier,
 
 		NewAttrStore: NewNopAttrStore,
 
-		AntiEntropyInterval: time.Duration(NewConfig().AntiEntropy.Interval),
-		MetricInterval:      0,
-		DiagnosticInterval:  0,
+		antiEntropyInterval: time.Minute * 10,
+		metricInterval:      0,
+		diagnosticInterval:  0,
 
-		Logger: NopLogger,
+		logger: NopLogger,
 	}
 
-	s.Handler.API = NewAPI()
-	s.Handler.API.Holder = s.Holder
-	return s
+	for _, opt := range opts {
+		err := opt(s)
+		if err != nil {
+			return nil, errors.Wrap(err, "applying option")
+		}
+	}
+
+	s.Holder.Logger = s.logger
+	s.Holder.Stats.SetLogger(s.logger)
+
+	s.Cluster.Logger = s.logger
+	s.Cluster.Holder = s.Holder
+	s.Cluster.RemoteClient = s.remoteClient
+	// update URI port with actual listener port. TODO this should probably be done outside of here.
+	if s.URI.Port() == 0 {
+		s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port))
+	}
+	return s, nil
 }
 
 // Open opens and initializes the server.
 func (s *Server) Open() error {
-	s.Handler.API.Logger = s.Logger // TODO do this in NewServer with functional options
-	s.Logger.Printf("open server")
+	s.logger.Printf("open server")
 	// s.ln can be configured prior to Open() via s.OpenListener().
 	if s.ln == nil {
-		if err := s.OpenListener(); err != nil {
-			return err
-		}
+		return errors.New("Must pass a listener option to NewServer")
 	}
 
 	// Get or create NodeID.
@@ -150,38 +267,36 @@ func (s *Server) Open() error {
 	s.Holder.Peek()
 
 	// Create default HTTP client
-	s.createDefaultClient(s.RemoteClient)
+	s.createDefaultClient(s.remoteClient)
 
 	// Create executor for executing queries.
-	e := NewExecutor(s.RemoteClient)
+	e := NewExecutor(s.remoteClient)
 	e.Holder = s.Holder
 	e.Node = node
 	e.Cluster = s.Cluster
-	e.MaxWritesPerRequest = s.MaxWritesPerRequest
+	e.MaxWritesPerRequest = s.maxWritesPerRequest
 
 	// Cluster settings.
 	s.Cluster.Broadcaster = s.Broadcaster
-	s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest
+	s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest
 
 	// Initialize HTTP handler.
-	s.Handler.API.Broadcaster = s.Broadcaster
-	s.Handler.API.BroadcastHandler = s
-	s.Handler.API.StatusHandler = s
-	s.Handler.API.URI = s.URI
-	s.Handler.API.Cluster = s.Cluster
-	s.Handler.Executor = e
-
-	s.Cluster.prefect = s.Handler
-	s.Handler.API.Executor = e
+	s.handler.API.Holder = s.Holder
+	s.handler.API.Broadcaster = s.Broadcaster
+	s.handler.API.BroadcastHandler = s
+	s.handler.API.StatusHandler = s
+	s.handler.API.URI = s.URI
+	s.handler.API.Cluster = s.Cluster
+	s.handler.API.Executor = e
 
 	// Initialize Holder.
 	s.Holder.Broadcaster = s.Broadcaster
 
 	// Serve HTTP.
 	go func() {
-		err := http.Serve(s.ln, s.Handler)
+		err := http.Serve(s.ln, s.handler)
 		if err != nil {
-			s.Logger.Printf("HTTP handler terminated with error: %s\n", err)
+			s.logger.Printf("HTTP handler terminated with error: %s\n", err)
 		}
 	}()
 
@@ -219,43 +334,6 @@ func (s *Server) Open() error {
 	return nil
 }
 
-// OpenListener opens a listener for the Server.
-func (s *Server) OpenListener() error {
-	s.Logger.Printf("open server listener: %s", s.URI)
-	if s.ln != nil {
-		return fmt.Errorf("a listener already exists for server: %s", s.URI)
-	}
-
-	var ln net.Listener
-	var err error
-
-	// If bind URI has the https scheme, enable TLS
-	if s.URI.Scheme() == "https" && s.TLS != nil {
-		ln, err = tls.Listen("tcp", s.URI.HostPort(), s.TLS)
-		if err != nil {
-			return err
-		}
-	} else if s.URI.Scheme() == "http" {
-		// Open HTTP listener to determine port (if specified as :0).
-		ln, err = net.Listen(s.Network, s.URI.HostPort())
-		if err != nil {
-			return fmt.Errorf("net.Listen: %v", err)
-		}
-	} else {
-		return fmt.Errorf("unsupported scheme: %s", s.URI.Scheme())
-	}
-
-	s.ln = ln
-
-	if s.URI.Port() == 0 {
-		// If the port is 0, it is set automatically.
-		// Find out automatically set port and update the host.
-		s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port))
-	}
-
-	return nil
-}
-
 // Close closes the server and waits for it to shutdown.
 func (s *Server) Close() error {
 	// Notify goroutines to stop.
@@ -283,7 +361,7 @@ func (s *Server) LoadNodeID() string {
 	}
 	nodeID, err := s.Holder.loadNodeID()
 	if err != nil {
-		s.Logger.Printf("loading NodeID: %v", err)
+		s.logger.Printf("loading NodeID: %v", err)
 		return s.NodeID
 	}
 	return nodeID
@@ -296,31 +374,12 @@ func (s *Server) Addr() net.Addr {
 	}
 	return s.ln.Addr()
 }
-func GetHTTPClient(t *tls.Config) *http.Client {
-	transport := &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
-			DualStack: true,
-		}).DialContext,
-		MaxIdleConns:          1000,
-		MaxIdleConnsPerHost:   200,
-		IdleConnTimeout:       90 * time.Second,
-		TLSHandshakeTimeout:   10 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
-	}
-	if t != nil {
-		transport.TLSClientConfig = t
-	}
-	return &http.Client{Transport: transport}
-}
 
 func (s *Server) monitorAntiEntropy() {
-	ticker := time.NewTicker(s.AntiEntropyInterval)
+	ticker := time.NewTicker(s.antiEntropyInterval)
 	defer ticker.Stop()
 
-	s.Logger.Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval)
+	s.logger.Printf("holder sync monitor initializing (%s interval)", s.antiEntropyInterval)
 
 	for {
 		// Wait for tick or a close.
@@ -331,7 +390,7 @@ func (s *Server) monitorAntiEntropy() {
 			s.Holder.Stats.Count("AntiEntropy", 1, 1.0)
 		}
 		t := time.Now()
-		s.Logger.Printf("holder sync beginning")
+		s.logger.Printf("holder sync beginning")
 
 		// Initialize syncer with local holder and remote client.
 		var syncer HolderSyncer
@@ -339,17 +398,17 @@ func (s *Server) monitorAntiEntropy() {
 		syncer.Node = s.Cluster.Node
 		syncer.Cluster = s.Cluster
 		syncer.Closing = s.closing
-		syncer.RemoteClient = s.RemoteClient
+		syncer.RemoteClient = s.remoteClient
 		syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer")
 
 		// Sync holders.
 		if err := syncer.SyncHolder(); err != nil {
-			s.Logger.Printf("holder sync error: err=%s", err)
+			s.logger.Printf("holder sync error: err=%s", err)
 			continue
 		}
 
 		// Record successful sync in log.
-		s.Logger.Printf("holder sync complete")
+		s.logger.Printf("holder sync complete")
 		dif := time.Since(t)
 		s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0)
 	}
@@ -369,9 +428,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
 			idx.SetRemoteMaxSlice(obj.Slice)
 		}
 	case *internal.CreateIndexMessage:
-		opt := IndexOptions{
-			TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum),
-		}
+		opt := IndexOptions{}
 		_, err := s.Holder.CreateIndex(obj.Index, opt)
 		if err != nil {
 			return err
@@ -473,7 +530,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
 func (s *Server) SendSync(pb proto.Message) error {
 	var eg errgroup.Group
 	for _, node := range s.Cluster.Nodes {
-		s.Logger.Printf("SendSync to: %s", node.URI)
+		s.logger.Printf("SendSync to: %s", node.URI)
 		// Don't forward the message to ourselves.
 		if s.URI == node.URI {
 			continue
@@ -495,7 +552,7 @@ func (s *Server) SendAsync(pb proto.Message) error {
 
 // SendTo represents an implementation of Broadcaster.
 func (s *Server) SendTo(to *Node, pb proto.Message) error {
-	s.Logger.Printf("SendTo: %s", to.URI)
+	s.logger.Printf("SendTo: %s", to.URI)
 	ctx := context.WithValue(context.Background(), "uri", &to.URI)
 	return s.defaultClient.SendMessage(ctx, pb)
 }
@@ -545,7 +602,7 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error {
 
 		err := s.mergeRemoteStatus(pb.(*internal.NodeStatus))
 		if err != nil {
-			s.Logger.Printf("merge remote status: %s", err)
+			s.logger.Printf("merge remote status: %s", err)
 		}
 	}()
 
@@ -570,7 +627,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
 		// if we don't know about an index locally, log an error because
 		// indexes should be created and synced prior to slice creation
 		if localIndex == nil {
-			s.Logger.Printf("Local Index not found: %s", index)
+			s.logger.Printf("Local Index not found: %s", index)
 			continue
 		}
 		if newMax > oldmaxslices[index] {
@@ -586,7 +643,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
 		// if we don't know about an index locally, log an error because
 		// indexes should be created and synced prior to slice creation
 		if localIndex == nil {
-			s.Logger.Printf("Local Index not found: %s", index)
+			s.logger.Printf("Local Index not found: %s", index)
 			continue
 		}
 		if newMaxInverse > oldMaxInverseSlices[index] {
@@ -601,14 +658,14 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
 // monitorDiagnostics periodically polls the Pilosa Indexes for cluster info.
 func (s *Server) monitorDiagnostics() {
 	// Do not send more than once a minute
-	if s.DiagnosticInterval < time.Minute {
-		s.Logger.Printf("diagnostics disabled")
+	if s.diagnosticInterval < time.Minute {
+		s.logger.Printf("diagnostics disabled")
 		return
 	} else {
-		s.Logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.DiagnosticInterval)
+		s.logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.diagnosticInterval)
 	}
 
-	s.diagnostics.Logger = s.Logger
+	s.diagnostics.Logger = s.logger
 	s.diagnostics.SetVersion(Version)
 	s.diagnostics.Set("Host", s.URI.host)
 	s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ","))
@@ -630,11 +687,11 @@ func (s *Server) monitorDiagnostics() {
 		s.diagnostics.CheckVersion()
 		err = s.diagnostics.Flush()
 		if err != nil {
-			s.Logger.Printf("Diagnostics error: %s", err)
+			s.logger.Printf("Diagnostics error: %s", err)
 		}
 	}
 
-	ticker := time.NewTicker(s.DiagnosticInterval)
+	ticker := time.NewTicker(s.diagnosticInterval)
 	defer ticker.Stop()
 	flush()
 	for {
@@ -651,24 +708,24 @@ func (s *Server) monitorDiagnostics() {
 // monitorRuntime periodically polls the Go runtime metrics.
 func (s *Server) monitorRuntime() {
 	// Disable metrics when poll interval is zero.
-	if s.MetricInterval <= 0 {
+	if s.metricInterval <= 0 {
 		return
 	}
 
 	var m runtime.MemStats
-	ticker := time.NewTicker(s.MetricInterval)
+	ticker := time.NewTicker(s.metricInterval)
 	defer ticker.Stop()
 
-	defer s.GCNotifier.Close()
+	defer s.gcNotifier.Close()
 
-	s.Logger.Printf("runtime stats initializing (%s interval)", s.MetricInterval)
+	s.logger.Printf("runtime stats initializing (%s interval)", s.metricInterval)
 
 	for {
 		// Wait for tick or a close.
 		select {
 		case <-s.closing:
 			return
-		case <-s.GCNotifier.AfterGC():
+		case <-s.gcNotifier.AfterGC():
 			// GC just ran.
 			s.Holder.Stats.Count("garbage_collection", 1, 1.0)
 		case <-ticker.C:

server/cluster_test.go

@@ -26,80 +26,18 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/gossip"
 	"github.com/pilosa/pilosa/test"
 )
 
 // Ensure program can send/receive broadcast messages.
 func TestMain_SendReceiveMessage(t *testing.T) {
-
-	m0 := test.MustRunMain()
+	ms := test.MustRunMainWithCluster(t, 2)
+	m0, m1 := ms[0], ms[1]
 	defer m0.Close()
-
-	m1 := test.MustRunMain()
 	defer m1.Close()
 
-	// Update cluster config
-	m0.Server.Cluster.Nodes = []*pilosa.Node{
-		{ID: m0.Server.NodeID, URI: m0.Server.URI},
-		{ID: m1.Server.NodeID, URI: m1.Server.URI},
-	}
-	m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes
-
-	// Configure node0
-
-	// get the host portion of addr to use for binding
-	m0.Config.Gossip.Port = "0"
-	m0.Config.Gossip.Seeds = []string{}
-
-	m0.Server.Cluster.Coordinator = m0.Server.NodeID
-	m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}}
-	m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.Logger)
-	gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Config, m0.Server)
-	if err != nil {
-		t.Fatal(err)
-	}
-	m0.Server.Cluster.MemberSet = gossipMemberSet0
-	m0.Server.Broadcaster = m0.Server
-	m0.Server.Gossiper = gossipMemberSet0
-	m0.Server.Handler.API.Broadcaster = m0.Server.Broadcaster
-	m0.Server.Holder.Broadcaster = m0.Server.Broadcaster
-	m0.Server.BroadcastReceiver = gossipMemberSet0
-
-	if err := m0.Server.BroadcastReceiver.Start(m0.Server); err != nil {
-		t.Fatal(err)
-	}
-	// Open Cluster management.
-	if err := m0.Server.Cluster.Open(); err != nil {
-		t.Fatal(err)
-	}
-
-	// Configure node1
-
-	// get the host portion of addr to use for binding
-	m1.Config.Gossip.Port = "0"
-	m1.Config.Gossip.Seeds = []string{gossipMemberSet0.GetBindAddr()}
-
-	m1.Server.Cluster.Coordinator = m0.Server.NodeID
-	m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.Logger)
-	gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Config, m1.Server)
-	if err != nil {
-		t.Fatal(err)
-	}
-	m1.Server.Cluster.MemberSet = gossipMemberSet1
-	m1.Server.Broadcaster = m1.Server
-	m1.Server.Gossiper = gossipMemberSet1
-	m1.Server.Handler.API.Broadcaster = m1.Server.Broadcaster
-	m1.Server.Holder.Broadcaster = m1.Server.Broadcaster
-	m1.Server.BroadcastReceiver = gossipMemberSet1
-
-	if err := m1.Server.BroadcastReceiver.Start(m1.Server); err != nil {
-		t.Fatal(err)
-	}
-	// Open Cluster management.
-	if err := m1.Server.Cluster.Open(); err != nil {
-		t.Fatal(err)
-	}
+	m0.Server.Cluster.SetState(pilosa.ClusterStateNormal)
+	m1.Server.Cluster.SetState(pilosa.ClusterStateNormal)
 
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 

server/config.go

@@ -0,0 +1,132 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"time"
+
+	"github.com/pilosa/pilosa/gossip"
+	"github.com/pilosa/pilosa/toml"
+)
+
+// Cluster types.
+const (
+	ClusterNone   = ""
+	ClusterStatic = "static"
+	ClusterGossip = "gossip"
+)
+
+// TLSConfig contains TLS configuration
+type TLSConfig struct {
+	// CertificatePath contains the path to the certificate (.crt or .pem file)
+	CertificatePath string `toml:"certificate-path"`
+	// CertificateKeyPath contains the path to the certificate key (.key file)
+	CertificateKeyPath string `toml:"certificate-key-path"`
+	// SkipVerify disables verification for self-signed certificates
+	SkipVerify bool `toml:"skip-verify"`
+}
+
+// Config represents the configuration for the command.
+type Config struct {
+	// DataDir is the directory where Pilosa stores both indexed data and
+	// running state such as cluster topology information.
+	DataDir string `toml:"data-dir"`
+	// Bind is the host:port on which Pilosa will listen.
+	Bind string `toml:"bind"`
+
+	// MaxWritesPerRequest limits the number of mutating commands that can be in
+	// a single request to the server. This includes SetBit, ClearBit,
+	// SetRowAttrs & SetColumnAttrs.
+	MaxWritesPerRequest int `toml:"max-writes-per-request"`
+
+	// LogPath configures where Pilosa will write logs.
+	LogPath string `toml:"log-path"`
+
+	// Verbose toggles verbose logging which can be useful for debugging.
+	Verbose bool `toml:"verbose"`
+
+	// TLS
+	TLS TLSConfig
+
+	Cluster struct {
+		// Disabled controls whether clustering functionality is enabled.
+		Disabled      bool          `toml:"disabled"`
+		Coordinator   bool          `toml:"coordinator"`
+		ReplicaN      int           `toml:"replicas"`
+		Hosts         []string      `toml:"hosts"`
+		LongQueryTime toml.Duration `toml:"long-query-time"`
+	} `toml:"cluster"`
+
+	// Gossip config is based around memberlist.Config.
+	Gossip gossip.Config `toml:"gossip"`
+
+	AntiEntropy struct {
+		Interval toml.Duration `toml:"interval"`
+	} `toml:"anti-entropy"`
+
+	Metric struct {
+		// Service can be statsd, expvar, or none.
+		Service string `toml:"service"`
+		// Host tells the statsd client where to write.
+		Host         string        `toml:"host"`
+		PollInterval toml.Duration `toml:"poll-interval"`
+		// Diagnostics toggles sending some limited diagnostic information to
+		// Pilosa's developers.
+		Diagnostics bool `toml:"diagnostics"`
+	} `toml:"metric"`
+}
+
+// NewConfig returns an instance of Config with default options.
+func NewConfig() *Config {
+	c := &Config{
+		DataDir:             "~/.pilosa",
+		Bind:                ":10101",
+		MaxWritesPerRequest: 5000,
+		// LogPath: "",
+		// Verbose: false,
+		TLS: TLSConfig{},
+	}
+
+	// Cluster config.
+	c.Cluster.Disabled = false
+	// c.Cluster.Coordinator = false
+	c.Cluster.ReplicaN = 1
+	c.Cluster.Hosts = []string{}
+	c.Cluster.LongQueryTime = toml.Duration(time.Minute)
+
+	// Gossip config.
+	c.Gossip.Port = "14000"
+	// c.Gossip.Seeds = []string{}
+	// c.Gossip.Key = ""
+	c.Gossip.StreamTimeout = toml.Duration(10 * time.Second)
+	c.Gossip.SuspicionMult = 4
+	c.Gossip.PushPullInterval = toml.Duration(30 * time.Second)
+	c.Gossip.ProbeInterval = toml.Duration(1 * time.Second)
+	c.Gossip.ProbeTimeout = toml.Duration(500 * time.Millisecond)
+	c.Gossip.Interval = toml.Duration(200 * time.Millisecond)
+	c.Gossip.Nodes = 3
+	c.Gossip.ToTheDeadTime = toml.Duration(30 * time.Second)
+
+	// AntiEntropy config.
+	c.AntiEntropy.Interval = toml.Duration(10 * time.Minute)
+
+	// Metric config.
+	c.Metric.Service = "none"
+	// c.Metric.Host = ""
+	c.Metric.PollInterval = toml.Duration(0 * time.Minute)
+	c.Metric.Diagnostics = true
+
+	return c
+}

server/config_test.go

@@ -12,34 +12,27 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package pilosa_test
+package server_test
 
 import (
 	"reflect"
 	"testing"
 	"time"
 
-	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/server"
+	"github.com/pilosa/pilosa/toml"
 )
 
 func Test_NewConfig(t *testing.T) {
-	c := pilosa.NewConfig()
+	c := server.NewConfig()
 
 	if c.Cluster.Disabled {
 		t.Fatalf("unexpected Cluster.Disabled: %v", c.Cluster.Disabled)
 	}
-
-	// Ensure that hosts can't be specificed on a non-disabled cluster.
-	c.Cluster.Hosts = []string{c.Bind, "localhost:10102"}
-
-	// Change cluster type from the default (gossip) to an invalid string.
-	if err := c.Validate(); err != pilosa.ErrConfigClusterEnabledHosts {
-		t.Fatal(err)
-	}
 }
 
 func TestDuration(t *testing.T) {
-	d := pilosa.Duration(time.Second * 182)
+	d := toml.Duration(time.Second * 182)
 	if d.String() != "3m2s" {
 		t.Fatalf("Unexpected time Duration %s", d)
 	}

server/server.go

@@ -24,10 +24,14 @@ import (
 	"io"
 	"log"
 	"math/rand"
+	"net"
+	"net/http"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"crypto/tls"
@@ -46,21 +50,17 @@ func init() {
 	rand.Seed(time.Now().UTC().UnixNano())
 }
 
-const (
-	// DefaultDataDir is the default data directory.
-	DefaultDataDir = "~/.pilosa"
-)
+type loggerLogger interface {
+	pilosa.Logger
+	Logger() *log.Logger
+}
 
 // Command represents the state of the pilosa server command.
 type Command struct {
 	Server *pilosa.Server
 
 	// Configuration.
-	Config *pilosa.Config
-
-	// Profiling options.
-	CPUProfile string
-	CPUTime    time.Duration
+	Config *Config
 
 	// Gossip transport
 	GossipTransport *gossip.Transport
@@ -75,14 +75,13 @@ type Command struct {
 
 	// Passed to the Gossip implementation.
 	logOutput io.Writer
-	logger    *log.Logger
+	logger    loggerLogger
 }
 
 // NewCommand returns a new instance of Main.
 func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command {
 	return &Command{
-		Server: pilosa.NewServer(),
-		Config: pilosa.NewConfig(),
+		Config: NewConfig(),
 
 		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
 
@@ -91,8 +90,8 @@ func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command {
 	}
 }
 
-// Run executes the pilosa server.
-func (m *Command) Run(args ...string) (err error) {
+// Start starts the pilosa server - it returns once the server is running.
+func (m *Command) Start() (err error) {
 	defer close(m.Started)
 	prefix := "~" + string(filepath.Separator)
 	if strings.HasPrefix(m.Config.DataDir, prefix) {
@@ -120,81 +119,66 @@ func (m *Command) Run(args ...string) (err error) {
 		return fmt.Errorf("server.Open: %v", err)
 	}
 
-	m.Server.Logger.Printf("Listening as %s\n", m.Server.URI)
+	m.logger.Printf("Listening as %s\n", m.Server.URI)
+
 	return nil
 }
 
-// SetupLogger sets up the logger based on the configuration.
-func (m *Command) SetupLogger() error {
+// Wait waits for the server to be closed or interrupted.
+func (m *Command) Wait() error {
+	// First SIGKILL causes server to shut down gracefully.
+	c := make(chan os.Signal, 2)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	select {
+	case sig := <-c:
+		m.logger.Printf("Received %s; gracefully shutting down...\n", sig.String())
+
+		// Second signal causes a hard shutdown.
+		go func() { <-c; os.Exit(1) }()
+		return errors.Wrap(m.Close(), "closing command")
+	case <-m.Done:
+		m.logger.Printf("Server closed externally")
+		return nil
+	}
+}
+
+// setupLogger sets up the logger based on the configuration.
+func (m *Command) setupLogger() error {
 	var err error
 	if m.Config.LogPath == "" {
 		m.logOutput = m.Stderr
 	} else {
 		m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
 		if err != nil {
-			return err
+			return errors.Wrap(err, "opening file")
 		}
 	}
 
 	if m.Config.Verbose {
-		vbl := pilosa.NewVerboseLogger(m.logOutput)
-		m.logger = vbl.Logger()
-		m.Server.Logger = vbl
+		m.logger = pilosa.NewVerboseLogger(m.logOutput)
 	} else {
-		sl := pilosa.NewStandardLogger(m.logOutput)
-		m.logger = sl.Logger()
-		m.Server.Logger = sl
+		m.logger = pilosa.NewStandardLogger(m.logOutput)
 	}
 	return nil
 }
 
 // SetupServer uses the cluster configuration to set up this server.
 func (m *Command) SetupServer() error {
-	err := m.Config.Validate()
+	err := m.setupLogger()
 	if err != nil {
-		return err
+		return errors.Wrap(err, "setting up logger")
 	}
 
-	m.Server.Handler.Logger = m.Server.Logger
-	m.Server.Holder.Logger = m.Server.Logger
-	m.Server.Holder.Stats.SetLogger(m.Server.Logger)
+	handler := pilosa.NewHandler()
+	handler.Logger = m.logger
+	handler.FileSystem = &statik.FileSystem{}
+	handler.API = pilosa.NewAPI()
+	handler.API.Logger = m.logger
 
 	uri, err := pilosa.AddressWithDefaults(m.Config.Bind)
-
 	if err != nil {
-		return err
+		return errors.Wrap(err, "processing bind address")
 	}
-	m.Server.URI = *uri
-
-	cluster := pilosa.NewCluster()
-	cluster.ReplicaN = m.Config.Cluster.ReplicaN
-	cluster.Holder = m.Server.Holder
-	cluster.Logger = m.Server.Logger
-
-	m.Server.Cluster = cluster
-
-	// Configure data directory (for Cluster .topology)
-	m.Server.Cluster.Path = m.Config.DataDir
-
-	m.Server.NewAttrStore = boltdb.NewAttrStore
-	m.Server.Holder.NewAttrStore = boltdb.NewAttrStore
-
-	// Configure holder.
-	m.Server.Logger.Printf("Using data from: %s\n", m.Config.DataDir)
-	m.Server.Holder.Path = m.Config.DataDir
-	m.Server.MetricInterval = time.Duration(m.Config.Metric.PollInterval)
-	if m.Config.Metric.Diagnostics {
-		m.Server.DiagnosticInterval = time.Duration(DefaultDiagnosticsInterval)
-	}
-	m.Server.SystemInfo = gopsutil.NewSystemInfo()
-	m.Server.GCNotifier = gcnotify.NewActiveGCNotifier()
-	m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
-	if err != nil {
-		return err
-	}
-
-	// Copy configuration flags.
-	m.Server.MaxWritesPerRequest = m.Config.MaxWritesPerRequest
 
 	// Setup TLS
 	var TLSConfig *tls.Config
@@ -207,27 +191,73 @@ func (m *Command) SetupServer() error {
 		}
 		cert, err := tls.LoadX509KeyPair(m.Config.TLS.CertificatePath, m.Config.TLS.CertificateKeyPath)
 		if err != nil {
-			return err
+			return errors.Wrap(err, "load x509 key pair")
 		}
-		m.Server.TLS = &tls.Config{
+		TLSConfig = &tls.Config{
 			Certificates:       []tls.Certificate{cert},
 			InsecureSkipVerify: m.Config.TLS.SkipVerify,
 		}
-
-		TLSConfig = m.Server.TLS
 	}
-	c := pilosa.GetHTTPClient(TLSConfig)
-	m.Server.RemoteClient = c
-	m.Server.Handler.API.RemoteClient = c
-	m.Server.Cluster.RemoteClient = c
 
-	// Statik file system.
-	m.Server.Handler.FileSystem = &statik.FileSystem{}
+	diagnosticsInterval := time.Duration(0)
+	if m.Config.Metric.Diagnostics {
+		diagnosticsInterval = time.Duration(DefaultDiagnosticsInterval)
+	}
 
-	// Set configuration options.
-	m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval)
-	m.Server.Cluster.LongQueryTime = time.Duration(m.Config.Cluster.LongQueryTime)
-	return nil
+	statsClient, err := NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
+	if err != nil {
+		return errors.Wrap(err, "new stats client")
+	}
+
+	ln, err := getListener(*uri, TLSConfig)
+	if err != nil {
+		return errors.Wrap(err, "getting listener")
+	}
+
+	c := GetHTTPClient(TLSConfig)
+	handler.API.RemoteClient = c
+
+	m.Server, err = pilosa.NewServer(
+		pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)),
+		pilosa.OptServerLongQueryTime(time.Duration(m.Config.Cluster.LongQueryTime)),
+		pilosa.OptServerDataDir(m.Config.DataDir),
+		pilosa.OptServerReplicaN(m.Config.Cluster.ReplicaN),
+		pilosa.OptServerMaxWritesPerRequest(m.Config.MaxWritesPerRequest),
+		pilosa.OptServerMetricInterval(time.Duration(m.Config.Metric.PollInterval)),
+		pilosa.OptServerDiagnosticsInterval(diagnosticsInterval),
+
+		pilosa.OptServerLogger(m.logger),
+		pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore),
+		pilosa.OptServerHandler(handler),
+		pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()),
+		pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()),
+		pilosa.OptServerStatsClient(statsClient),
+		pilosa.OptServerListener(ln),
+		pilosa.OptServerURI(uri),
+		pilosa.OptServerRemoteClient(c),
+	)
+
+	return errors.Wrap(err, "new server")
+}
+
+func GetHTTPClient(t *tls.Config) *http.Client {
+	transport := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		MaxIdleConns:          1000,
+		MaxIdleConnsPerHost:   200,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+	if t != nil {
+		transport.TLSClientConfig = t
+	}
+	return &http.Client{Transport: transport}
 }
 
 // SetupNetworking sets up internode communication based on the configuration.
@@ -266,7 +296,7 @@ func (m *Command) SetupNetworking() error {
 	if m.GossipTransport != nil {
 		transport = m.GossipTransport
 	} else {
-		transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger)
+		transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger())
 		if err != nil {
 			return err
 		}
@@ -277,8 +307,9 @@ func (m *Command) SetupNetworking() error {
 		m.Server.Cluster.Coordinator = m.Server.NodeID
 	}
 
-	m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.Logger)
-	gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Config, m.Server, gossip.WithLogger(m.logger), gossip.WithTransport(transport))
+	gossipEventReceiver := gossip.NewGossipEventReceiver(m.logger)
+	m.Server.Cluster.EventReceiver = gossipEventReceiver
+	gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, gossipEventReceiver, m.Server, gossip.WithLogger(m.logger.Logger()), gossip.WithTransport(transport))
 	if err != nil {
 		return err
 	}
@@ -318,3 +349,24 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) {
 		return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].")
 	}
 }
+
+// getListener gets a net.Listener based on the config.
+func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) {
+	// If bind URI has the https scheme, enable TLS
+	if uri.Scheme() == "https" && tlsconf != nil {
+		ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf)
+		if err != nil {
+			return nil, errors.Wrap(err, "tls.Listener")
+		}
+	} else if uri.Scheme() == "http" {
+		// Open HTTP listener to determine port (if specified as :0).
+		ln, err = net.Listen("tcp", uri.HostPort())
+		if err != nil {
+			return nil, errors.Wrap(err, "net.Listen")
+		}
+	} else {
+		return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme())
+	}
+
+	return ln, nil
+}

server/server_test.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/server"
 	"github.com/pilosa/pilosa/test"
 )
 
@@ -44,7 +45,7 @@ func TestMain_Set_Quick(t *testing.T) {
 		defer m.Close()
 
 		// Create client.
-		client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
+		client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -321,11 +322,11 @@ func TestMain_FrameRestore(t *testing.T) {
 	defer m21.Close()
 
 	// Import from first cluster.
-	client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
+	client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), server.GetHTTPClient(nil))
 	if err != nil {
 		t.Fatal("new client:", err)
 	}
-	client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
+	client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), server.GetHTTPClient(nil))
 	if err != nil {
 		t.Fatal("new client:", err)
 	}
@@ -489,8 +490,8 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand {
 }
 
 // ParseConfig parses s into a Config.
-func ParseConfig(s string) (pilosa.Config, error) {
-	var c pilosa.Config
+func ParseConfig(s string) (server.Config, error) {
+	var c server.Config
 	_, err := toml.Decode(s, &c)
 	return c, err
 }

test/cluster.go

@@ -50,6 +50,7 @@ func NewCluster(n int) *pilosa.Cluster {
 
 	c.Node = c.Nodes[0]
 	c.Coordinator = c.Nodes[0].ID
+	c.SetState(pilosa.ClusterStateNormal)
 
 	return c
 }

test/executor.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/pql"
+	"github.com/pilosa/pilosa/server"
 )
 
 // Executor represents a test wrapper for pilosa.Executor.
@@ -30,7 +31,7 @@ type Executor struct {
 var remoteClient *http.Client
 
 func init() {
-	remoteClient = pilosa.GetHTTPClient(nil)
+	remoteClient = server.GetHTTPClient(nil)
 }
 
 // NewExecutor returns a new instance of Executor.

test/handler.go

@@ -47,8 +47,6 @@ func NewHandler() *Handler {
 	// Handler test messages can no-op.
 	h.API.Broadcaster = pilosa.NopBroadcaster
 
-	h.SetNormal()
-
 	return h
 }
 

test/pilosa.go

@@ -49,15 +49,13 @@ func NewMain() *Main {
 	}
 
 	m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)}
-	m.Server.Network = *Network
-	m.Server.NewAttrStore = NewAttrStore
-	m.Server.Holder.NewAttrStore = NewAttrStore
 	m.Config.DataDir = path
 	m.Config.Bind = "http://localhost:0"
 	m.Config.Cluster.Disabled = true
 	m.Command.Stdin = &m.Stdin
 	m.Command.Stdout = &m.Stdout
 	m.Command.Stderr = &m.Stderr
+	m.SetupServer()
 
 	if testing.Verbose() {
 		m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout)
@@ -101,6 +99,7 @@ func runMainWithCluster(size int) ([]*Main, error) {
 
 	for i := 0; i < size; i++ {
 		m := NewMainWithCluster(i == 0)
+		m.Config.Cluster.Disabled = false
 
 		gossipSeeds[i], err = m.RunWithTransport(gossipHost, gossipPort, gossipSeeds[:i])
 		if err != nil {
@@ -117,7 +116,7 @@ func runMainWithCluster(size int) ([]*Main, error) {
 func MustRunMain() *Main {
 	m := NewMain()
 	m.Config.Metric.Diagnostics = false // Disable diagnostics.
-	if err := m.Run(); err != nil {
+	if err := m.Start(); err != nil {
 		panic(err)
 	}
 	return m
@@ -136,15 +135,19 @@ func (m *Main) Reopen() error {
 	}
 
 	// Create new main with the same config.
-	config := m.Config
+	config := m.Command.Config
 	m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr)
-	m.Server.Network = *Network
+	m.Command.Config = config
+	err := m.SetupServer()
+	if err != nil {
+		return errors.Wrap(err, "setting up server")
+	}
+
 	m.Server.NewAttrStore = boltdb.NewAttrStore
 	m.Server.Holder.NewAttrStore = m.Server.NewAttrStore
-	m.Config = config
 
 	// Run new program.
-	if err := m.Run(); err != nil {
+	if err := m.Start(); err != nil {
 		return err
 	}
 	return nil
@@ -174,12 +177,6 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) (
 		return seed, err
 	}
 
-	// Open server listener.
-	err = m.Server.OpenListener()
-	if err != nil {
-		return seed, err
-	}
-
 	// Open gossip transport to use in SetupServer.
 	transport, err := gossip.NewTransport(host, bindPort, nil)
 	if err != nil {
@@ -221,7 +218,7 @@ func (m *Main) URL() string { return "http://" + m.Server.Addr().String() }
 
 // Client returns a client to connect to the program.
 func (m *Main) Client() *pilosa.InternalHTTPClient {
-	client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
+	client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil))
 	if err != nil {
 		panic(err)
 	}

test/test.go

@@ -13,10 +13,3 @@
 // limitations under the License.
 
 package test
-
-import "flag"
-
-// Test flags.
-var (
-	Network = flag.String("network", "tcp", "network name")
-)

toml/toml.go

@@ -0,0 +1,30 @@
+package toml
+
+import "time"
+
+// Duration is a TOML wrapper type for time.Duration.
+type Duration time.Duration
+
+// String returns the string representation of the duration.
+func (d Duration) String() string { return time.Duration(d).String() }
+
+// UnmarshalText parses a TOML value into a duration value.
+func (d *Duration) UnmarshalText(text []byte) error {
+	v, err := time.ParseDuration(string(text))
+	if err != nil {
+		return err
+	}
+
+	*d = Duration(v)
+	return nil
+}
+
+// MarshalText writes duration value in text format.
+func (d Duration) MarshalText() (text []byte, err error) {
+	return []byte(d.String()), nil
+}
+
+// MarshalTOML write duration into valid TOML.
+func (d Duration) MarshalTOML() ([]byte, error) {
+	return []byte(d.String()), nil
+}