Merge pull request #1472 from codysoyland/merge-develop-master-1.0.0

Merge develop into master for 1.0.0

.travis.yml

@@ -13,7 +13,7 @@ env:
     - GOARCH=amd64
     - GOARCH=amd64 ENTERPRISE=1
 install:
-  - make install-dep install-statik vendor generate-statik
+  - make install-dep vendor
 script:
   - make test
 before_deploy:
@@ -27,9 +27,16 @@ deploy:
       go: "1.10"
       condition: $GOARCH = amd64
 matrix:
+  # Excluding or allowing failures on non-primary matrix configurations due to long running times.
+  fast_finish: true
   allow_failures:
     - go: master
-  fast_finish: true
+    - go: 1.9
+  exclude:
+    - go: 1.9
+      env: GOARCH=386
+    - go: 1.9
+      env: GOARCH=amd64
 notifications:
   slack:
     secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI="

CHANGELOG.md

@@ -5,6 +5,50 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [v1.0.0] - 2018-07-05
+
+### Added
+
+- ID-Key Translation ([#1337](https://github.com/pilosa/pilosa/pull/1337))
+- Add CORS support to handler ([#1327](https://github.com/pilosa/pilosa/pull/1327))
+
+### Changed
+
+- HTTP handler updates ([#1408](https://github.com/pilosa/pilosa/pull/1408), [#1399](https://github.com/pilosa/pilosa/pull/1399), [#1441](https://github.com/pilosa/pilosa/pull/1441), [#1375](https://github.com/pilosa/pilosa/pull/1375), [#1433](https://github.com/pilosa/pilosa/pull/1433), [#1444](https://github.com/pilosa/pilosa/pull/1444), [#1388](https://github.com/pilosa/pilosa/pull/1388), [#1309](https://github.com/pilosa/pilosa/pull/1309), [#1302](https://github.com/pilosa/pilosa/pull/1302), [#1304](https://github.com/pilosa/pilosa/pull/1304))
+- Refactor/improve tests ([#1437](https://github.com/pilosa/pilosa/pull/1437), [#1434](https://github.com/pilosa/pilosa/pull/1434), [#1435](https://github.com/pilosa/pilosa/pull/1435), [#1425](https://github.com/pilosa/pilosa/pull/1425), [#1418](https://github.com/pilosa/pilosa/pull/1418), [#1419](https://github.com/pilosa/pilosa/pull/1419), [#1413](https://github.com/pilosa/pilosa/pull/1413), [#1394](https://github.com/pilosa/pilosa/pull/1394), [#1387](https://github.com/pilosa/pilosa/pull/1387), [#1386](https://github.com/pilosa/pilosa/pull/1386), [#1378](https://github.com/pilosa/pilosa/pull/1378), [#1364](https://github.com/pilosa/pilosa/pull/1364), [#1348](https://github.com/pilosa/pilosa/pull/1348), [#1340](https://github.com/pilosa/pilosa/pull/1340), [#1297](https://github.com/pilosa/pilosa/pull/1297))
+- Simplify inter-node communication ([#1428](https://github.com/pilosa/pilosa/pull/1428), [#1427](https://github.com/pilosa/pilosa/pull/1427), [#1412](https://github.com/pilosa/pilosa/pull/1412), [#1398](https://github.com/pilosa/pilosa/pull/1398), [#1391](https://github.com/pilosa/pilosa/pull/1391), [#1389](https://github.com/pilosa/pilosa/pull/1389))
+- Make gossip's interface to Pilosa the API struct ([#1452](https://github.com/pilosa/pilosa/pull/1452))
+- Rename slice to shard ([#1426](https://github.com/pilosa/pilosa/pull/1426))
+- Clearbit for time fields ([#1424](https://github.com/pilosa/pilosa/pull/1424))
+- Update docs ([#1390](https://github.com/pilosa/pilosa/pull/1390), [#1329](https://github.com/pilosa/pilosa/pull/1329), [#1305](https://github.com/pilosa/pilosa/pull/1305), [#1296](https://github.com/pilosa/pilosa/pull/1296))
+- Simplify server setup ([#1417](https://github.com/pilosa/pilosa/pull/1417), [#1393](https://github.com/pilosa/pilosa/pull/1393),[#1451](https://github.com/pilosa/pilosa/pull/1451))
+- Refactor API ([#1407](https://github.com/pilosa/pilosa/pull/1407))
+- Modify PQL ([#1382](https://github.com/pilosa/pilosa/pull/1382), [#1402](https://github.com/pilosa/pilosa/pull/1402), [#1354](https://github.com/pilosa/pilosa/pull/1354))
+- Rename "frame" to "field" ([#1395](https://github.com/pilosa/pilosa/pull/1395), [#1362](https://github.com/pilosa/pilosa/pull/1362), [#1360](https://github.com/pilosa/pilosa/pull/1360), [#1358](https://github.com/pilosa/pilosa/pull/1358), [#1357](https://github.com/pilosa/pilosa/pull/1357), [#1355](https://github.com/pilosa/pilosa/pull/1355))
+- Optimize count ([#1365](https://github.com/pilosa/pilosa/pull/1365))
+- Simplify bitmap max function ([#1333](https://github.com/pilosa/pilosa/pull/1333))
+- Rename "bit" to "column" for clarity ([#1326](https://github.com/pilosa/pilosa/pull/1326))
+- Rename pilosa.Bitmap to Row ([#1311](https://github.com/pilosa/pilosa/pull/1311))
+
+### Removed
+
+- Rename (unexport) many items to reduce public API footprint prior to 1.0 release ([#1470](https://github.com/pilosa/pilosa/pull/1470), [#1458](https://github.com/pilosa/pilosa/pull/1458), [#1450](https://github.com/pilosa/pilosa/pull/1450), [#1449](https://github.com/pilosa/pilosa/pull/1449), [#1448](https://github.com/pilosa/pilosa/pull/1448), [#1447](https://github.com/pilosa/pilosa/pull/1447), [#1446](https://github.com/pilosa/pilosa/pull/1446), [#1438](https://github.com/pilosa/pilosa/pull/1438), [#1443](https://github.com/pilosa/pilosa/pull/1443), [#1440](https://github.com/pilosa/pilosa/pull/1440), [#1439](https://github.com/pilosa/pilosa/pull/1439), [#1409](https://github.com/pilosa/pilosa/pull/1409), [#1392](https://github.com/pilosa/pilosa/pull/1392), [#1374](https://github.com/pilosa/pilosa/pull/1374), [#1372](https://github.com/pilosa/pilosa/pull/1372), [#1369](https://github.com/pilosa/pilosa/pull/1369), [#1367](https://github.com/pilosa/pilosa/pull/1367), [#1366](https://github.com/pilosa/pilosa/pull/1366), [#1351](https://github.com/pilosa/pilosa/pull/1351), [#1420](https://github.com/pilosa/pilosa/pull/1420), [#1416](https://github.com/pilosa/pilosa/pull/1416), [#1397](https://github.com/pilosa/pilosa/pull/1397))
+- Remove dead code ([#1432](https://github.com/pilosa/pilosa/pull/1432), [#1457](https://github.com/pilosa/pilosa/pull/1457), [#1421](https://github.com/pilosa/pilosa/pull/1421), [#1411](https://github.com/pilosa/pilosa/pull/1411), [#1377](https://github.com/pilosa/pilosa/pull/1377), [#1393](https://github.com/pilosa/pilosa/pull/1393))
+- Remove view argument from Field.SetBit and Field.ClearBit ([#1396](https://github.com/pilosa/pilosa/pull/1396))
+- Remove WebUI (now contained in a separate package) ([#1363](https://github.com/pilosa/pilosa/pull/1363))
+- Remove bench command ([#1347](https://github.com/pilosa/pilosa/pull/1347))
+- Remove "view" from API, handler, docs ([#1346](https://github.com/pilosa/pilosa/pull/1346))
+- Remove backup/restore stuff ([#1339](https://github.com/pilosa/pilosa/pull/1339), [#1341](https://github.com/pilosa/pilosa/pull/1341))
+- Remove inverse frame functionality ([#1335](https://github.com/pilosa/pilosa/pull/1335))
+- Remove rangeEnabled option ([#1332](https://github.com/pilosa/pilosa/pull/1332))
+
+### Fixed
+
+- Fix a few data races ([#1423](https://github.com/pilosa/pilosa/pull/1423))
+- Fix for crash while removing containers ([#1401](https://github.com/pilosa/pilosa/pull/1401))
+- Allow dashes in frame names ([#1415](https://github.com/pilosa/pilosa/pull/1415))
+- Fix generate-config command, use single toml lib ([#1350](https://github.com/pilosa/pilosa/pull/1350))
+
 ## [v0.10.0] - 2018-05-15
 
 This version contains 93 contribution from 8 contributors. There are 93 files changed, 4,495 insertions, and 5,392 deletions.

CONTRIBUTING.md

@@ -78,7 +78,7 @@ Pilosa includes a Makefile that automates several tasks:
     make install
     ```
 
-- Install build dependencies (dep, statik, and protoc):
+- Install build dependencies (dep and protoc):
 
     ```sh
     make install-build-deps
@@ -114,12 +114,6 @@ Pilosa includes a Makefile that automates several tasks:
     make release
     ```
 
-- Generate static assets for the WebUI:
-
-    ```sh
-    make generate-statik
-    ```
-
 - Regenerate protocol buffer files in `internal/`:
 
     ```sh

Dockerfile

@@ -3,7 +3,7 @@ FROM golang:1.10.2 as builder
 COPY . /go/src/github.com/pilosa/pilosa/
 
 RUN cd /go/src/github.com/pilosa/pilosa \
-    && CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a"
+    && CGO_ENABLED=0 make install-dep install FLAGS="-a"
 
 FROM scratch
 

Gopkg.lock

@@ -1,12 +1,6 @@
 # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
 
 
-[[projects]]
-  name = "github.com/BurntSushi/toml"
-  packages = ["."]
-  revision = "b26d9c308763d68093482582cea63d69be07a0f0"
-  version = "v0.3.0"
-
 [[projects]]
   branch = "master"
   name = "github.com/CAFxX/gcnotifier"
@@ -76,6 +70,18 @@
   packages = ["proto"]
   revision = "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9"
 
+[[projects]]
+  name = "github.com/google/go-cmp"
+  packages = [
+    "cmp",
+    "cmp/cmpopts",
+    "cmp/internal/diff",
+    "cmp/internal/function",
+    "cmp/internal/value"
+  ]
+  revision = "3af367b6b30c263d47e8895973edcca9a49cf029"
+  version = "v0.2.0"
+
 [[projects]]
   name = "github.com/gorilla/context"
   packages = ["."]
@@ -191,12 +197,6 @@
   revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
   version = "v0.8.0"
 
-[[projects]]
-  name = "github.com/rakyll/statik"
-  packages = ["fs"]
-  revision = "fd36b3595eb2ec8da4b8153b107f7ea08504899d"
-  version = "v0.1.1"
-
 [[projects]]
   name = "github.com/satori/go.uuid"
   packages = ["."]
@@ -212,14 +212,22 @@
 [[projects]]
   name = "github.com/shirou/gopsutil"
   packages = [
+    "cpu",
     "host",
     "internal/common",
     "mem",
+    "net",
     "process"
   ]
   revision = "bfe3c2e8f406bf352bc8df81f98c752224867349"
   version = "v2.17.11"
 
+[[projects]]
+  branch = "master"
+  name = "github.com/shirou/w32"
+  packages = ["."]
+  revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b"
+
 [[projects]]
   branch = "master"
   name = "github.com/spf13/afero"
@@ -302,6 +310,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "8f633d73d966ca439d2fdf3704a41d8ea59be8ed9a2cab0ab73de4b72c5772ba"
+  inputs-digest = "da6d02118ca77527c4ff00e9522880032fc052fb39bc8efe6c76602857c8c84e"
   solver-name = "gps-cdcl"
   solver-version = 1

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test
+.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-peg test
 
 CLONE_URL=github.com/pilosa/pilosa
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@@ -48,8 +48,8 @@ build: vendor
 # Create a single release build under the build directory
 release-build: vendor
 	$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1
-	cp NOTICE LICENSE README.md build/pilosa-$(VERSION_ID)
-	$(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID))
+	cp NOTICE README.md build/pilosa-$(VERSION_ID)
+	$(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID),cp LICENSE build/pilosa-$(VERSION_ID))
 	tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
 	@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
 
@@ -88,16 +88,15 @@ install: vendor
 generate-protoc: require-protoc require-protoc-gen-gofast
 	go generate github.com/pilosa/pilosa/internal
 
-# `go generate` statik assets (WebUI)
-generate-statik: require-statik
-	go generate github.com/pilosa/pilosa/statik
-
 # `go generate` stringers
 generate-stringer:
 	go generate github.com/pilosa/pilosa
 
+generate-pql: require-peg
+	cd pql && peg -inline pql.peg && cd ..
+
 # `go generate` all needed packages
-generate: generate-protoc generate-statik generate-stringer
+generate: generate-protoc generate-stringer generate-pql
 
 # Create Docker image from Dockerfile
 docker:
@@ -126,23 +125,20 @@ endef
 require-dep:
 	$(call require,dep)
 
-require-statik:
-	$(call require,statik)
-
 require-protoc-gen-gofast:
 	$(call require,protoc-gen-gofast)
 
 require-protoc:
 	$(call require,protoc)
 
-install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc install-stringer
+require-peg:
+	$(call require,peg)
+
+install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer install-peg
 
 install-dep:
 	go get -u github.com/golang/dep/cmd/dep
 
-install-statik:
-	go get -u github.com/rakyll/statik
-
 install-stringer:
 	go get -u golang.org/x/tools/cmd/stringer
 
@@ -151,3 +147,6 @@ install-protoc-gen-gofast:
 
 install-protoc:
 	@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
+
+install-peg:
+	go get github.com/pointlander/peg

NOTES

@@ -14,13 +14,13 @@
         │0000000000000000000000000000000000000000│
         │────────────────────────────────────────┤
     F  ▶│0000000000000000000000000000000000000000│
-    r  ││0000000000000000000000000000000000000000│
-    a  ││0000000000000000000000000000000000000000│
-    m  ││0000000000000000000000000000000000000000│
-    e  ▶│0000000000000000000000000000000000000000│
+    i  ││0000000000000000000000000000000000000000│
+    e  ││0000000000000000000000000000000000000000│
+    l  ││0000000000000000000000000000000000000000│
+    d  ▶│0000000000000000000000000000000000000000│
         └────────────────────────────────────────┘
                                      ▲───────────▲
-                                         Slice    
+                                         Shard    
 
 
-Fragment=intersection of frame & slice
+Fragment=intersection of field & shard

apimethod_string.go

@@ -4,9 +4,9 @@ package pilosa
 
 import "strconv"
 
-const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews"
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViews"
 
-var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 87, 101, 114, 126, 135, 155, 172, 188, 197, 211, 219, 235, 253, 261, 281, 294, 308, 323, 340, 353, 373, 381}
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 143, 159, 168, 182, 190, 206, 214, 234, 247, 261, 278, 291, 299}
 
 func (i apiMethod) String() string {
 	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

attr.go

@@ -24,10 +24,10 @@ import (
 
 // Attribute data type enum.
 const (
-	AttrTypeString = 1
-	AttrTypeInt    = 2
-	AttrTypeBool   = 3
-	AttrTypeFloat  = 4
+	attrTypeString = 1
+	attrTypeInt    = 2
+	attrTypeBool   = 3
+	attrTypeFloat  = 4
 )
 
 // AttrStore represents an interface for handling row/column attributes.
@@ -42,57 +42,39 @@ type AttrStore interface {
 	BlockData(i uint64) (map[uint64]map[string]interface{}, error)
 }
 
-func init() {
-	NopAttrStore = &nopAttrStore{}
-}
+// nopStore represents an AttrStore that doesn't do anything.
+var nopStore AttrStore = nopAttrStore{}
 
-// NopAttrStore represents an AttrStore that doesn't do anything.
-var NopAttrStore AttrStore
-
-func NewNopAttrStore(string) AttrStore {
-	return &nopAttrStore{}
-}
+// newNopAttrStore returns an attr store which does nothing. It returns a global
+// object to avoid unecessary allocations.
+func newNopAttrStore(string) AttrStore { return nopStore }
 
 // nopAttrStore represents a no-op implementation of the AttrStore interface.
 type nopAttrStore struct{}
 
 // Path is a no-op implementation of AttrStore Path method.
-func (s *nopAttrStore) Path() string { return "" }
+func (s nopAttrStore) Path() string { return "" }
 
 // Open is a no-op implementation of AttrStore Open method.
-func (s *nopAttrStore) Open() error {
-	return nil
-}
+func (s nopAttrStore) Open() error { return nil }
 
 // Close is a no-op implementation of AttrStore Close method.
-func (s *nopAttrStore) Close() error {
-	return nil
-}
+func (s nopAttrStore) Close() error { return nil }
 
 // Attrs is a no-op implementation of AttrStore Attrs method.
-func (s *nopAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
-	return nil, nil
-}
+func (s nopAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return nil, nil }
 
 // SetAttrs is a no-op implementation of AttrStore SetAttrs method.
-func (s *nopAttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
-	return nil
-}
+func (s nopAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { return nil }
 
 // SetBulkAttrs is a no-op implementation of AttrStore SetBulkAttrs method.
-func (s *nopAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
-	return nil
-}
+func (s nopAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { return nil }
 
 // Blocks is a no-op implementation of AttrStore Blocks method.
-func (s *nopAttrStore) Blocks() ([]AttrBlock, error) {
-	return nil, nil
-}
+func (s nopAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil }
 
 // BlockData is a no-op implementation of AttrStore BlockData method.
-func (s *nopAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
-	return nil, nil
-}
+func (s nopAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil }
 
 // AttrBlock represents a checksummed block of the attribute store.
 type AttrBlock struct {
@@ -100,12 +82,12 @@ type AttrBlock struct {
 	Checksum []byte `json:"checksum"`
 }
 
-// AttrBlocks represents a list of blocks.
-type AttrBlocks []AttrBlock
+// attrBlocks represents a list of blocks.
+type attrBlocks []AttrBlock
 
 // Diff returns a list of block ids that are different or are new in other.
 // Block lists must be in sorted order.
-func (a AttrBlocks) Diff(other []AttrBlock) []uint64 {
+func (a attrBlocks) Diff(other []AttrBlock) []uint64 {
 	var ids []uint64
 	for {
 		// Read next block from each list.
@@ -165,19 +147,19 @@ func encodeAttr(key string, value interface{}) *internal.Attr {
 	pb := &internal.Attr{Key: key}
 	switch value := value.(type) {
 	case string:
-		pb.Type = AttrTypeString
+		pb.Type = attrTypeString
 		pb.StringValue = value
 	case float64:
-		pb.Type = AttrTypeFloat
+		pb.Type = attrTypeFloat
 		pb.FloatValue = value
 	case uint64:
-		pb.Type = AttrTypeInt
+		pb.Type = attrTypeInt
 		pb.IntValue = int64(value)
 	case int64:
-		pb.Type = AttrTypeInt
+		pb.Type = attrTypeInt
 		pb.IntValue = value
 	case bool:
-		pb.Type = AttrTypeBool
+		pb.Type = attrTypeBool
 		pb.BoolValue = value
 	}
 	return pb
@@ -186,13 +168,13 @@ func encodeAttr(key string, value interface{}) *internal.Attr {
 // decodeAttr converts from an Attr internal representation to a key/value pair.
 func decodeAttr(attr *internal.Attr) (key string, value interface{}) {
 	switch attr.Type {
-	case AttrTypeString:
+	case attrTypeString:
 		return attr.Key, attr.StringValue
-	case AttrTypeInt:
+	case attrTypeInt:
 		return attr.Key, attr.IntValue
-	case AttrTypeBool:
+	case attrTypeBool:
 		return attr.Key, attr.BoolValue
-	case AttrTypeFloat:
+	case attrTypeFloat:
 		return attr.Key, attr.FloatValue
 	default:
 		return attr.Key, nil
@@ -221,3 +203,31 @@ func DecodeAttrs(v []byte) (map[string]interface{}, error) {
 	}
 	return decodeAttrs(pb.GetAttrs()), nil
 }
+
+func newMemAttrStore() AttrStore {
+	return &memAttrStore{
+		store: make(map[uint64]map[string]interface{}),
+	}
+}
+
+// memAttrStore represents an in-memory implementation of the AttrStore interface.
+type memAttrStore struct {
+	store map[uint64]map[string]interface{}
+}
+
+func (s *memAttrStore) Path() string                                          { return "" }
+func (s *memAttrStore) Open() error                                           { return nil }
+func (s *memAttrStore) Close() error                                          { return nil }
+func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return s.store[id], nil }
+func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
+	s.store[id] = m
+	return nil
+}
+func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
+	for id, v := range m {
+		s.store[id] = v
+	}
+	return nil
+}
+func (s *memAttrStore) Blocks() ([]AttrBlock, error)                                  { return nil, nil }
+func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil }

attr_test.go

@@ -15,15 +15,20 @@
 package pilosa_test
 
 import (
+	"io/ioutil"
+	"os"
 	"reflect"
+	"runtime"
+	"sync"
 	"testing"
 
-	"github.com/pilosa/pilosa/test"
+	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/boltdb"
 )
 
 // Ensure database can set and retrieve column attributes.
 func TestAttrStore_Attrs(t *testing.T) {
-	s := test.MustOpenAttrStore()
+	s := MustOpenAttrStore()
 	defer s.Close()
 
 	// Set attributes.
@@ -52,7 +57,7 @@ func TestAttrStore_Attrs(t *testing.T) {
 
 // Ensure database returns a non-nil empty map if unset.
 func TestAttrStore_Attrs_Empty(t *testing.T) {
-	s := test.MustOpenAttrStore()
+	s := MustOpenAttrStore()
 	defer s.Close()
 
 	if m, err := s.Attrs(100); err != nil {
@@ -64,7 +69,7 @@ func TestAttrStore_Attrs_Empty(t *testing.T) {
 
 // Ensure database can unset attributes if explicitly set to nil.
 func TestAttrStore_Attrs_Unset(t *testing.T) {
-	s := test.MustOpenAttrStore()
+	s := MustOpenAttrStore()
 	defer s.Close()
 
 	// Set attributes.
@@ -84,7 +89,7 @@ func TestAttrStore_Attrs_Unset(t *testing.T) {
 
 // Ensure attribute block checksums can be returned.
 func TestAttrStore_Blocks(t *testing.T) {
-	s := test.MustOpenAttrStore()
+	s := MustOpenAttrStore()
 	defer s.Close()
 
 	// Set attributes.
@@ -123,3 +128,67 @@ func TestAttrStore_Blocks(t *testing.T) {
 		t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2])
 	}
 }
+
+// AttrStore represents a test wrapper for pilosa.AttrStore.
+type AttrStore struct {
+	pilosa.AttrStore
+}
+
+// NewAttrStore returns a new instance of AttrStore.
+func NewAttrStore(string) pilosa.AttrStore {
+	f, err := ioutil.TempFile("", "pilosa-attr-")
+	if err != nil {
+		panic(err)
+	}
+	f.Close()
+	os.Remove(f.Name())
+
+	return &AttrStore{boltdb.NewAttrStore(f.Name())}
+}
+
+func BenchmarkAttrStore_Duplicate(b *testing.B) {
+	s := MustOpenAttrStore()
+	defer s.Close()
+
+	// Set attributes.
+	const n = 5
+	for i := 0; i < n; i++ {
+		if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	// Update attributes with an existing subset.
+	cpuN := runtime.GOMAXPROCS(0)
+	var wg sync.WaitGroup
+	for i := 0; i < cpuN; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < b.N/cpuN; j++ {
+				if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil {
+					b.Fatal(err)
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
+func MustOpenAttrStore() pilosa.AttrStore {
+	s := NewAttrStore("")
+	if err := s.Open(); err != nil {
+		panic(err)
+	}
+	return s
+}
+
+// Close closes the database and removes the underlying data.
+func (s *AttrStore) Close() error {
+	defer os.RemoveAll(s.Path())
+	return s.AttrStore.Close()
+}

boltdb/attrstore.go

@@ -30,17 +30,17 @@ import (
 	"github.com/pkg/errors"
 )
 
-// AttrBlockSize is the size of attribute blocks for anti-entropy.
-const AttrBlockSize = 100
+// attrBlockSize is the size of attribute blocks for anti-entropy.
+const attrBlockSize = 100
 
-// AttrCache represents a cache for attributes.
-type AttrCache struct {
+// attrCache represents a cache for attributes.
+type attrCache struct {
 	mu    sync.RWMutex
 	attrs map[uint64]map[string]interface{}
 }
 
 // Get returns the cached attributes for a given id.
-func (c *AttrCache) Get(id uint64) map[string]interface{} {
+func (c *attrCache) Get(id uint64) map[string]interface{} {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
 	attrs := c.attrs[id]
@@ -57,40 +57,40 @@ func (c *AttrCache) Get(id uint64) map[string]interface{} {
 }
 
 // Set updates the cached attributes for a given id.
-func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) {
+func (c *attrCache) Set(id uint64, attrs map[string]interface{}) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	c.attrs[id] = attrs
 }
 
-// AttrStore represents a storage layer for attributes.
-type AttrStore struct {
+// attrStore represents a storage layer for attributes.
+type attrStore struct {
 	mu        sync.RWMutex
 	path      string
 	db        *bolt.DB
-	attrCache *AttrCache
+	attrCache *attrCache
 }
 
-// NewAttrCache returns a new instance of AttrCache.
-func NewAttrCache() *AttrCache {
-	return &AttrCache{
+// newAttrCache returns a new instance of AttrCache.
+func newAttrCache() *attrCache {
+	return &attrCache{
 		attrs: make(map[uint64]map[string]interface{}),
 	}
 }
 
 // NewAttrStore returns a new instance of AttrStore.
 func NewAttrStore(path string) pilosa.AttrStore {
-	return &AttrStore{
+	return &attrStore{
 		path:      path,
-		attrCache: NewAttrCache(),
+		attrCache: newAttrCache(),
 	}
 }
 
 // Path returns path to the store's data file.
-func (s *AttrStore) Path() string { return s.path }
+func (s *attrStore) Path() string { return s.path }
 
 // Open opens and initializes the store.
-func (s *AttrStore) Open() error {
+func (s *attrStore) Open() error {
 	// Open storage.
 	db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
 	if err != nil {
@@ -112,7 +112,7 @@ func (s *AttrStore) Open() error {
 }
 
 // Close closes the store.
-func (s *AttrStore) Close() error {
+func (s *attrStore) Close() error {
 	if s.db != nil {
 		s.db.Close()
 	}
@@ -120,7 +120,7 @@ func (s *AttrStore) Close() error {
 }
 
 // Attrs returns a set of attributes by ID.
-func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
+func (s *attrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 
@@ -147,7 +147,7 @@ func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
 }
 
 // SetAttrs sets attribute values for a given ID.
-func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
+func (s *attrStore) SetAttrs(id uint64, m map[string]interface{}) error {
 	// Ignore empty maps.
 	if len(m) == 0 {
 		return nil
@@ -184,7 +184,7 @@ func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
 }
 
 // SetBulkAttrs sets attribute values for a set of ids.
-func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
+func (s *attrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
@@ -220,7 +220,7 @@ func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
 }
 
 // Blocks returns a list of all blocks in the store.
-func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) {
+func (s *attrStore) Blocks() ([]pilosa.AttrBlock, error) {
 	tx, err := s.db.Begin(false)
 	if err != nil {
 		return nil, errors.Wrap(err, "starting transaction")
@@ -228,7 +228,7 @@ func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) {
 	defer tx.Rollback()
 
 	// Wrap cursor to segment by block.
-	cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize)
+	cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), attrBlockSize)
 
 	// Iterate over each block.
 	var blocks []pilosa.AttrBlock
@@ -251,7 +251,7 @@ func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) {
 }
 
 // BlockData returns all data for a single block.
-func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
+func (s *attrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
 	m := make(map[uint64]map[string]interface{})
 
 	// Start read-only transaction.
@@ -262,8 +262,8 @@ func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, erro
 	defer tx.Rollback()
 
 	// Move to the start of the block.
-	min := u64tob(uint64(i) * AttrBlockSize)
-	max := u64tob(uint64(i+1) * AttrBlockSize)
+	min := u64tob(uint64(i) * attrBlockSize)
+	max := u64tob(uint64(i+1) * attrBlockSize)
 	cur := tx.Bucket([]byte("attrs")).Cursor()
 	for k, v := cur.Seek(min); k != nil; k, v = cur.Next() {
 		// Exit if we're past the end of the block.

broadcast.go

@@ -16,223 +16,150 @@ package pilosa
 
 import (
 	"fmt"
-	"reflect"
 
-	"github.com/gogo/protobuf/proto"
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pkg/errors"
 )
 
-// MemberSet represents an interface for Node membership and inter-node communication.
-type MemberSet interface {
-	// Open starts any network activity implemented by the MemberSet
-	// Node is the local node, used for membership broadcasts.
-	Open(n *Node) error
+// Serializer is an interface for serializing pilosa types to bytes and back.
+type Serializer interface {
+	Marshal(Message) ([]byte, error)
+	Unmarshal([]byte, Message) error
 }
 
-// StaticMemberSet represents a basic MemberSet for testing.
-type StaticMemberSet struct {
-	nodes []*Node
+// broadcaster is an interface for broadcasting messages.
+type broadcaster interface {
+	SendSync(Message) error
+	SendAsync(Message) error
+	SendTo(*Node, Message) error
 }
 
-// NewStaticMemberSet creates a statically defined MemberSet.
-func NewStaticMemberSet(nodes []*Node) *StaticMemberSet {
-	return &StaticMemberSet{
-		nodes: nodes,
-	}
-}
-
-// Open implements the MemberSet interface to start network activity, but for a static MemberSet it does nothing.
-func (s *StaticMemberSet) Open(n *Node) error {
-	return nil
-}
-
-// Broadcaster is an interface for broadcasting messages.
-type Broadcaster interface {
-	SendSync(pb proto.Message) error
-	SendAsync(pb proto.Message) error
-	SendTo(to *Node, pb proto.Message) error
-}
+// Message is the interface implemented by all core pilosa types which can be serialized to messages.
+// TODO add at least a single "isMessage()" method.
+type Message interface{}
 
 func init() {
 	NopBroadcaster = &nopBroadcaster{}
-	NopGossiper = &nopGossiper{}
 }
 
 // NopBroadcaster represents a Broadcaster that doesn't do anything.
-var NopBroadcaster Broadcaster
+var NopBroadcaster broadcaster
 
 type nopBroadcaster struct{}
 
 // SendSync A no-op implementation of Broadcaster SendSync method.
-func (n *nopBroadcaster) SendSync(pb proto.Message) error {
-	return nil
-}
+func (nopBroadcaster) SendSync(Message) error { return nil }
 
 // SendAsync A no-op implementation of Broadcaster SendAsync method.
-func (n *nopBroadcaster) SendAsync(pb proto.Message) error {
-	return nil
-}
+func (nopBroadcaster) SendAsync(Message) error { return nil }
 
 // SendTo is a no-op implementation of Broadcaster SendTo method.
-func (c *nopBroadcaster) SendTo(to *Node, pb proto.Message) error {
-	return nil
-}
-
-// BroadcastHandler is the interface for the pilosa object which knows how to
-// handle broadcast messages. (Hint: this is implemented by pilosa.Server)
-type BroadcastHandler interface {
-	ReceiveMessage(pb proto.Message) error
-}
-
-// BroadcastReceiver is the interface for the object which will listen for and
-// decode broadcast messages before passing them to pilosa to handle. The
-// implementation of this could be an http server which listens for messages,
-// gets the protobuf payload, and then passes it to
-// BroadcastHandler.ReceiveMessage.
-type BroadcastReceiver interface {
-	// Start starts listening for broadcast messages - it should return
-	// immediately, spawning a goroutine if necessary.
-	Start(BroadcastHandler) error
-}
-
-type nopBroadcastReceiver struct{}
-
-func (n *nopBroadcastReceiver) Start(b BroadcastHandler) error { return nil }
-
-// NopBroadcastReceiver is a no-op implementation of the BroadcastReceiver.
-var NopBroadcastReceiver = &nopBroadcastReceiver{}
-
-// Gossiper is an interface for sharing messages via gossip.
-type Gossiper interface {
-	SendAsync(pb proto.Message) error
-}
-
-// NopBroadcaster represents a Broadcaster that doesn't do anything.
-var NopGossiper Gossiper
-
-type nopGossiper struct{}
-
-// SendAsync A no-op implementation of Gossiper SendAsync method.
-func (n *nopGossiper) SendAsync(pb proto.Message) error {
-	return nil
-}
+func (nopBroadcaster) SendTo(*Node, Message) error { return nil }
 
 // Broadcast message types.
 const (
-	MessageTypeCreateSlice = iota
-	MessageTypeCreateIndex
-	MessageTypeDeleteIndex
-	MessageTypeCreateFrame
-	MessageTypeDeleteFrame
-	MessageTypeCreateView
-	MessageTypeDeleteView
-	MessageTypeCreateField
-	MessageTypeDeleteField
-	MessageTypeClusterStatus
-	MessageTypeResizeInstruction
-	MessageTypeResizeInstructionComplete
-	MessageTypeSetCoordinator
-	MessageTypeUpdateCoordinator
-	MessageTypeNodeState
-	MessageTypeRecalculateCaches
-	MessageTypeNodeEvent
+	messageTypeCreateShard = iota
+	messageTypeCreateIndex
+	messageTypeDeleteIndex
+	messageTypeCreateField
+	messageTypeDeleteField
+	messageTypeCreateView
+	messageTypeDeleteView
+	messageTypeClusterStatus
+	messageTypeResizeInstruction
+	messageTypeResizeInstructionComplete
+	messageTypeSetCoordinator
+	messageTypeUpdateCoordinator
+	messageTypeNodeState
+	messageTypeRecalculateCaches
+	messageTypeNodeEvent
+	messageTypeNodeStatus
 )
 
-// MarshalMessage encodes the protobuf message into a byte slice.
-func MarshalMessage(m proto.Message) ([]byte, error) {
-	var typ uint8
-	switch obj := m.(type) {
-	case *internal.CreateSliceMessage:
-		typ = MessageTypeCreateSlice
-	case *internal.CreateIndexMessage:
-		typ = MessageTypeCreateIndex
-	case *internal.DeleteIndexMessage:
-		typ = MessageTypeDeleteIndex
-	case *internal.CreateFrameMessage:
-		typ = MessageTypeCreateFrame
-	case *internal.DeleteFrameMessage:
-		typ = MessageTypeDeleteFrame
-	case *internal.CreateViewMessage:
-		typ = MessageTypeCreateView
-	case *internal.DeleteViewMessage:
-		typ = MessageTypeDeleteView
-	case *internal.CreateFieldMessage:
-		typ = MessageTypeCreateField
-	case *internal.DeleteFieldMessage:
-		typ = MessageTypeDeleteField
-	case *internal.ClusterStatus:
-		typ = MessageTypeClusterStatus
-	case *internal.ResizeInstruction:
-		typ = MessageTypeResizeInstruction
-	case *internal.ResizeInstructionComplete:
-		typ = MessageTypeResizeInstructionComplete
-	case *internal.SetCoordinatorMessage:
-		typ = MessageTypeSetCoordinator
-	case *internal.UpdateCoordinatorMessage:
-		typ = MessageTypeUpdateCoordinator
-	case *internal.NodeStateMessage:
-		typ = MessageTypeNodeState
-	case *internal.RecalculateCaches:
-		typ = MessageTypeRecalculateCaches
-	case *internal.NodeEventMessage:
-		typ = MessageTypeNodeEvent
-	default:
-		return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj))
-	}
-	buf, err := proto.Marshal(m)
+// MarshalInternalMessage serializes the pilosa message and adds pilosa internal
+// type info which is used by the internal messaging stuff.
+func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) {
+	typ := getMessageType(m)
+	buf, err := s.Marshal(m)
 	if err != nil {
-		return nil, errors.Wrap(err, "marshalling")
+		return nil, errors.Wrap(err, "marshaling")
 	}
 	return append([]byte{typ}, buf...), nil
 }
 
-// UnmarshalMessage decodes the byte slice into a protobuf message.
-func UnmarshalMessage(buf []byte) (proto.Message, error) {
-	typ, buf := buf[0], buf[1:]
-
-	var m proto.Message
+func getMessage(typ byte) Message {
 	switch typ {
-	case MessageTypeCreateSlice:
-		m = &internal.CreateSliceMessage{}
-	case MessageTypeCreateIndex:
-		m = &internal.CreateIndexMessage{}
-	case MessageTypeDeleteIndex:
-		m = &internal.DeleteIndexMessage{}
-	case MessageTypeCreateFrame:
-		m = &internal.CreateFrameMessage{}
-	case MessageTypeDeleteFrame:
-		m = &internal.DeleteFrameMessage{}
-	case MessageTypeCreateView:
-		m = &internal.CreateViewMessage{}
-	case MessageTypeDeleteView:
-		m = &internal.DeleteViewMessage{}
-	case MessageTypeCreateField:
-		m = &internal.CreateFieldMessage{}
-	case MessageTypeDeleteField:
-		m = &internal.DeleteFieldMessage{}
-	case MessageTypeClusterStatus:
-		m = &internal.ClusterStatus{}
-	case MessageTypeResizeInstruction:
-		m = &internal.ResizeInstruction{}
-	case MessageTypeResizeInstructionComplete:
-		m = &internal.ResizeInstructionComplete{}
-	case MessageTypeSetCoordinator:
-		m = &internal.SetCoordinatorMessage{}
-	case MessageTypeUpdateCoordinator:
-		m = &internal.UpdateCoordinatorMessage{}
-	case MessageTypeNodeState:
-		m = &internal.NodeStateMessage{}
-	case MessageTypeRecalculateCaches:
-		m = &internal.RecalculateCaches{}
-	case MessageTypeNodeEvent:
-		m = &internal.NodeEventMessage{}
+	case messageTypeCreateShard:
+		return &CreateShardMessage{}
+	case messageTypeCreateIndex:
+		return &CreateIndexMessage{}
+	case messageTypeDeleteIndex:
+		return &DeleteIndexMessage{}
+	case messageTypeCreateField:
+		return &CreateFieldMessage{}
+	case messageTypeDeleteField:
+		return &DeleteFieldMessage{}
+	case messageTypeCreateView:
+		return &CreateViewMessage{}
+	case messageTypeDeleteView:
+		return &DeleteViewMessage{}
+	case messageTypeClusterStatus:
+		return &ClusterStatus{}
+	case messageTypeResizeInstruction:
+		return &ResizeInstruction{}
+	case messageTypeResizeInstructionComplete:
+		return &ResizeInstructionComplete{}
+	case messageTypeSetCoordinator:
+		return &SetCoordinatorMessage{}
+	case messageTypeUpdateCoordinator:
+		return &UpdateCoordinatorMessage{}
+	case messageTypeNodeState:
+		return &NodeStateMessage{}
+	case messageTypeRecalculateCaches:
+		return &RecalculateCaches{}
+	case messageTypeNodeEvent:
+		return &NodeEvent{}
+	case messageTypeNodeStatus:
+		return &NodeStatus{}
 	default:
-		return nil, fmt.Errorf("invalid message type: %d", typ)
+		panic(fmt.Sprintf("unknown message type %d", typ))
+	}
+}
+
+func getMessageType(m Message) byte {
+	switch m.(type) {
+	case *CreateShardMessage:
+		return messageTypeCreateShard
+	case *CreateIndexMessage:
+		return messageTypeCreateIndex
+	case *DeleteIndexMessage:
+		return messageTypeDeleteIndex
+	case *CreateFieldMessage:
+		return messageTypeCreateField
+	case *DeleteFieldMessage:
+		return messageTypeDeleteField
+	case *CreateViewMessage:
+		return messageTypeCreateView
+	case *DeleteViewMessage:
+		return messageTypeDeleteView
+	case *ClusterStatus:
+		return messageTypeClusterStatus
+	case *ResizeInstruction:
+		return messageTypeResizeInstruction
+	case *ResizeInstructionComplete:
+		return messageTypeResizeInstructionComplete
+	case *SetCoordinatorMessage:
+		return messageTypeSetCoordinator
+	case *UpdateCoordinatorMessage:
+		return messageTypeUpdateCoordinator
+	case *NodeStateMessage:
+		return messageTypeNodeState
+	case *RecalculateCaches:
+		return messageTypeRecalculateCaches
+	case *NodeEvent:
+		return messageTypeNodeEvent
+	case *NodeStatus:
+		return messageTypeNodeStatus
+	default:
+		panic(fmt.Sprintf("don't have type for message %#v", m))
 	}
-
-	if err := proto.Unmarshal(buf, m); err != nil {
-		return nil, errors.Wrap(err, "unmarshalling")
-	}
-	return m, nil
 }

broadcast_test.go

@@ -1,119 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa_test
-
-import (
-	"bytes"
-	"reflect"
-	"testing"
-
-	"io/ioutil"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/internal"
-	"github.com/pilosa/pilosa/server"
-)
-
-// Ensure a message can be marshaled and unmarshaled.
-func TestMessage_Marshal(t *testing.T) {
-
-	testMessageMarshal(t, &internal.CreateSliceMessage{
-		Index: "i",
-		Slice: 8,
-	})
-
-	testMessageMarshal(t, &internal.DeleteIndexMessage{
-		Index: "i",
-	})
-}
-
-func testMessageMarshal(t *testing.T, m proto.Message) {
-	marshalled, err := pilosa.MarshalMessage(m)
-	if err != nil {
-		t.Fatal(err)
-	}
-	unmarshalled, err := pilosa.UnmarshalMessage(marshalled)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(unmarshalled, m) {
-		t.Fatalf("unexpected message marshalling: %s", unmarshalled)
-	}
-}
-
-// Ensure that BroadcastReceiver can register a BroadcastHandler.
-func TestBroadcast_BroadcastReceiver(t *testing.T) {
-	path, err := ioutil.TempDir("", "pilosa-")
-	if err != nil {
-		panic(err)
-	}
-	com := server.NewCommand(bytes.NewBuffer([]byte{}), ioutil.Discard, ioutil.Discard)
-	com.Config.Bind = "localhost:0"
-	com.Config.DataDir = path
-	err = com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO
-	if err != nil {
-		t.Fatalf("setting up server: %v", err)
-	}
-	s := com.Server
-
-	sbr := NewSimpleBroadcastReceiver()
-	sbh := NewSimpleBroadcastHandler()
-
-	s.BroadcastReceiver = sbr
-	s.BroadcastReceiver.Start(sbh)
-
-	msg := &internal.DeleteIndexMessage{
-		Index: "i",
-	}
-
-	s.BroadcastReceiver.(*SimpleBroadcastReceiver).Receive(msg)
-
-	// Make sure the message received is what was sentd
-	if !reflect.DeepEqual(sbh.receivedMessage, msg) {
-		t.Fatalf("unexpected message: %s", sbh.receivedMessage)
-	}
-}
-
-type SimpleBroadcastReceiver struct {
-	broadcastHandler pilosa.BroadcastHandler
-}
-
-func NewSimpleBroadcastReceiver() *SimpleBroadcastReceiver {
-	return &SimpleBroadcastReceiver{}
-}
-
-func (r *SimpleBroadcastReceiver) Start(h pilosa.BroadcastHandler) error {
-	r.broadcastHandler = h
-	return nil
-}
-
-func (r *SimpleBroadcastReceiver) Receive(pb proto.Message) error {
-	r.broadcastHandler.ReceiveMessage(pb)
-	return nil
-}
-
-type SimpleBroadcastHandler struct {
-	receivedMessage proto.Message
-}
-
-func NewSimpleBroadcastHandler() *SimpleBroadcastHandler {
-	return &SimpleBroadcastHandler{}
-}
-
-func (h *SimpleBroadcastHandler) ReceiveMessage(pb proto.Message) error {
-	h.receivedMessage = pb.(proto.Message)
-	return nil
-}

cache.go

@@ -22,17 +22,16 @@ import (
 	"sync"
 	"time"
 
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pilosa/pilosa/lru"
 )
 
 const (
-	// ThresholdFactor is used to calculate the threshold for new items entering the cache
-	ThresholdFactor = 1.1
+	// thresholdFactor is used to calculate the threshold for new items entering the cache
+	thresholdFactor = 1.1
 )
 
-// Cache represents a cache of counts.
-type Cache interface {
+// cache represents a cache of counts.
+type cache interface {
 	Add(id uint64, n uint64)
 	BulkAdd(id uint64, n uint64)
 	Get(id uint64) uint64
@@ -48,22 +47,22 @@ type Cache interface {
 	Recalculate()
 
 	// Returns an ordered list of the top ranked bitmaps.
-	Top() []BitmapPair
+	Top() []bitmapPair
 
 	// SetStats defines the stats client used in the cache.
 	SetStats(s StatsClient)
 }
 
-// LRUCache represents a least recently used Cache implementation.
-type LRUCache struct {
+// lruCache represents a least recently used Cache implementation.
+type lruCache struct {
 	cache  *lru.Cache
 	counts map[uint64]uint64
 	stats  StatsClient
 }
 
-// NewLRUCache returns a new instance of LRUCache.
-func NewLRUCache(maxEntries uint32) *LRUCache {
-	c := &LRUCache{
+// newLRUCache returns a new instance of LRUCache.
+func newLRUCache(maxEntries uint32) *lruCache {
+	c := &lruCache{
 		cache:  lru.New(int(maxEntries)),
 		counts: make(map[uint64]uint64),
 		stats:  NopStatsClient,
@@ -73,34 +72,34 @@ func NewLRUCache(maxEntries uint32) *LRUCache {
 }
 
 // BulkAdd adds a count to the cache unsorted. You should Invalidate after completion.
-func (c *LRUCache) BulkAdd(id, n uint64) {
+func (c *lruCache) BulkAdd(id, n uint64) {
 	c.Add(id, n)
 }
 
 // Add adds a count to the cache.
-func (c *LRUCache) Add(id, n uint64) {
+func (c *lruCache) Add(id, n uint64) {
 	c.cache.Add(id, n)
 	c.counts[id] = n
 }
 
 // Get returns a count for a given id.
-func (c *LRUCache) Get(id uint64) uint64 {
+func (c *lruCache) Get(id uint64) uint64 {
 	n, _ := c.cache.Get(id)
 	nn, _ := n.(uint64)
 	return nn
 }
 
 // Len returns the number of items in the cache.
-func (c *LRUCache) Len() int { return c.cache.Len() }
+func (c *lruCache) Len() int { return c.cache.Len() }
 
 // Invalidate is a no-op.
-func (c *LRUCache) Invalidate() {}
+func (c *lruCache) Invalidate() {}
 
 // Recalculate is a no-op.
-func (c *LRUCache) Recalculate() {}
+func (c *lruCache) Recalculate() {}
 
 // IDs returns a list of all IDs in the cache.
-func (c *LRUCache) IDs() []uint64 {
+func (c *lruCache) IDs() []uint64 {
 	a := make([]uint64, 0, len(c.counts))
 	for id := range c.counts {
 		a = append(a, id)
@@ -110,33 +109,33 @@ func (c *LRUCache) IDs() []uint64 {
 }
 
 // Top returns all counts in the cache.
-func (c *LRUCache) Top() []BitmapPair {
-	a := make([]BitmapPair, 0, len(c.counts))
+func (c *lruCache) Top() []bitmapPair {
+	a := make([]bitmapPair, 0, len(c.counts))
 	for id, n := range c.counts {
-		a = append(a, BitmapPair{
+		a = append(a, bitmapPair{
 			ID:    id,
 			Count: uint64(n),
 		})
 	}
-	sort.Sort(BitmapPairs(a))
+	sort.Sort(bitmapPairs(a))
 	return a
 }
 
 // SetStats defines the stats client used in the cache.
-func (c *LRUCache) SetStats(s StatsClient) {
+func (c *lruCache) SetStats(s StatsClient) {
 	c.stats = s
 }
 
-func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) }
+func (c *lruCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) }
 
 // Ensure LRUCache implements Cache.
-var _ Cache = &LRUCache{}
+var _ cache = &lruCache{}
 
-// RankCache represents a cache with sorted entries.
-type RankCache struct {
+// rankCache represents a cache with sorted entries.
+type rankCache struct {
 	mu       sync.Mutex
 	entries  map[uint64]uint64
-	rankings []BitmapPair // cached, ordered list
+	rankings []bitmapPair // cached, ordered list
 
 	updateN    int
 	updateTime time.Time
@@ -155,17 +154,17 @@ type RankCache struct {
 }
 
 // NewRankCache returns a new instance of RankCache.
-func NewRankCache(maxEntries uint32) *RankCache {
-	return &RankCache{
+func NewRankCache(maxEntries uint32) *rankCache {
+	return &rankCache{
 		maxEntries:      maxEntries,
-		thresholdBuffer: int(ThresholdFactor * float64(maxEntries)),
+		thresholdBuffer: int(thresholdFactor * float64(maxEntries)),
 		entries:         make(map[uint64]uint64),
 		stats:           NopStatsClient,
 	}
 }
 
 // Add adds a count to the cache.
-func (c *RankCache) Add(id uint64, n uint64) {
+func (c *rankCache) Add(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	// Ignore if the column count is below the threshold.
@@ -179,7 +178,7 @@ func (c *RankCache) Add(id uint64, n uint64) {
 }
 
 // BulkAdd adds a count to the cache unsorted. You should Invalidate after completion.
-func (c *RankCache) BulkAdd(id uint64, n uint64) {
+func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if n < c.thresholdValue {
@@ -190,21 +189,21 @@ func (c *RankCache) BulkAdd(id uint64, n uint64) {
 }
 
 // Get returns a count for a given id.
-func (c *RankCache) Get(id uint64) uint64 {
+func (c *rankCache) Get(id uint64) uint64 {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	return c.entries[id]
 }
 
 // Len returns the number of items in the cache.
-func (c *RankCache) Len() int {
+func (c *rankCache) Len() int {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	return len(c.entries)
 }
 
 // IDs returns a list of all IDs in the cache.
-func (c *RankCache) IDs() []uint64 {
+func (c *rankCache) IDs() []uint64 {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	a := make([]uint64, 0, len(c.entries))
@@ -216,21 +215,21 @@ func (c *RankCache) IDs() []uint64 {
 }
 
 // Invalidate recalculates the entries by rank.
-func (c *RankCache) Invalidate() {
+func (c *rankCache) Invalidate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	c.invalidate()
 }
 
 // Recalculate rebuilds the cache.
-func (c *RankCache) Recalculate() {
+func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	c.stats.Count("cache.recalculate", 1, 1.0)
 	c.recalculate()
 }
 
-func (c *RankCache) invalidate() {
+func (c *rankCache) invalidate() {
 	// Don't invalidate more than once every X seconds.
 	// TODO: consider making this configurable.
 	if time.Since(c.updateTime).Seconds() < 10 {
@@ -240,23 +239,23 @@ func (c *RankCache) invalidate() {
 	c.recalculate()
 }
 
-func (c *RankCache) recalculate() {
+func (c *rankCache) recalculate() {
 	// Convert cache to a sorted list.
-	rankings := make([]BitmapPair, 0, len(c.entries))
+	rankings := make([]bitmapPair, 0, len(c.entries))
 	for id, cnt := range c.entries {
-		rankings = append(rankings, BitmapPair{
+		rankings = append(rankings, bitmapPair{
 			ID:    id,
 			Count: cnt,
 		})
 	}
-	sort.Sort(BitmapPairs(rankings))
+	sort.Sort(bitmapPairs(rankings))
 
 	// Store the count of the item at the threshold index.
 	c.rankings = rankings
 	length := len(c.rankings)
 	c.stats.Gauge("RankCache", float64(length), 1.0)
 
-	var removeItems []BitmapPair // cached, ordered list
+	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
 		c.thresholdValue = rankings[c.maxEntries].Count
 		removeItems = c.rankings[c.maxEntries:]
@@ -278,38 +277,38 @@ func (c *RankCache) recalculate() {
 }
 
 // SetStats defines the stats client used in the cache.
-func (c *RankCache) SetStats(s StatsClient) {
+func (c *rankCache) SetStats(s StatsClient) {
 	c.stats = s
 }
 
 // Top returns an ordered list of pairs.
-func (c *RankCache) Top() []BitmapPair { return c.rankings }
+func (c *rankCache) Top() []bitmapPair { return c.rankings }
 
 // WriteTo writes the cache to w.
-func (c *RankCache) WriteTo(w io.Writer) (n int64, err error) {
+func (c *rankCache) WriteTo(w io.Writer) (n int64, err error) {
 	panic("FIXME: TODO")
 }
 
 // ReadFrom read from r into the cache.
-func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) {
+func (c *rankCache) ReadFrom(r io.Reader) (n int64, err error) {
 	panic("FIXME: TODO")
 }
 
 // Ensure RankCache implements Cache.
-var _ Cache = &RankCache{}
+var _ cache = &rankCache{}
 
-// BitmapPair represents a id/count pair with an associated identifier.
-type BitmapPair struct {
+// bitmapPair represents a id/count pair with an associated identifier.
+type bitmapPair struct {
 	ID    uint64
 	Count uint64
 }
 
-// BitmapPairs is a sortable list of BitmapPair objects.
-type BitmapPairs []BitmapPair
+// bitmapPairs is a sortable list of BitmapPair objects.
+type bitmapPairs []bitmapPair
 
-func (p BitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
-func (p BitmapPairs) Len() int           { return len(p) }
-func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
+func (p bitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p bitmapPairs) Len() int           { return len(p) }
+func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
 
 // Pair holds an id/count pair.
 type Pair struct {
@@ -318,22 +317,6 @@ type Pair struct {
 	Count uint64 `json:"count"`
 }
 
-func encodePair(p Pair) *internal.Pair {
-	return &internal.Pair{
-		ID:    p.ID,
-		Key:   p.Key,
-		Count: p.Count,
-	}
-}
-
-func decodePair(pb *internal.Pair) Pair {
-	return Pair{
-		ID:    pb.ID,
-		Key:   pb.Key,
-		Count: pb.Count,
-	}
-}
-
 // Pairs is a sortable slice of Pair objects.
 type Pairs []Pair
 
@@ -341,14 +324,14 @@ func (p Pairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p Pairs) Len() int           { return len(p) }
 func (p Pairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
 
-// PairHeap is a heap implementation over a group of Pairs.
-type PairHeap struct {
+// pairHeap is a heap implementation over a group of Pairs.
+type pairHeap struct {
 	Pairs
 }
 
 // Less implemets the Sort interface.
 // reports whether the element with index i should sort before the element with index j.
-func (p PairHeap) Less(i, j int) bool { return p.Pairs[i].Count < p.Pairs[j].Count }
+func (p pairHeap) Less(i, j int) bool { return p.Pairs[i].Count < p.Pairs[j].Count }
 
 // Push appends the element onto the Pair slice.
 func (p *Pairs) Push(x interface{}) {
@@ -409,22 +392,6 @@ func (p Pairs) String() string {
 	return buf.String()
 }
 
-func encodePairs(a Pairs) []*internal.Pair {
-	other := make([]*internal.Pair, len(a))
-	for i := range a {
-		other[i] = encodePair(a[i])
-	}
-	return other
-}
-
-func decodePairs(a []*internal.Pair) []Pair {
-	other := make([]Pair, len(a))
-	for i := range a {
-		other[i] = decodePair(a[i])
-	}
-	return other
-}
-
 // uint64Slice represents a sortable slice of uint64 numbers.
 type uint64Slice []uint64
 
@@ -461,60 +428,52 @@ func (p uint64Slice) merge(other []uint64) []uint64 {
 	return ret
 }
 
-// BitmapCache provides an interface for caching full bitmaps.
-type BitmapCache interface {
+// bitmapCache provides an interface for caching full bitmaps.
+type bitmapCache interface {
 	Fetch(id uint64) (*Row, bool)
 	Add(id uint64, b *Row)
 }
 
-// SimpleCache implements BitmapCache
+// simpleCache implements BitmapCache
 // it is meant to be a short-lived cache for cases where writes are continuing to access
 // the same row within a short time frame (i.e. good for write-heavy loads)
 // A read-heavy use case would cause the cache to get bigger, potentially causing the
 // node to run out of memory.
-type SimpleCache struct {
+type simpleCache struct {
 	cache map[uint64]*Row
 }
 
 // Fetch retrieves the bitmap at the id in the cache.
-func (s *SimpleCache) Fetch(id uint64) (*Row, bool) {
+func (s *simpleCache) Fetch(id uint64) (*Row, bool) {
 	m, ok := s.cache[id]
 	return m, ok
 }
 
 // Add adds the bitmap to the cache, keyed on the id.
-func (s *SimpleCache) Add(id uint64, b *Row) {
+func (s *simpleCache) Add(id uint64, b *Row) {
 	s.cache[id] = b
 }
 
-// NopCache represents a no-op Cache implementation.
-type NopCache struct {
+// nopCache represents a no-op Cache implementation.
+type nopCache struct {
 	stats StatsClient
 }
 
 // Ensure NopCache implements Cache.
-var _ Cache = &NopCache{}
-
-// NewNopCache returns a new instance of NopCache.
-func NewNopCache() *NopCache {
-	return &NopCache{
-		stats: NopStatsClient,
-	}
+var globalNopCache cache = nopCache{
+	stats: NopStatsClient,
 }
 
-func (c *NopCache) Add(id uint64, n uint64)     {}
-func (c *NopCache) BulkAdd(id uint64, n uint64) {}
-func (c *NopCache) Get(id uint64) uint64        { return 0 }
-func (c *NopCache) IDs() []uint64               { return make([]uint64, 0) }
+func (c nopCache) Add(uint64, uint64)     {}
+func (c nopCache) BulkAdd(uint64, uint64) {}
+func (c nopCache) Get(uint64) uint64      { return 0 }
+func (c nopCache) IDs() []uint64          { return []uint64{} }
 
-func (c *NopCache) Invalidate() {}
-func (c *NopCache) Len() int    { return 0 }
-func (c *NopCache) Recalculate() {
-}
-func (c *NopCache) SetStats(s StatsClient) {
-	c.stats = s
-}
+func (c nopCache) Invalidate()          {}
+func (c nopCache) Len() int             { return 0 }
+func (c nopCache) Recalculate()         {}
+func (c nopCache) SetStats(StatsClient) {}
 
-func (c *NopCache) Top() []BitmapPair {
-	return []BitmapPair{}
+func (c nopCache) Top() []bitmapPair {
+	return []bitmapPair{}
 }

client_test.go

@@ -1,353 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa_test
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"reflect"
-	"testing"
-
-	"github.com/davecgh/go-spew/spew"
-	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/internal"
-	"github.com/pilosa/pilosa/pql"
-	"github.com/pilosa/pilosa/server"
-	"github.com/pilosa/pilosa/test"
-)
-
-func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) {
-	numNodes := len(c.Nodes)
-	hldr := make([]*test.Holder, numNodes)
-	server := make([]*test.Server, numNodes)
-	for i := 0; i < numNodes; i++ {
-		hldr[i] = test.MustOpenHolder()
-		server[i] = test.NewServer()
-		server[i].Handler.API.Cluster = c
-		server[i].Handler.API.Cluster.Nodes[i].URI = server[i].HostURI()
-		server[i].Handler.API.Holder = hldr[i].Holder
-	}
-	return server, hldr
-}
-
-var defaultClient *http.Client
-
-func init() {
-	defaultClient = server.GetHTTPClient(nil)
-
-}
-
-// Test distributed TopN Row count across 3 nodes.
-func TestClient_MultiNode(t *testing.T) {
-	cluster := test.NewCluster(3)
-	s, hldr := createCluster(cluster)
-
-	for i := 0; i < len(cluster.Nodes); i++ {
-		defer hldr[i].Close()
-		defer s[i].Close()
-	}
-
-	s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
-		e := pilosa.NewExecutor(defaultClient)
-		e.Holder = hldr[0].Holder
-		e.Node = cluster.Nodes[0]
-		e.Cluster = cluster
-		return e.Execute(ctx, index, query, slices, opt)
-	}
-	s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
-		e := pilosa.NewExecutor(defaultClient)
-		e.Holder = hldr[1].Holder
-		e.Node = cluster.Nodes[1]
-		e.Cluster = cluster
-		return e.Execute(ctx, index, query, slices, opt)
-	}
-	s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
-		e := pilosa.NewExecutor(defaultClient)
-		e.Holder = hldr[2].Holder
-		e.Node = cluster.Nodes[2]
-		e.Cluster = cluster
-		return e.Execute(ctx, index, query, slices, opt)
-	}
-
-	// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
-	sliceNums := []uint64{1, 2, 6}
-	for i, num := range sliceNums {
-		owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())
-		ownsNum := false
-		for _, ownNum := range owns {
-			if ownNum == num {
-				ownsNum = true
-				break
-			}
-		}
-		if !ownsNum {
-			t.Fatalf("Trying to use slice %d on host %s, but it doesn't own that slice. It owns %v", num, s[i].Host(), owns)
-		}
-	}
-
-	baseBit0 := SliceWidth * sliceNums[0]
-	baseBit1 := SliceWidth * sliceNums[1]
-	baseBit2 := SliceWidth * sliceNums[2]
-
-	maxSlice := uint64(0)
-	for _, x := range sliceNums {
-		if x > maxSlice {
-			maxSlice = x
-		}
-	}
-
-	hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(100, baseBit0+10)
-	hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12)
-	hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15)
-	hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4)
-	hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5)
-	hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(22, baseBit0+1, baseBit0+2, baseBit0+10)
-
-	hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4)
-	hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10)
-	hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6)
-	hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(1, baseBit1+4)
-	hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5)
-
-	hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14)
-	hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13)
-	hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(21, baseBit2+10)
-	hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(100, baseBit2+10)
-	hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(99, baseBit2+10, baseBit2+11, baseBit2+12)
-	hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(98, baseBit2+10, baseBit2+11)
-	hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(22, baseBit2+10, baseBit2+11, baseBit2+12)
-
-	// Rebuild the RankCache.
-	// We have to do this to avoid the 10-second cache invalidation delay
-	// built into cache.Invalidate()
-	hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).RecalculateCache()
-	hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).RecalculateCache()
-	hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache()
-
-	// Connect to each node to compare results.
-	client := make([]*test.Client, 3)
-	client[0] = test.MustNewClient(s[0].Host(), defaultClient)
-	client[1] = test.MustNewClient(s[1].Host(), defaultClient)
-	client[2] = test.MustNewClient(s[2].Host(), defaultClient)
-
-	topN := 4
-	queryRequest := &internal.QueryRequest{
-		Query:  fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN),
-		Remote: false,
-	}
-	result, err := client[0].Query(context.Background(), "i", queryRequest)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Check the results before every node has the correct max slice value.
-	pairs := result.Results[0].Pairs
-	for _, pair := range pairs {
-		if pair.ID == 22 && pair.Count != 3 {
-			t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair)
-		}
-	}
-
-	// Set max slice to correct value.
-	hldr[0].Index("i").SetRemoteMaxSlice(maxSlice)
-	hldr[1].Index("i").SetRemoteMaxSlice(maxSlice)
-	hldr[2].Index("i").SetRemoteMaxSlice(maxSlice)
-
-	result, err = client[0].Query(context.Background(), "i", queryRequest)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Test must return exactly N results.
-	if len(result.Results[0].Pairs) != topN {
-		t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result))
-	}
-	p := []*internal.Pair{
-		{ID: 100, Count: 12},
-		{ID: 22, Count: 11},
-		{ID: 98, Count: 8},
-		{ID: 99, Count: 7}}
-
-	// Valdidate the Top 4 result counts.
-	if !reflect.DeepEqual(result.Results[0].Pairs, p) {
-		t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
-	}
-
-	result1, err := client[1].Query(context.Background(), "i", queryRequest)
-	if err != nil {
-		t.Fatal(err)
-	}
-	result2, err := client[2].Query(context.Background(), "i", queryRequest)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Compare TopN results across all nodes in the cluster.
-	if !reflect.DeepEqual(result, result1) {
-		t.Fatalf("TopN result should be the same on node0 and node1: %s", spew.Sdump(result1))
-	}
-
-	if !reflect.DeepEqual(result, result2) {
-		t.Fatalf("TopN result should be the same on node0 and node2: %s", spew.Sdump(result2))
-	}
-}
-
-// Ensure client can bulk import data.
-func TestClient_Import(t *testing.T) {
-	hldr := test.MustOpenHolder()
-	defer hldr.Close()
-
-	// Load bitmap into cache to ensure cache gets updated.
-	f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
-	f.Row(0)
-
-	s := test.NewServer()
-	defer s.Close()
-	s.Handler.API.Cluster = test.NewCluster(1)
-	s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
-	s.Handler.API.Holder = hldr.Holder
-
-	// Send import request.
-	c := test.MustNewClient(s.Host(), defaultClient)
-	if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
-		{RowID: 0, ColumnID: 1},
-		{RowID: 0, ColumnID: 5},
-		{RowID: 200, ColumnID: 6},
-	}); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify data.
-	if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) {
-		t.Fatalf("unexpected columns: %+v", a)
-	}
-	if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{6}) {
-		t.Fatalf("unexpected columns: %+v", a)
-	}
-}
-
-// Ensure client can bulk import value data.
-func TestClient_ImportValue(t *testing.T) {
-	hldr := test.MustOpenHolder()
-	defer hldr.Close()
-
-	fld := pilosa.Field{
-		Name: "fld",
-		Type: pilosa.FieldTypeInt,
-		Min:  -100,
-		Max:  100,
-	}
-
-	// Load bitmap into cache to ensure cache gets updated.
-	index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-	frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{Fields: []*pilosa.Field{&fld}})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	s := test.NewServer()
-	defer s.Close()
-	s.Handler.API.Cluster = test.NewCluster(1)
-	s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
-	s.Handler.API.Holder = hldr.Holder
-
-	// Send import request.
-	c := test.MustNewClient(s.Host(), defaultClient)
-	if err := c.ImportValue(context.Background(), "i", "f", fld.Name, 0, []pilosa.FieldValue{
-		{ColumnID: 1, Value: -10},
-		{ColumnID: 2, Value: 20},
-		{ColumnID: 3, Value: 40},
-	}); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify Sum.
-	sum, cnt, err := frame.FieldSum(nil, fld.Name)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if sum != 50 || cnt != 3 {
-		t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt)
-	}
-
-	// Verify Min.
-	min, cnt, err := frame.FieldMin(nil, fld.Name)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if min != -10 || cnt != 1 {
-		t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-10, cnt=1", min, cnt)
-	}
-
-	// Verify Min with Filter.
-	filter, err := frame.FieldRange(fld.Name, pql.GT, 40)
-	if err != nil {
-		t.Fatal(err)
-	}
-	min, cnt, err = frame.FieldMin(filter, fld.Name)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if min != -100 || cnt != 0 {
-		t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt)
-	}
-
-	// Verify Max.
-	max, cnt, err := frame.FieldMax(nil, fld.Name)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if max != 40 || cnt != 1 {
-		t.Fatalf("unexpected values: got max=%v, count=%v; expected max=40, cnt=1", max, cnt)
-	}
-}
-
-// Ensure client can retrieve a list of all checksums for blocks in a fragment.
-func TestClient_FragmentBlocks(t *testing.T) {
-	hldr := test.MustOpenHolder()
-	defer hldr.Close()
-
-	// Set two bits on blocks 0 & 3.
-	hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
-	hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100)
-
-	// Set a bit on a different slice.
-	hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1)
-
-	s := test.NewServer()
-	defer s.Close()
-	s.Handler.API.Cluster = test.NewCluster(1)
-	s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
-	s.Handler.API.Holder = hldr.Holder
-
-	// Retrieve blocks.
-	c := test.MustNewClient(s.Host(), defaultClient)
-	blocks, err := c.FragmentBlocks(context.Background(), "i", "f", 0)
-	if err != nil {
-		t.Fatal(err)
-	} else if len(blocks) != 2 {
-		t.Fatalf("unexpected blocks: %s", spew.Sdump(blocks))
-	} else if blocks[0].ID != 0 {
-		t.Fatalf("unexpected block id(0): %d", blocks[0].ID)
-	} else if blocks[1].ID != 3 {
-		t.Fatalf("unexpected block id(1): %d", blocks[1].ID)
-	}
-
-	// Verify data matches local blocks.
-	if a := hldr.Fragment("i", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) {
-		t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks))
-	}
-}

cluster_internal_test.go

@@ -15,12 +15,16 @@
 package pilosa
 
 import (
+	"bytes"
 	"io/ioutil"
+	"math/rand"
 	"reflect"
 	"strings"
 	"testing"
+	"testing/quick"
 
-	"github.com/pilosa/pilosa/internal"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/pkg/errors"
 )
 
 // Ensure that fragCombos creates the correct fragment mapping.
@@ -38,20 +42,20 @@ func TestFragCombos(t *testing.T) {
 	node0 := &Node{ID: "node0", URI: *uri0}
 	node1 := &Node{ID: "node1", URI: *uri1}
 
-	c := NewCluster()
+	c := newCluster()
 	c.addNodeBasicSorted(node0)
 	c.addNodeBasicSorted(node1)
 
 	tests := []struct {
 		idx        string
-		maxSlice   uint64
-		frameViews viewsByFrame
+		maxShard   uint64
+		fieldViews viewsByField
 		expected   fragsByHost
 	}{
 		{
 			idx:        "i",
-			maxSlice:   uint64(2),
-			frameViews: viewsByFrame{"f": []string{"v1", "v2"}},
+			maxShard:   uint64(2),
+			fieldViews: viewsByField{"f": []string{"v1", "v2"}},
 			expected: fragsByHost{
 				"node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}},
 				"node1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}},
@@ -59,8 +63,8 @@ func TestFragCombos(t *testing.T) {
 		},
 		{
 			idx:        "foo",
-			maxSlice:   uint64(3),
-			frameViews: viewsByFrame{"f": []string{"v0"}},
+			maxShard:   uint64(3),
+			fieldViews: viewsByField{"f": []string{"v0"}},
 			expected: fragsByHost{
 				"node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}},
 				"node1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}},
@@ -69,7 +73,7 @@ func TestFragCombos(t *testing.T) {
 	}
 	for _, test := range tests {
 
-		actual := c.fragCombos(test.idx, test.maxSlice, test.frameViews)
+		actual := c.fragCombos(test.idx, test.maxShard, test.fieldViews)
 		if !reflect.DeepEqual(actual, test.expected) {
 			t.Errorf("expected: %v, but got: %v", test.expected, actual)
 		}
@@ -115,29 +119,29 @@ func TestFragSources(t *testing.T) {
 	node2 := &Node{ID: "node2", URI: *uri2}
 	node3 := &Node{ID: "node3", URI: *uri3}
 
-	c1 := NewCluster()
+	c1 := newCluster()
 	c1.ReplicaN = 1
 	c1.addNodeBasicSorted(node0)
 	c1.addNodeBasicSorted(node1)
 
-	c2 := NewCluster()
+	c2 := newCluster()
 	c2.ReplicaN = 1
 	c2.addNodeBasicSorted(node0)
 	c2.addNodeBasicSorted(node1)
 	c2.addNodeBasicSorted(node2)
 
-	c3 := NewCluster()
+	c3 := newCluster()
 	c3.ReplicaN = 2
 	c3.addNodeBasicSorted(node0)
 	c3.addNodeBasicSorted(node1)
 
-	c4 := NewCluster()
+	c4 := newCluster()
 	c4.ReplicaN = 2
 	c4.addNodeBasicSorted(node0)
 	c4.addNodeBasicSorted(node1)
 	c4.addNodeBasicSorted(node2)
 
-	c5 := NewCluster()
+	c5 := newCluster()
 	c5.ReplicaN = 2
 	c5.addNodeBasicSorted(node0)
 	c5.addNodeBasicSorted(node1)
@@ -145,44 +149,44 @@ func TestFragSources(t *testing.T) {
 	c5.addNodeBasicSorted(node3)
 
 	idx := newIndexWithTempPath("i")
-	frame, err := idx.CreateFrameIfNotExists("f", FrameOptions{})
+	field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = frame.SetBit("standard", 1, 101, nil)
+	_, err = field.SetBit(1, 101, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = frame.SetBit("standard", 1, 1300000, nil)
+	_, err = field.SetBit(1, 1300000, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = frame.SetBit("standard", 1, 2600000, nil)
+	_, err = field.SetBit(1, 2600000, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = frame.SetBit("standard", 1, 3900000, nil)
+	_, err = field.SetBit(1, 3900000, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	tests := []struct {
-		from     *Cluster
-		to       *Cluster
+		from     *cluster
+		to       *cluster
 		idx      *Index
-		expected map[string][]*internal.ResizeSource
+		expected map[string][]*ResizeSource
 		err      string
 	}{
 		{
 			from: c1,
 			to:   c2,
 			idx:  idx,
-			expected: map[string][]*internal.ResizeSource{
-				"node0": []*internal.ResizeSource{},
-				"node1": []*internal.ResizeSource{},
-				"node2": []*internal.ResizeSource{
-					{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
-					{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)},
+			expected: map[string][]*ResizeSource{
+				"node0": []*ResizeSource{},
+				"node1": []*ResizeSource{},
+				"node2": []*ResizeSource{
+					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
+					{&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@@ -191,13 +195,13 @@ func TestFragSources(t *testing.T) {
 			from: c4,
 			to:   c3,
 			idx:  idx,
-			expected: map[string][]*internal.ResizeSource{
-				"node0": []*internal.ResizeSource{
-					{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)},
+			expected: map[string][]*ResizeSource{
+				"node0": []*ResizeSource{
+					{&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)},
 				},
-				"node1": []*internal.ResizeSource{
-					{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
-					{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)},
+				"node1": []*ResizeSource{
+					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
+					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@@ -206,15 +210,15 @@ func TestFragSources(t *testing.T) {
 			from: c5,
 			to:   c4,
 			idx:  idx,
-			expected: map[string][]*internal.ResizeSource{
-				"node0": []*internal.ResizeSource{
-					{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)},
-					{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)},
+			expected: map[string][]*ResizeSource{
+				"node0": []*ResizeSource{
+					{&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)},
+					{&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)},
 				},
-				"node1": []*internal.ResizeSource{
-					{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)},
+				"node1": []*ResizeSource{
+					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)},
 				},
-				"node2": []*internal.ResizeSource{},
+				"node2": []*ResizeSource{},
 			},
 			err: "",
 		},
@@ -287,19 +291,19 @@ func TestResizeJob(t *testing.T) {
 		{
 			existingNodes: []*Node{node0, node1},
 			node:          node2,
-			action:        ResizeJobActionAdd,
+			action:        resizeJobActionAdd,
 			expectedIDs:   map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false},
 		},
 		{
 			existingNodes: []*Node{node0, node1, node2},
 			node:          node2,
-			action:        ResizeJobActionRemove,
+			action:        resizeJobActionRemove,
 			expectedIDs:   map[string]bool{node0.ID: false, node1.ID: false},
 		},
 	}
 	for _, test := range tests {
 
-		actual := NewResizeJob(test.existingNodes, test.node, test.action)
+		actual := newResizeJob(test.existingNodes, test.node, test.action)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -308,3 +312,463 @@ func TestResizeJob(t *testing.T) {
 		}
 	}
 }
+
+// Ensure the cluster can fairly distribute partitions across the nodes.
+func TestCluster_Owners(t *testing.T) {
+	c := cluster{
+		Nodes: []*Node{
+			{URI: NewTestURIFromHostPort("serverA", 1000)},
+			{URI: NewTestURIFromHostPort("serverB", 1000)},
+			{URI: NewTestURIFromHostPort("serverC", 1000)},
+		},
+		Hasher:   NewTestModHasher(),
+		ReplicaN: 2,
+	}
+
+	// Verify nodes are distributed.
+	if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) {
+		t.Fatalf("unexpected owners: %s", spew.Sdump(a))
+	}
+
+	// Verify nodes go around the ring.
+	if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) {
+		t.Fatalf("unexpected owners: %s", spew.Sdump(a))
+	}
+}
+
+// Ensure the partitioner can assign a fragment to a partition.
+func TestCluster_Partition(t *testing.T) {
+	if err := quick.Check(func(index string, shard uint64, partitionN int) bool {
+		c := newCluster()
+		c.partitionN = partitionN
+
+		partitionID := c.partition(index, shard)
+		if partitionID < 0 || partitionID >= partitionN {
+			t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN)
+		}
+
+		return true
+	}, &quick.Config{
+		Values: func(values []reflect.Value, rand *rand.Rand) {
+			values[0], _ = quick.Value(reflect.TypeOf(""), rand)
+			values[1] = reflect.ValueOf(uint64(rand.Uint32()))
+			values[2] = reflect.ValueOf(rand.Intn(1000) + 1)
+		},
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Ensure the hasher can hash correctly.
+func TestHasher(t *testing.T) {
+	for _, tt := range []struct {
+		key    uint64
+		bucket []int
+	}{
+		// Generated from the reference C++ code
+		{0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
+		{1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}},
+		{0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}},
+		{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
+	} {
+		for i, v := range tt.bucket {
+			if got := newHasher().Hash(tt.key, i+1); got != v {
+				t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
+			}
+		}
+	}
+}
+
+// Ensure ContainsShards can find the actual shard list for node and index.
+func TestCluster_ContainsShards(t *testing.T) {
+	c := NewTestCluster(5)
+	c.ReplicaN = 3
+	shards := c.containsShards("test", 10, c.Nodes[2])
+
+	if !reflect.DeepEqual(shards, []uint64{0, 2, 3, 5, 6, 9, 10}) {
+		t.Fatalf("unexpected shars for node's index: %v", shards)
+	}
+}
+
+func TestCluster_Nodes(t *testing.T) {
+	uri0 := NewTestURIFromHostPort("node0", 0)
+	uri1 := NewTestURIFromHostPort("node1", 0)
+	uri2 := NewTestURIFromHostPort("node2", 0)
+	uri3 := NewTestURIFromHostPort("node3", 0)
+
+	node0 := &Node{ID: "node0", URI: uri0}
+	node1 := &Node{ID: "node1", URI: uri1}
+	node2 := &Node{ID: "node2", URI: uri2}
+	node3 := &Node{ID: "node3", URI: uri3}
+
+	nodes := []*Node{node0, node1, node2}
+
+	t.Run("NodeIDs", func(t *testing.T) {
+		actual := Nodes(nodes).IDs()
+		expected := []string{node0.ID, node1.ID, node2.ID}
+		if !reflect.DeepEqual(actual, expected) {
+			t.Errorf("expected: %v, but got: %v", expected, actual)
+		}
+	})
+
+	t.Run("Filter", func(t *testing.T) {
+		actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
+		expected := []URI{uri0, uri2}
+		if !reflect.DeepEqual(actual, expected) {
+			t.Errorf("expected: %v, but got: %v", expected, actual)
+		}
+	})
+
+	t.Run("FilterURI", func(t *testing.T) {
+		actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
+		expected := []URI{uri0, uri2}
+		if !reflect.DeepEqual(actual, expected) {
+			t.Errorf("expected: %v, but got: %v", expected, actual)
+		}
+	})
+
+	t.Run("Contains", func(t *testing.T) {
+		actualTrue := Nodes(nodes).Contains(node1)
+		actualFalse := Nodes(nodes).Contains(node3)
+		if !reflect.DeepEqual(actualTrue, true) {
+			t.Errorf("expected: %v, but got: %v", true, actualTrue)
+		}
+		if !reflect.DeepEqual(actualFalse, false) {
+			t.Errorf("expected: %v, but got: %v", false, actualTrue)
+		}
+	})
+
+	t.Run("Clone", func(t *testing.T) {
+		clone := Nodes(nodes).Clone()
+		actual := Nodes(clone).URIs()
+		expected := []URI{uri0, uri1, uri2}
+		if !reflect.DeepEqual(actual, expected) {
+			t.Errorf("expected: %v, but got: %v", expected, actual)
+		}
+	})
+}
+
+// NEXT: move this test to internal and unexport IsCoordinator
+func TestCluster_Coordinator(t *testing.T) {
+	uri1 := NewTestURIFromHostPort("node1", 0)
+	uri2 := NewTestURIFromHostPort("node2", 0)
+
+	node1 := &Node{ID: "node1", URI: uri1}
+	node2 := &Node{ID: "node2", URI: uri2}
+
+	c1 := *newCluster()
+	c1.Node = node1
+	c1.Coordinator = node1.ID
+	c2 := *newCluster()
+	c2.Node = node2
+	c2.Coordinator = node1.ID
+
+	t.Run("IsCoordinator", func(t *testing.T) {
+		if !c1.isCoordinator() {
+			t.Errorf("!IsCoordinator error: %v", c1.Node)
+		} else if c2.isCoordinator() {
+			t.Errorf("IsCoordinator error: %v", c2.Node)
+		}
+	})
+}
+
+func TestCluster_Topology(t *testing.T) {
+	c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
+
+	uri0 := NewTestURIFromHostPort("host0", 0)
+	uri1 := NewTestURIFromHostPort("host1", 0)
+	uri2 := NewTestURIFromHostPort("host2", 0)
+	invalid := NewTestURIFromHostPort("invalid", 0)
+
+	node0 := &Node{ID: "node0", URI: uri0}
+	node1 := &Node{ID: "node1", URI: uri1}
+	node2 := &Node{ID: "node2", URI: uri2}
+	nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
+
+	t.Run("AddNode", func(t *testing.T) {
+		err := c1.addNode(node1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// add the same host.
+		err = c1.addNode(node1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = c1.addNode(node2)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		actual := c1.nodeIDs()
+		expected := []string{node0.ID, node1.ID, node2.ID}
+
+		if !reflect.DeepEqual(actual, expected) {
+			t.Errorf("expected: %v, but got: %v", expected, actual)
+		}
+	})
+
+	t.Run("ContainsID", func(t *testing.T) {
+		if !c1.Topology.ContainsID(node1.ID) {
+			t.Errorf("!ContainsHost error: %v", node1.ID)
+		} else if c1.Topology.ContainsID(nodeinvalid.ID) {
+			t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
+		}
+	})
+}
+
+// Ensure that general cluster functionality works as expected.
+func TestCluster_ResizeStates(t *testing.T) {
+
+	t.Run("Single node, no data", func(t *testing.T) {
+		tc := NewClusterCluster(1)
+
+		// Open TestCluster.
+		if err := tc.Open(); err != nil {
+			t.Fatal(err)
+		}
+
+		node := tc.Clusters[0]
+
+		// Ensure that node comes up in state NORMAL.
+		if node.State() != ClusterStateNormal {
+			t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
+		}
+
+		expectedTop := &Topology{
+			nodeIDs: []string{node.Node.ID},
+		}
+
+		// Verify topology file.
+		if !reflect.DeepEqual(node.Topology.nodeIDs, expectedTop.nodeIDs) {
+			t.Errorf("expected topology: %v, but got: %v", expectedTop.nodeIDs, node.Topology.nodeIDs)
+		}
+
+		// Close TestCluster.
+		if err := tc.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("Single node, in topology", func(t *testing.T) {
+		tc := NewClusterCluster(0)
+		tc.addNode()
+
+		node := tc.Clusters[0]
+
+		// write topology to data file
+		top := &Topology{
+			nodeIDs: []string{node.Node.ID},
+		}
+		tc.WriteTopology(node.Path, top)
+
+		// Open TestCluster.
+		if err := tc.Open(); err != nil {
+			t.Fatal(err)
+		}
+
+		// Ensure that node comes up in state NORMAL.
+		if node.State() != ClusterStateNormal {
+			t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
+		}
+
+		// Close TestCluster.
+		if err := tc.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("Single node, not in topology", func(t *testing.T) {
+		tc := NewClusterCluster(0)
+		tc.addNode()
+
+		node := tc.Clusters[0]
+
+		// write topology to data file
+		top := &Topology{
+			nodeIDs: []string{"some-other-host"},
+		}
+		tc.WriteTopology(node.Path, top)
+
+		// Open TestCluster.
+		expected := "coordinator node0 is not in topology: [some-other-host]"
+		err := tc.Open()
+		if err == nil || errors.Cause(err).Error() != expected {
+			t.Errorf("did not receive expected error, got: %s", errors.Cause(err).Error())
+		}
+
+		// Close TestCluster.
+		if err := tc.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("Multiple nodes, no data", func(t *testing.T) {
+		tc := NewClusterCluster(0)
+		tc.addNode()
+
+		// Open TestCluster.
+		if err := tc.Open(); err != nil {
+			t.Fatal(err)
+		}
+
+		tc.addNode()
+
+		node0 := tc.Clusters[0]
+		node1 := tc.Clusters[1]
+
+		// Ensure that nodes comes up in state NORMAL.
+		if node0.State() != ClusterStateNormal {
+			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
+		} else if node1.State() != ClusterStateNormal {
+			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
+		}
+
+		expectedTop := &Topology{
+			nodeIDs: []string{node0.Node.ID, node1.Node.ID},
+		}
+
+		// Verify topology file.
+		if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
+			t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
+		} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
+			t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
+		}
+
+		// Close TestCluster.
+		if err := tc.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
+		tc := NewClusterCluster(0)
+		tc.addNode()
+		node0 := tc.Clusters[0]
+
+		// write topology to data file
+		top := &Topology{
+			nodeIDs: []string{"node0", "node2"},
+		}
+		tc.WriteTopology(node0.Path, top)
+
+		// Open TestCluster.
+		if err := tc.Open(); err != nil {
+			t.Fatal(err)
+		}
+
+		// Ensure that node is in state STARTING before the other node joins.
+		if node0.State() != ClusterStateStarting {
+			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
+		}
+
+		// Expect an error by adding a node not in the topology.
+		expectedError := "host is not in topology: node1"
+		err := tc.addNode()
+		if err == nil || err.Error() != expectedError {
+			t.Errorf("did not receive expected error: %s", expectedError)
+		}
+
+		tc.addNode()
+		node2 := tc.Clusters[2]
+
+		// Ensure that node comes up in state NORMAL.
+		if node0.State() != ClusterStateNormal {
+			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
+		} else if node2.State() != ClusterStateNormal {
+			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State())
+		}
+
+		// Close TestCluster.
+		if err := tc.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("Multiple nodes, with data", func(t *testing.T) {
+		tc := NewClusterCluster(0)
+		tc.addNode()
+		node0 := tc.Clusters[0]
+
+		// Open TestCluster.
+		if err := tc.Open(); err != nil {
+			t.Fatal(err)
+		}
+
+		// Add Bit Data to node0.
+		if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil {
+			t.Fatal(err)
+		}
+		tc.SetBit("i", "f", 1, 101, nil)
+		tc.SetBit("i", "f", 1, 1300000, nil)
+
+		// Before starting the resize, get the CheckSum to use for
+		// comparison later.
+		node0Field := node0.holder.Field("i", "f")
+		node0View := node0Field.view("standard")
+		node0Fragment := node0View.Fragment(1)
+		node0Checksum := node0Fragment.Checksum()
+
+		// addNode needs to block until the resize process has completed.
+		tc.addNode()
+		node1 := tc.Clusters[1]
+
+		// Ensure that nodes come up in state NORMAL.
+		if node0.State() != ClusterStateNormal {
+			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
+		} else if node1.State() != ClusterStateNormal {
+			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
+		}
+
+		expectedTop := &Topology{
+			nodeIDs: []string{node0.Node.ID, node1.Node.ID},
+		}
+
+		// Verify topology file.
+		if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
+			t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
+		} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
+			t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
+		}
+
+		// Bits
+		// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
+		node1Field := node1.holder.Field("i", "f")
+		node1View := node1Field.view("standard")
+		node1Fragment := node1View.Fragment(1)
+
+		// Ensure checksums are the same.
+		if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
+			t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
+		}
+
+		// Close TestCluster.
+		if err := tc.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+}
+
+// Ensures that coordinator can be changed.
+func TestCluster_UpdateCoordinator(t *testing.T) {
+	t.Run("UpdateCoordinator", func(t *testing.T) {
+		c := NewTestCluster(2)
+
+		oldNode := c.Nodes[0]
+		newNode := c.Nodes[1]
+
+		// Update coordinator to the same value.
+		if c.updateCoordinator(oldNode) {
+			t.Errorf("did not expect coordinator to change")
+		} else if c.Coordinator != oldNode.ID {
+			t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
+		}
+
+		// Update coordinator to a new value.
+		if !c.updateCoordinator(newNode) {
+			t.Errorf("expected coordinator to change")
+		} else if c.Coordinator != newNode.ID {
+			t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
+		}
+	})
+}

cluster_test.go

@@ -1,528 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa
-
-import (
-	"bytes"
-	"math/rand"
-	"reflect"
-	"testing"
-	"testing/quick"
-
-	"github.com/davecgh/go-spew/spew"
-)
-
-// Ensure the cluster can fairly distribute partitions across the nodes.
-func TestCluster_Owners(t *testing.T) {
-	c := Cluster{
-		Nodes: []*Node{
-			{URI: NewTestURIFromHostPort("serverA", 1000)},
-			{URI: NewTestURIFromHostPort("serverB", 1000)},
-			{URI: NewTestURIFromHostPort("serverC", 1000)},
-		},
-		Hasher:   NewTestModHasher(),
-		ReplicaN: 2,
-	}
-
-	// Verify nodes are distributed.
-	if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) {
-		t.Fatalf("unexpected owners: %s", spew.Sdump(a))
-	}
-
-	// Verify nodes go around the ring.
-	if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) {
-		t.Fatalf("unexpected owners: %s", spew.Sdump(a))
-	}
-}
-
-// Ensure the partitioner can assign a fragment to a partition.
-func TestCluster_Partition(t *testing.T) {
-	if err := quick.Check(func(index string, slice uint64, partitionN int) bool {
-		c := NewCluster()
-		c.PartitionN = partitionN
-
-		partitionID := c.Partition(index, slice)
-		if partitionID < 0 || partitionID >= partitionN {
-			t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN)
-		}
-
-		return true
-	}, &quick.Config{
-		Values: func(values []reflect.Value, rand *rand.Rand) {
-			values[0], _ = quick.Value(reflect.TypeOf(""), rand)
-			values[1] = reflect.ValueOf(uint64(rand.Uint32()))
-			values[2] = reflect.ValueOf(rand.Intn(1000) + 1)
-		},
-	}); err != nil {
-		t.Fatal(err)
-	}
-}
-
-// Ensure the hasher can hash correctly.
-func TestHasher(t *testing.T) {
-	for _, tt := range []struct {
-		key    uint64
-		bucket []int
-	}{
-		// Generated from the reference C++ code
-		{0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
-		{1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}},
-		{0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}},
-		{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
-	} {
-		for i, v := range tt.bucket {
-			if got := NewHasher().Hash(tt.key, i+1); got != v {
-				t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
-			}
-		}
-	}
-}
-
-// Ensure OwnsSlices can find the actual slice list for node and index.
-func TestCluster_OwnsSlices(t *testing.T) {
-	c := NewTestCluster(5)
-	slices := c.OwnsSlices("test", 10, NewTestURIFromHostPort("host2", 0))
-
-	if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) {
-		t.Fatalf("unexpected slices for node's index: %v", slices)
-	}
-}
-
-// Ensure ContainsSlices can find the actual slice list for node and index.
-func TestCluster_ContainsSlices(t *testing.T) {
-	c := NewTestCluster(5)
-	c.ReplicaN = 3
-	slices := c.ContainsSlices("test", 10, c.Nodes[2])
-
-	if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) {
-		t.Fatalf("unexpected slices for node's index: %v", slices)
-	}
-}
-
-func TestCluster_Nodes(t *testing.T) {
-	uri0 := NewTestURIFromHostPort("node0", 0)
-	uri1 := NewTestURIFromHostPort("node1", 0)
-	uri2 := NewTestURIFromHostPort("node2", 0)
-	uri3 := NewTestURIFromHostPort("node3", 0)
-
-	node0 := &Node{ID: "node0", URI: uri0}
-	node1 := &Node{ID: "node1", URI: uri1}
-	node2 := &Node{ID: "node2", URI: uri2}
-	node3 := &Node{ID: "node3", URI: uri3}
-
-	nodes := []*Node{node0, node1, node2}
-
-	t.Run("NodeIDs", func(t *testing.T) {
-		actual := Nodes(nodes).IDs()
-		expected := []string{node0.ID, node1.ID, node2.ID}
-		if !reflect.DeepEqual(actual, expected) {
-			t.Errorf("expected: %v, but got: %v", expected, actual)
-		}
-	})
-
-	t.Run("Filter", func(t *testing.T) {
-		actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
-		expected := []URI{uri0, uri2}
-		if !reflect.DeepEqual(actual, expected) {
-			t.Errorf("expected: %v, but got: %v", expected, actual)
-		}
-	})
-
-	t.Run("FilterURI", func(t *testing.T) {
-		actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
-		expected := []URI{uri0, uri2}
-		if !reflect.DeepEqual(actual, expected) {
-			t.Errorf("expected: %v, but got: %v", expected, actual)
-		}
-	})
-
-	t.Run("Contains", func(t *testing.T) {
-		actualTrue := Nodes(nodes).Contains(node1)
-		actualFalse := Nodes(nodes).Contains(node3)
-		if !reflect.DeepEqual(actualTrue, true) {
-			t.Errorf("expected: %v, but got: %v", true, actualTrue)
-		}
-		if !reflect.DeepEqual(actualFalse, false) {
-			t.Errorf("expected: %v, but got: %v", false, actualTrue)
-		}
-	})
-
-	t.Run("Clone", func(t *testing.T) {
-		clone := Nodes(nodes).Clone()
-		actual := Nodes(clone).URIs()
-		expected := []URI{uri0, uri1, uri2}
-		if !reflect.DeepEqual(actual, expected) {
-			t.Errorf("expected: %v, but got: %v", expected, actual)
-		}
-	})
-}
-
-func TestCluster_Coordinator(t *testing.T) {
-	uri1 := NewTestURIFromHostPort("node1", 0)
-	uri2 := NewTestURIFromHostPort("node2", 0)
-
-	node1 := &Node{ID: "node1", URI: uri1}
-	node2 := &Node{ID: "node2", URI: uri2}
-
-	c1 := *NewCluster()
-	c1.Node = node1
-	c1.Coordinator = node1.ID
-	c2 := *NewCluster()
-	c2.Node = node2
-	c2.Coordinator = node1.ID
-
-	t.Run("IsCoordinator", func(t *testing.T) {
-		if !c1.IsCoordinator() {
-			t.Errorf("!IsCoordinator error: %v", c1.Node)
-		} else if c2.IsCoordinator() {
-			t.Errorf("IsCoordinator error: %v", c2.Node)
-		}
-	})
-}
-
-func TestCluster_Topology(t *testing.T) {
-	c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
-
-	uri0 := NewTestURIFromHostPort("host0", 0)
-	uri1 := NewTestURIFromHostPort("host1", 0)
-	uri2 := NewTestURIFromHostPort("host2", 0)
-	invalid := NewTestURIFromHostPort("invalid", 0)
-
-	node0 := &Node{ID: "node0", URI: uri0}
-	node1 := &Node{ID: "node1", URI: uri1}
-	node2 := &Node{ID: "node2", URI: uri2}
-	nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
-
-	t.Run("AddNode", func(t *testing.T) {
-		err := c1.AddNode(node1)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// add the same host.
-		err = c1.AddNode(node1)
-		if err != nil {
-			t.Fatal(err)
-		}
-		err = c1.AddNode(node2)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		actual := c1.NodeIDs()
-		expected := []string{node0.ID, node1.ID, node2.ID}
-
-		if !reflect.DeepEqual(actual, expected) {
-			t.Errorf("expected: %v, but got: %v", expected, actual)
-		}
-	})
-
-	t.Run("ContainsID", func(t *testing.T) {
-		if !c1.Topology.ContainsID(node1.ID) {
-			t.Errorf("!ContainsHost error: %v", node1.ID)
-		} else if c1.Topology.ContainsID(nodeinvalid.ID) {
-			t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
-		}
-	})
-}
-
-// Ensure that general cluster functionality works as expected.
-func TestCluster_ResizeStates(t *testing.T) {
-
-	t.Run("Single node, no data", func(t *testing.T) {
-		tc := NewClusterCluster(1)
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		node := tc.Clusters[0]
-
-		// Ensure that node comes up in state NORMAL.
-		if node.State() != ClusterStateNormal {
-			t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
-		}
-
-		expectedTop := &Topology{
-			NodeIDs: []string{node.Node.ID},
-		}
-
-		// Verify topology file.
-		if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) {
-			t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs)
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Single node, in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		tc.AddNode(false)
-
-		node := tc.Clusters[0]
-
-		// write topology to data file
-		top := &Topology{
-			NodeIDs: []string{node.Node.ID},
-		}
-		tc.WriteTopology(node.Path, top)
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		// Ensure that node comes up in state NORMAL.
-		if node.State() != ClusterStateNormal {
-			t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Single node, not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		tc.AddNode(false)
-
-		node := tc.Clusters[0]
-
-		// write topology to data file
-		top := &Topology{
-			NodeIDs: []string{"some-other-host"},
-		}
-		tc.WriteTopology(node.Path, top)
-
-		// Open TestCluster.
-		expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]"
-		err := tc.Open()
-		if err == nil || err.Error() != expected {
-			t.Errorf("did not receive expected error: %s", expected)
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Multiple nodes, no data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		tc.AddNode(false)
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		tc.AddNode(false)
-
-		node0 := tc.Clusters[0]
-		node1 := tc.Clusters[1]
-
-		// Ensure that nodes comes up in state NORMAL.
-		if node0.State() != ClusterStateNormal {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
-		} else if node1.State() != ClusterStateNormal {
-			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
-		}
-
-		expectedTop := &Topology{
-			NodeIDs: []string{node0.Node.ID, node1.Node.ID},
-		}
-
-		// Verify topology file.
-		if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
-			t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
-		} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
-			t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		tc.AddNode(false)
-		node0 := tc.Clusters[0]
-
-		// write topology to data file
-		top := &Topology{
-			NodeIDs: []string{"node0", "node2"},
-		}
-		tc.WriteTopology(node0.Path, top)
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		// Ensure that node is in state STARTING before the other node joins.
-		if node0.State() != ClusterStateStarting {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
-		}
-
-		// Expect an error by adding a node not in the topology.
-		expectedError := "host is not in topology: node1"
-		err := tc.AddNode(false)
-		if err == nil || err.Error() != expectedError {
-			t.Errorf("did not receive expected error: %s", expectedError)
-		}
-
-		tc.AddNode(false)
-		node2 := tc.Clusters[2]
-
-		// Ensure that node comes up in state NORMAL.
-		if node0.State() != ClusterStateNormal {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
-		} else if node2.State() != ClusterStateNormal {
-			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State())
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Multiple nodes, with data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		tc.AddNode(false)
-		node0 := tc.Clusters[0]
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		// Add Bit Data to node0.
-		if err := tc.CreateFrame("i", "f", FrameOptions{}); err != nil {
-			t.Fatal(err)
-		}
-		tc.SetBit("i", "f", "standard", 1, 101, nil)
-		tc.SetBit("i", "f", "standard", 1, 1300000, nil)
-
-		// Add Field Data to node0.
-		if err := tc.CreateFrame("i", "fields", FrameOptions{
-			Fields: []*Field{
-				{
-					Name: "fld0",
-					Type: FieldTypeInt,
-					Min:  -100,
-					Max:  100,
-				},
-			},
-		}); err != nil {
-			t.Fatal(err)
-		}
-		tc.SetFieldValue("i", "fields", 1, "fld0", -10)
-		tc.SetFieldValue("i", "fields", 1, "fld0", 10)
-		tc.SetFieldValue("i", "fields", 1300000, "fld0", -99)
-		tc.SetFieldValue("i", "fields", 1300000, "fld0", 99)
-
-		// Before starting the resize, get the CheckSum to use for
-		// comparison later.
-		node0Frame := node0.Holder.Frame("i", "f")
-		node0View := node0Frame.View("standard")
-		node0Fragment := node0View.Fragment(1)
-		node0Checksum := node0Fragment.Checksum()
-
-		node0Frame = node0.Holder.Frame("i", "fields")
-		node0View = node0Frame.View("field_fld0")
-		node0Fragment = node0View.Fragment(1)
-		node0ChecksumFld := node0Fragment.Checksum()
-
-		// AddNode needs to block until the resize process has completed.
-		tc.AddNode(false)
-		node1 := tc.Clusters[1]
-
-		// Ensure that nodes come up in state NORMAL.
-		if node0.State() != ClusterStateNormal {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
-		} else if node1.State() != ClusterStateNormal {
-			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
-		}
-
-		expectedTop := &Topology{
-			NodeIDs: []string{node0.Node.ID, node1.Node.ID},
-		}
-
-		// Verify topology file.
-		if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
-			t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
-		} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
-			t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
-		}
-
-		// Bits
-		// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
-		node1Frame := node1.Holder.Frame("i", "f")
-		node1View := node1Frame.View("standard")
-		node1Fragment := node1View.Fragment(1)
-
-		// Ensure checksums are the same.
-		if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
-			t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
-		}
-
-		// Values
-		// Verify that node-1 contains the fragment (i/fields/field_fld0/1) transferred from node-0.
-		node1Frame = node1.Holder.Frame("i", "fields")
-		node1View = node1Frame.View("field_fld0")
-		node1Fragment = node1View.Fragment(1)
-
-		// Ensure checksums are the same.
-		if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0ChecksumFld) {
-			t.Fatalf("expected checksum to match: %x - %x", chksum, node0ChecksumFld)
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-}
-
-// Ensures that coordinator can be changed.
-func TestCluster_UpdateCoordinator(t *testing.T) {
-	t.Run("UpdateCoordinator", func(t *testing.T) {
-		c := NewTestCluster(2)
-
-		oldNode := c.Nodes[0]
-		newNode := c.Nodes[1]
-
-		// Update coordinator to the same value.
-		if c.UpdateCoordinator(oldNode) {
-			t.Errorf("did not expect coordinator to change")
-		} else if c.Coordinator != oldNode.ID {
-			t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
-		}
-
-		// Update coordinator to a new value.
-		if !c.UpdateCoordinator(newNode) {
-			t.Errorf("expected coordinator to change")
-		} else if c.Coordinator != newNode.ID {
-			t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
-		}
-	})
-}

cmd/bench.go

@@ -1,57 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package cmd
-
-import (
-	"context"
-	"io"
-	"os"
-
-	"github.com/spf13/cobra"
-
-	"github.com/pilosa/pilosa/ctl"
-)
-
-var Bencher *ctl.BenchCommand
-
-func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
-	Bencher = ctl.NewBenchCommand(os.Stdin, os.Stdout, os.Stderr)
-	benchCmd := &cobra.Command{
-		Use:   "bench",
-		Short: "Benchmark operations.",
-		Long: `
-Executes a benchmark for a given operation against the index.
-`,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := Bencher.Run(context.Background()); err != nil {
-				return err
-			}
-			return nil
-		},
-	}
-	flags := benchCmd.Flags()
-	flags.StringVarP(&Bencher.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
-	flags.StringVarP(&Bencher.Index, "index", "i", "", "Pilosa index to benchmark.")
-	flags.StringVarP(&Bencher.Frame, "frame", "f", "", "Frame to benchmark.")
-	flags.StringVarP(&Bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]")
-	flags.IntVarP(&Bencher.N, "num", "n", 0, "Number of operations to perform.")
-	ctl.SetTLSConfig(flags, &Bencher.TLS.CertificatePath, &Bencher.TLS.CertificateKeyPath, &Bencher.TLS.SkipVerify)
-
-	return benchCmd
-}
-
-func init() {
-	subcommandFns["bench"] = NewBenchCommand
-}

cmd/bench_test.go

@@ -1,54 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package cmd_test
-
-import (
-	"strings"
-	"testing"
-
-	"github.com/pilosa/pilosa/cmd"
-)
-
-func TestBenchHelp(t *testing.T) {
-	output, err := ExecNewRootCommand(t, "bench", "--help")
-	if !strings.Contains(output, "Usage:") ||
-		!strings.Contains(output, "Flags:") ||
-		!strings.Contains(output, "pilosa bench") || err != nil {
-		t.Fatalf("Command 'bench --help' not working, err: '%v', output: '%s'", err, output)
-	}
-}
-
-func TestBenchConfig(t *testing.T) {
-	tests := []commandTest{
-		{
-			args: []string{"bench", "--operation", "set-bit"},
-			env:  map[string]string{"PILOSA_HOST": "localhost:12345"},
-			cfgFileContent: `
-index = "myindex"
-frame = "f1"
-`,
-			validation: func() error {
-				v := validator{}
-				v.Check(cmd.Bencher.Host, "localhost:12345")
-				v.Check(cmd.Bencher.Index, "myindex")
-				v.Check(cmd.Bencher.Frame, "f1")
-				v.Check(cmd.Bencher.Op, "set-bit")
-				v.Check(cmd.Bencher.N, 0)
-				return v.Error()
-			},
-		},
-	}
-	executeDry(t, tests)
-}

cmd/check.go

@@ -25,10 +25,10 @@ import (
 	"github.com/pilosa/pilosa/ctl"
 )
 
-var Checker *ctl.CheckCommand
+var checker *ctl.CheckCommand
 
-func NewCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
-	Checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr)
+func newCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr)
 	checkCmd := &cobra.Command{
 		Use:   "check <path> [path2]...",
 		Short: "Do a consistency check on a pilosa data file.",
@@ -39,8 +39,8 @@ Performs a consistency check on data files.
 			if len(args) == 0 {
 				return fmt.Errorf("path required")
 			}
-			Checker.Paths = args
-			if err := Checker.Run(context.Background()); err != nil {
+			checker.Paths = args
+			if err := checker.Run(context.Background()); err != nil {
 				return err
 			}
 			return nil
@@ -50,5 +50,5 @@ Performs a consistency check on data files.
 }
 
 func init() {
-	subcommandFns["check"] = NewCheckCommand
+	subcommandFns["check"] = newCheckCommand
 }

cmd/config.go

@@ -25,10 +25,10 @@ import (
 	"github.com/pilosa/pilosa/server"
 )
 
-var Conf *ctl.ConfigCommand
+var conf *ctl.ConfigCommand
 
-func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
-	Conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr)
+func newConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr)
 	Server := server.NewCommand(stdin, stdout, stderr)
 	confCmd := &cobra.Command{
 		Use:   "config",
@@ -36,8 +36,8 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
 		Long:  `config prints the current configuration to stdout`,
 
 		RunE: func(cmd *cobra.Command, args []string) error {
-			Conf.Config = Server.Config
-			if err := Conf.Run(context.Background()); err != nil {
+			conf.Config = Server.Config
+			if err := conf.Run(context.Background()); err != nil {
 				return err
 			}
 			return nil
@@ -51,5 +51,5 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
 }
 
 func init() {
-	subcommandFns["config"] = NewConfigCommand
+	subcommandFns["config"] = newConfigCommand
 }

cmd/export.go

@@ -26,7 +26,7 @@ import (
 
 var Exporter *ctl.ExportCommand
 
-func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+func newExportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
 	Exporter = ctl.NewExportCommand(os.Stdin, os.Stdout, os.Stderr)
 	exportCmd := &cobra.Command{
 		Use:   "export",
@@ -52,7 +52,7 @@ The file does not contain any headers.
 
 	flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
 	flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export")
-	flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export")
+	flags.StringVarP(&Exporter.Field, "field", "f", "", "Field to export")
 	flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
 	ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.SkipVerify)
 
@@ -60,5 +60,5 @@ The file does not contain any headers.
 }
 
 func init() {
-	subcommandFns["export"] = NewExportCommand
+	subcommandFns["export"] = newExportCommand
 }

cmd/export_test.go

@@ -37,13 +37,13 @@ func TestExportConfig(t *testing.T) {
 			env:  map[string]string{"PILOSA_HOST": "localhost:12345"},
 			cfgFileContent: `
 index = "myindex"
-frame = "f1"
+field = "f1"
 `,
 			validation: func() error {
 				v := validator{}
 				v.Check(cmd.Exporter.Host, "localhost:12345")
 				v.Check(cmd.Exporter.Index, "myindex")
-				v.Check(cmd.Exporter.Frame, "f1")
+				v.Check(cmd.Exporter.Field, "f1")
 				v.Check(cmd.Exporter.Path, "/somefile")
 				return v.Error()
 			},

cmd/generate_config.go

@@ -24,17 +24,17 @@ import (
 	"github.com/pilosa/pilosa/ctl"
 )
 
-var GenerateConf *ctl.GenerateConfigCommand
+var generateConf *ctl.GenerateConfigCommand
 
-func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
-	GenerateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr)
+func newGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	generateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr)
 	confCmd := &cobra.Command{
 		Use:   "generate-config",
 		Short: "Print the default configuration.",
 		Long: `generate-config prints the default configuration to stdout
 `,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := GenerateConf.Run(context.Background()); err != nil {
+			if err := generateConf.Run(context.Background()); err != nil {
 				return err
 			}
 			return nil
@@ -45,5 +45,5 @@ func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.
 }
 
 func init() {
-	subcommandFns["generate-config"] = NewGenerateConfigCommand
+	subcommandFns["generate-config"] = newGenerateConfigCommand
 }

cmd/import.go

@@ -20,20 +20,19 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/ctl"
 )
 
 var Importer *ctl.ImportCommand
 
-// NewImportCommand runs the Pilosa import subcommand for ingesting bulk data.
-func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+// newImportCommand runs the Pilosa import subcommand for ingesting bulk data.
+func newImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
 	Importer = ctl.NewImportCommand(stdin, stdout, stderr)
 	importCmd := &cobra.Command{
 		Use:   "import",
 		Short: "Bulk load data into pilosa.",
-		Long: `Bulk imports one or more CSV files to a host's index and frame. The data
-of the CSV file are grouped by slice for the most efficient import.
+		Long: `Bulk imports one or more CSV files to a host's index and field. The data
+of the CSV file are grouped by shard for the most efficient import.
 
 The format of the CSV file is:
 
@@ -54,20 +53,19 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags := importCmd.Flags()
 	flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
 	flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.")
-	flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.")
-	flags.StringVarP(&Importer.Field, "field", "", "", "Field to import into.")
+	flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
 	flags.BoolVar(&Importer.StringKeys, "string-keys", false, "Treat payload as string keys.")
 	flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
 	flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
 	flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
-	flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame")
-	flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked")
-	flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame")
+	//flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Time quantum for the field")
+	//flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Cache type for the field; valid values: none, lru, ranked")
+	//flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Cache size for the field")
 	ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify)
 
 	return importCmd
 }
 
 func init() {
-	subcommandFns["import"] = NewImportCommand
+	subcommandFns["import"] = newImportCommand
 }

cmd/import_test.go

@@ -37,13 +37,13 @@ func TestImportConfig(t *testing.T) {
 			env:  map[string]string{"PILOSA_HOST": "localhost:12345"},
 			cfgFileContent: `
 index = "myindex"
-frame = "f1"
+field = "f1"
 `,
 			validation: func() error {
 				v := validator{}
 				v.Check(cmd.Importer.Host, "localhost:12345")
 				v.Check(cmd.Importer.Index, "myindex")
-				v.Check(cmd.Importer.Frame, "f1")
+				v.Check(cmd.Importer.Field, "f1")
 				return v.Error()
 			},
 		},

cmd/inspect.go

@@ -25,10 +25,10 @@ import (
 	"github.com/pilosa/pilosa/ctl"
 )
 
-var Inspector *ctl.InspectCommand
+var inspector *ctl.InspectCommand
 
-func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
-	Inspector = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr)
+func newInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	inspector = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr)
 
 	inspectCmd := &cobra.Command{
 		Use:   "inspect",
@@ -42,8 +42,8 @@ Inspects a data file and provides stats.
 			} else if len(args) > 1 {
 				return fmt.Errorf("only one path allowed")
 			}
-			Inspector.Path = args[0]
-			if err := Inspector.Run(context.Background()); err != nil {
+			inspector.Path = args[0]
+			if err := inspector.Run(context.Background()); err != nil {
 				return err
 			}
 			return nil
@@ -53,5 +53,5 @@ Inspects a data file and provides stats.
 }
 
 func init() {
-	subcommandFns["inspect"] = NewInspectCommand
+	subcommandFns["inspect"] = newInspectCommand
 }

cmd/server.go

@@ -27,8 +27,8 @@ import (
 // Server is global so that tests can control and verify it.
 var Server *server.Command
 
-// NewServeCmd creates a pilosa server and runs it with command line flags.
-func NewServeCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+// newServeCmd creates a pilosa server and runs it with command line flags.
+func newServeCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
 	Server = server.NewCommand(stdin, stdout, stderr)
 	serveCmd := &cobra.Command{
 		Use:   "server",
@@ -52,5 +52,5 @@ on the configured port.`,
 }
 
 func init() {
-	subcommandFns["server"] = NewServeCmd
+	subcommandFns["server"] = newServeCmd
 }

cmd/server_test.go

@@ -15,7 +15,6 @@
 package cmd_test
 
 import (
-	"errors"
 	"io/ioutil"
 	"strings"
 	"testing"
@@ -24,6 +23,7 @@ import (
 	"github.com/pilosa/pilosa/cmd"
 	_ "github.com/pilosa/pilosa/test"
 	"github.com/pilosa/pilosa/toml"
+	"github.com/pkg/errors"
 )
 
 func TestServerHelp(t *testing.T) {

ctl/bench.go

@@ -1,116 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ctl
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"math/rand"
-	"time"
-
-	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/internal"
-	"github.com/pilosa/pilosa/server"
-	"github.com/pkg/errors"
-)
-
-// BenchCommand represents a command for benchmarking index operations.
-type BenchCommand struct {
-	// Destination host and port.
-	Host string
-
-	// Name of the index & frame to execute against.
-	Index string
-	Frame string
-
-	// Type of operation and number to execute.
-	Op string
-	N  int
-
-	// Standard input/output
-	*pilosa.CmdIO
-
-	TLS server.TLSConfig
-}
-
-// NewBenchCommand returns a new instance of BenchCommand.
-func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *BenchCommand {
-	return &BenchCommand{
-		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
-	}
-}
-
-// Run executes the bench command.
-func (cmd *BenchCommand) Run(ctx context.Context) error {
-	// Create a client to the server.
-	client, err := CommandClient(cmd)
-	if err != nil {
-		return errors.Wrap(err, "creating client")
-	}
-
-	switch cmd.Op {
-	case "set-bit":
-		return cmd.runSetBit(ctx, client)
-	case "":
-		return errors.New("op required")
-	default:
-		return fmt.Errorf("unknown bench op: %q", cmd.Op)
-	}
-}
-
-// runSetBit executes a benchmark of random SetBit() operations.
-func (cmd *BenchCommand) runSetBit(ctx context.Context, client pilosa.InternalClient) error {
-	if cmd.N == 0 {
-		return errors.New("operation count required")
-	} else if cmd.Index == "" {
-		return pilosa.ErrIndexRequired
-	} else if cmd.Frame == "" {
-		return pilosa.ErrFrameRequired
-	}
-
-	const maxRowID = 1000
-	const maxColumnID = 100000
-
-	startTime := time.Now()
-
-	// Execute operation continuously.
-	for i := 0; i < cmd.N; i++ {
-		rowID := rand.Intn(maxRowID)
-		columnID := rand.Intn(maxColumnID)
-
-		queryRequest := &internal.QueryRequest{
-			Query:  fmt.Sprintf(`SetBit(row=%d, frame="%s", col=%d)`, rowID, cmd.Frame, columnID),
-			Remote: false,
-		}
-		if _, err := client.Query(ctx, cmd.Index, queryRequest); err != nil {
-			return err
-		}
-	}
-
-	// Print results.
-	elapsed := time.Since(startTime)
-	fmt.Fprintf(cmd.Stdout, "Executed %d operations in %s (%0.3f op/sec)\n", cmd.N, elapsed, float64(cmd.N)/elapsed.Seconds())
-
-	return nil
-}
-
-func (cmd *BenchCommand) TLSHost() string {
-	return cmd.Host
-}
-
-func (cmd *BenchCommand) TLSConfiguration() server.TLSConfig {
-	return cmd.TLS
-}

ctl/bench_test.go

@@ -1,98 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ctl
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"fmt"
-	"io"
-	"os"
-	"testing"
-
-	"github.com/pilosa/pilosa"
-	"github.com/pkg/errors"
-)
-
-func TestBenchCommand_InvalidOption(t *testing.T) {
-	buf := bytes.Buffer{}
-	stdin, stdout, stderr := GetIO(buf)
-
-	cm := NewBenchCommand(stdin, stdout, stderr)
-	err := cm.Run(context.Background())
-	if errors.Cause(err) != pilosa.ErrHostRequired {
-		t.Fatalf("Expect err: %s, actual err: %s", pilosa.ErrHostRequired, err)
-	}
-
-	cm.Host = "localhost:10101"
-	err = cm.Run(context.Background())
-	if err.Error() != "op required" {
-		t.Fatalf("Expect err: %s, actual err: %s", "op required", err)
-	}
-
-	cm.Op = "test"
-	err = cm.Run(context.Background())
-	if err.Error() != "unknown bench op: \"test\"" {
-		t.Fatalf("Expect err: %s, actual err: %s", "unknown bench op: test", err)
-	}
-
-}
-
-func TestBenchCommand_Run(t *testing.T) {
-	rder := []byte{}
-	stdin := bytes.NewReader(rder)
-	r, w, _ := os.Pipe()
-
-	cm := NewBenchCommand(stdin, w, w)
-	cm.Op = "set-bit"
-	cm.Host = "localhost:10101"
-
-	err := cm.Run(context.Background())
-	if err.Error() != "operation count required" {
-		t.Fatalf("Expect error: %s, actual err: %s", "operation count required", err)
-	}
-
-	cm.N = 1
-	err = cm.Run(context.Background())
-	if err != pilosa.ErrIndexRequired {
-		t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrIndexRequired, err)
-	}
-
-	cm.Index = "i"
-	err = cm.Run(context.Background())
-	if err != pilosa.ErrFrameRequired {
-		t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrFrameRequired, err)
-	}
-
-	cm.Frame = "f"
-	err = cm.Run(context.Background())
-	w.Close()
-	var buf bytes.Buffer
-	io.Copy(&buf, r)
-	fmt.Println(buf.String())
-	if err != nil {
-		fmt.Println(buf.String())
-	}
-}
-
-// declare stdin, stdout, stderr
-func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
-	rder := []byte{}
-	stdin := bytes.NewReader(rder)
-	stdout := bufio.NewWriter(&buf)
-	stderr := bufio.NewWriter(&buf)
-	return stdin, stdout, stderr
-}

ctl/common.go

@@ -17,7 +17,7 @@ package ctl
 import (
 	"crypto/tls"
 
-	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/http"
 	"github.com/pilosa/pilosa/server"
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
@@ -36,8 +36,8 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP
 	flags.BoolVarP(skipVerify, "tls.skip-verify", "", false, "Skip TLS certificate verification (not secure)")
 }
 
-// CommandClient returns a pilosa.InternalHTTPClient for the command
-func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error) {
+// commandClient returns a pilosa.InternalHTTPClient for the command
+func commandClient(cmd CommandWithTLSSupport) (*http.InternalClient, error) {
 	tlsConfig := cmd.TLSConfiguration()
 	var TLSConfig *tls.Config
 	if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" {
@@ -50,7 +50,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error
 			InsecureSkipVerify: tlsConfig.SkipVerify,
 		}
 	}
-	client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig))
+	client, err := http.NewInternalClient(cmd.TLSHost(), http.GetHTTPClient(TLSConfig))
 	if err != nil {
 		return nil, errors.Wrap(err, "getting internal client")
 	}

ctl/export.go

@@ -30,9 +30,9 @@ type ExportCommand struct {
 	// Remote host and port.
 	Host string
 
-	// Name of the index & frame to export from.
+	// Name of the index & field to export from.
 	Index string
-	Frame string
+	Field string
 
 	// Filename to export to.
 	Path string
@@ -57,8 +57,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
 	// Validate arguments.
 	if cmd.Index == "" {
 		return pilosa.ErrIndexRequired
-	} else if cmd.Frame == "" {
-		return pilosa.ErrFrameRequired
+	} else if cmd.Field == "" {
+		return pilosa.ErrFieldRequired
 	}
 
 	// Use output file, if specified.
@@ -75,21 +75,21 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
 	}
 
 	// Create a client to the server.
-	client, err := CommandClient(cmd)
+	client, err := commandClient(cmd)
 	if err != nil {
 		return errors.Wrap(err, "creating client")
 	}
 
-	// Determine slice count.
-	maxSlices, err := client.MaxSliceByIndex(ctx)
+	// Determine shard count.
+	maxShards, err := client.MaxShardByIndex(ctx)
 	if err != nil {
-		return errors.Wrap(err, "getting slice count")
+		return errors.Wrap(err, "getting shard count")
 	}
 
-	// Export each slice.
-	for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ {
-		logger.Printf("exporting slice: %d", slice)
-		if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil {
+	// Export each shard.
+	for shard := uint64(0); shard <= maxShards[cmd.Index]; shard++ {
+		logger.Printf("exporting shard: %d", shard)
+		if err := client.ExportCSV(ctx, cmd.Index, cmd.Field, shard, w); err != nil {
 			return errors.Wrap(err, "exporting")
 		}
 	}

ctl/export_test.go

@@ -38,31 +38,25 @@ func TestExportCommand_Validation(t *testing.T) {
 
 	cm.Index = "i"
 	err = cm.Run(context.Background())
-	if err != pilosa.ErrFrameRequired {
-		t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err)
+	if err != pilosa.ErrFieldRequired {
+		t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFieldRequired, err)
 	}
 }
 
 func TestExportCommand_Run(t *testing.T) {
+	cmd := test.MustRunCluster(t, 1)[0]
+
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewExportCommand(stdin, stdout, stderr)
+	hostport := cmd.API.Node().URI.HostPort()
+	cm.Host = hostport
 
-	hldr := test.MustOpenHolder()
-	defer hldr.Close()
-	s := test.NewServer()
-	defer s.Close()
-
-	s.Handler.API.Cluster = test.NewCluster(1)
-	s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
-	s.Handler.API.Holder = hldr.Holder
-	cm.Host = s.Host()
-
-	http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
-	http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader("")))
+	http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader("")))
+	http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i/field/f", strings.NewReader("")))
 
 	cm.Index = "i"
-	cm.Frame = "f"
+	cm.Field = "f"
 	if err := cm.Run(context.Background()); err != nil {
 		t.Fatalf("Export Run doesn't work: %s", err)
 	}

ctl/generate_config.go

@@ -18,9 +18,11 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"strings"
 
+	"github.com/pelletier/go-toml"
 	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/server"
+	"github.com/pkg/errors"
 )
 
 // GenerateConfigCommand represents a command for printing a default config.
@@ -37,28 +39,11 @@ func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *Genera
 
 // Run prints out the default config.
 func (cmd *GenerateConfigCommand) Run(ctx context.Context) error {
-	fmt.Fprintln(cmd.Stdout, strings.TrimSpace(`
-data-dir = "~/.pilosa"
-bind = "localhost:10101"
-max-writes-per-request = 5000
-
-[cluster]
-  replicas = 1
-  hosts = [
-    "localhost:10101",
-  ]
-
-[anti-entropy]
-  interval = "10m0s"
-
-[profile]
-  cpu = ""
-  cpu-time = "30s"
-
-[metric]
-  service = "statsd"
-  host = "127.0.0.1:8125"
-  poll-interval = "0m15s"
-`)+"\n")
+	conf := server.NewConfig()
+	ret, err := toml.Marshal(*conf)
+	if err != nil {
+		return errors.Wrap(err, "unmarshaling default config")
+	}
+	fmt.Fprintf(cmd.Stdout, "%s\n", ret)
 	return nil
 }

ctl/generate_config_test.go

@@ -34,7 +34,7 @@ func TestGenerateConfigCommand_Run(t *testing.T) {
 	io.Copy(&buf, r)
 	if err != nil {
 		t.Fatalf("Config Run doesn't work: %s", err)
-	} else if !strings.Contains(buf.String(), "localhost:10101") {
+	} else if !strings.Contains(buf.String(), ":10101") {
 		t.Fatalf("Unexpected config: %s", buf.String())
 	}
 }

ctl/import.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/http"
 	"github.com/pilosa/pilosa/server"
 	"github.com/pkg/errors"
 )
@@ -35,20 +36,16 @@ type ImportCommand struct {
 	// Destination host and port.
 	Host string `json:"host"`
 
-	// Name of the index & frame to import into.
+	// Name of the index & field to import into.
 	Index string `json:"index"`
-	Frame string `json:"frame"`
+	Field string `json:"field"`
 
-	// Options for index & frame to be created if they don't exist
-	IndexOptions pilosa.IndexOptions
-	FrameOptions pilosa.FrameOptions
+	// Options for index & field to be created if they don't exist
+	indexOptions pilosa.IndexOptions
 
 	// CreateSchema ensures the schema exists before import
 	CreateSchema bool
 
-	// For Range-Encoded fields, name of the Field to import into.
-	Field string `json:"field"`
-
 	// Indicates that the payload should be treated as string keys.
 	StringKeys bool `json:"StringKeys"`
 
@@ -62,7 +59,7 @@ type ImportCommand struct {
 	Sort bool `json:"sort"`
 
 	// Reusable client.
-	Client pilosa.InternalClient `json:"-"`
+	client pilosa.InternalClient `json:"-"`
 
 	// Standard input/output
 	*pilosa.CmdIO
@@ -83,20 +80,20 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 	logger := log.New(cmd.Stderr, "", log.LstdFlags)
 
 	// Validate arguments.
-	// Index and frame are validated early before the files are parsed.
+	// Index and field are validated early before the files are parsed.
 	if cmd.Index == "" {
 		return pilosa.ErrIndexRequired
-	} else if cmd.Frame == "" {
-		return pilosa.ErrFrameRequired
+	} else if cmd.Field == "" {
+		return pilosa.ErrFieldRequired
 	} else if len(cmd.Paths) == 0 {
 		return errors.New("path required")
 	}
 	// Create a client to the server.
-	client, err := CommandClient(cmd)
+	client, err := commandClient(cmd)
 	if err != nil {
 		return errors.Wrap(err, "creating client")
 	}
-	cmd.Client = client
+	cmd.client = client
 
 	if cmd.CreateSchema {
 		err := cmd.ensureSchema(ctx)
@@ -105,10 +102,26 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 		}
 	}
 
-	// Import each path and import by slice.
+	// Determine the field type in order to correctly handle the input data.
+	fieldType := pilosa.DefaultFieldType
+	schema, err := cmd.client.Schema(ctx)
+	if err != nil {
+		return errors.Wrap(err, "getting schema")
+	}
+	for _, index := range schema {
+		if index.Name == cmd.Index {
+			for _, field := range index.Fields {
+				if field.Name == cmd.Field {
+					fieldType = field.Options.Type
+				}
+			}
+		}
+	}
+
+	// Import each path and import by shard.
 	for _, path := range cmd.Paths {
 		logger.Printf("parsing: %s", path)
-		if err := cmd.importPath(ctx, path); err != nil {
+		if err := cmd.importPath(ctx, fieldType, path); err != nil {
 			return err
 		}
 	}
@@ -117,22 +130,22 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 }
 
 func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
-	err := cmd.Client.EnsureIndex(ctx, cmd.Index, cmd.IndexOptions)
+	err := cmd.client.EnsureIndex(ctx, cmd.Index, cmd.indexOptions)
 	if err != nil {
 		return fmt.Errorf("Error Creating Index: %s", err)
 	}
-	err = cmd.Client.EnsureFrame(ctx, cmd.Index, cmd.Frame, cmd.FrameOptions)
+	err = cmd.client.EnsureField(ctx, cmd.Index, cmd.Field)
 	if err != nil {
-		return fmt.Errorf("Error Creating Frame: %s", err)
+		return fmt.Errorf("Error Creating Field: %s", err)
 	}
 	return nil
 }
 
 // importPath parses a path into bits and imports it to the server.
-func (cmd *ImportCommand) importPath(ctx context.Context, path string) error {
-	// If a field is provided, treat the import data as values to be range-encoded.
-	if cmd.Field != "" {
-		return cmd.bufferFieldValues(ctx, path)
+func (cmd *ImportCommand) importPath(ctx context.Context, fieldType, path string) error {
+	// If fieldType is `int`, treat the import data as values to be range-encoded.
+	if fieldType == pilosa.FieldTypeInt {
+		return cmd.bufferValues(ctx, path)
 	} else {
 		if cmd.StringKeys {
 			return cmd.bufferBitsK(ctx, path)
@@ -230,18 +243,18 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error {
 func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) error {
 	logger := log.New(cmd.Stderr, "", log.LstdFlags)
 
-	// Group bits by slice.
+	// Group bits by shard.
 	logger.Printf("grouping %d bits", len(bits))
-	bitsBySlice := pilosa.Bits(bits).GroupBySlice()
+	bitsByShard := http.Bits(bits).GroupByShard()
 
 	// Parse path into bits.
-	for slice, chunk := range bitsBySlice {
+	for shard, chunk := range bitsByShard {
 		if cmd.Sort {
-			sort.Sort(pilosa.BitsByPos(chunk))
+			sort.Sort(http.BitsByPos(chunk))
 		}
 
-		logger.Printf("importing slice: %d, n=%d", slice, len(chunk))
-		if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, chunk); err != nil {
+		logger.Printf("importing shard: %d, n=%d", shard, len(chunk))
+		if err := cmd.client.Import(ctx, cmd.Index, cmd.Field, shard, chunk); err != nil {
 			return errors.Wrap(err, "importing")
 		}
 	}
@@ -338,15 +351,15 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er
 	// TODO: does it help to sort the rowKeys?
 
 	logger.Printf("importing keys: n=%d", len(bits))
-	if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, bits); err != nil {
+	if err := cmd.client.ImportK(ctx, cmd.Index, cmd.Field, bits); err != nil {
 		return errors.Wrap(err, "importing keys")
 	}
 
 	return nil
 }
 
-// bufferFieldValues buffers slices of fieldValues to be imported as a batch.
-func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) error {
+// bufferValues buffers slices of FieldValues to be imported as a batch.
+func (cmd *ImportCommand) bufferValues(ctx context.Context, path string) error {
 	a := make([]pilosa.FieldValue, 0, cmd.BufferSize)
 
 	var r *csv.Reader
@@ -394,7 +407,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
 		}
 		val.ColumnID = columnID
 
-		// Parse field value.
+		// Parse FieldValue.
 		value, err := strconv.ParseInt(record[1], 10, 64)
 		if err != nil {
 			return fmt.Errorf("invalid value on row %d: %q", rnum, record[1])
@@ -403,9 +416,9 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
 
 		a = append(a, val)
 
-		// If we've reached the buffer size then import field values.
+		// If we've reached the buffer size then import FieldValues.
 		if len(a) == cmd.BufferSize {
-			if err := cmd.importFieldValues(ctx, a); err != nil {
+			if err := cmd.importValues(ctx, a); err != nil {
 				return err
 			}
 			a = a[:0]
@@ -413,29 +426,29 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
 	}
 
 	// If there are still values in the buffer then flush them.
-	if err := cmd.importFieldValues(ctx, a); err != nil {
+	if err := cmd.importValues(ctx, a); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-// importFieldValues sends batches of fieldValues to the server.
-func (cmd *ImportCommand) importFieldValues(ctx context.Context, vals []pilosa.FieldValue) error {
+// importValues sends batches of FieldValues to the server.
+func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldValue) error {
 	logger := log.New(cmd.Stderr, "", log.LstdFlags)
 
-	// Group vals by slice.
+	// Group vals by shard.
 	logger.Printf("grouping %d vals", len(vals))
-	valsBySlice := pilosa.FieldValues(vals).GroupBySlice()
+	valsByShard := http.FieldValues(vals).GroupByShard()
 
-	// Parse path into field values.
-	for slice, vals := range valsBySlice {
+	// Parse path into FieldValues.
+	for shard, vals := range valsByShard {
 		if cmd.Sort {
-			sort.Sort(pilosa.FieldValues(vals))
+			sort.Sort(http.FieldValues(vals))
 		}
 
-		logger.Printf("importing slice: %d, n=%d", slice, len(vals))
-		if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Frame, cmd.Field, slice, vals); err != nil {
+		logger.Printf("importing shard: %d, n=%d", shard, len(vals))
+		if err := cmd.client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals); err != nil {
 			return errors.Wrap(err, "importing values")
 		}
 	}

ctl/import_test.go

@@ -15,6 +15,7 @@
 package ctl
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"io"
@@ -38,11 +39,11 @@ func TestImportCommand_Validation(t *testing.T) {
 
 	cm.Index = "i"
 	err = cm.Run(context.Background())
-	if err != pilosa.ErrFrameRequired {
-		t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err)
+	if err != pilosa.ErrFieldRequired {
+		t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFieldRequired, err)
 	}
 
-	cm.Frame = "f"
+	cm.Field = "f"
 	err = cm.Run(context.Background())
 	if err.Error() != "path required" {
 		t.Fatalf("Command not working, expect: %s, actual: '%s'", "path required", err)
@@ -50,7 +51,6 @@ func TestImportCommand_Validation(t *testing.T) {
 }
 
 func TestImportCommand_Run(t *testing.T) {
-
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewImportCommand(stdin, stdout, stderr)
@@ -61,18 +61,11 @@ func TestImportCommand_Run(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	hldr := test.MustOpenHolder()
-	defer hldr.Close()
-	s := test.NewServer()
-	defer s.Close()
-
-	s.Handler.API.Cluster = test.NewCluster(1)
-	s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
-	s.Handler.API.Holder = hldr.Holder
-	cm.Host = s.Host()
+	cmd := test.MustRunCluster(t, 1)[0]
+	cm.Host = cmd.API.Node().URI.HostPort()
 
 	cm.Index = "i"
-	cm.Frame = "f"
+	cm.Field = "f"
 	cm.CreateSchema = true
 	cm.Paths = []string{file.Name()}
 	err = cm.Run(ctx)
@@ -81,10 +74,8 @@ func TestImportCommand_Run(t *testing.T) {
 	}
 }
 
-// Ensure that the ImportValue path runs (note: we have specified a value
-// for cm.Field.)
+// Ensure that the ImportValue path runs.
 func TestImportCommand_RunValue(t *testing.T) {
-
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewImportCommand(stdin, stdout, stderr)
@@ -95,22 +86,14 @@ func TestImportCommand_RunValue(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	hldr := test.MustOpenHolder()
-	defer hldr.Close()
-	s := test.NewServer()
-	defer s.Close()
+	cmd := test.MustRunCluster(t, 1)[0]
+	cm.Host = cmd.API.Node().URI.HostPort()
 
-	s.Handler.API.Cluster = test.NewCluster(1)
-	s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
-	s.Handler.API.Holder = hldr.Holder
-	cm.Host = s.Host()
-
-	http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
-	http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`)))
+	http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
+	http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`)))
 
 	cm.Index = "i"
-	cm.Frame = "f"
-	cm.Field = "foo"
+	cm.Field = "f"
 	cm.Paths = []string{file.Name()}
 	err = cm.Run(ctx)
 	if err != nil {
@@ -119,13 +102,14 @@ func TestImportCommand_RunValue(t *testing.T) {
 }
 
 func TestImportCommand_InvalidFile(t *testing.T) {
+	cmd := test.MustRunCluster(t, 1)[0]
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewImportCommand(stdin, stdout, stderr)
-	cm.Host = "anyhost"
+	cm.Host = cmd.API.Node().URI.HostPort()
 	cm.Index = "i"
-	cm.Frame = "f"
+	cm.Field = "f"
 	file, err := ioutil.TempFile("", "import.csv")
 	file.Write([]byte("a,2\n3,5\n5,6"))
 	if err != nil {
@@ -175,8 +159,63 @@ func TestImportCommand_InvalidFile(t *testing.T) {
 // MustNewHTTPRequest creates a new HTTP request. Panic on error.
 func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request {
 	req, err := http.NewRequest(method, urlStr, body)
+	req.Header.Add("Accept", "application/json")
 	if err != nil {
 		panic(err)
 	}
 	return req
 }
+
+// declare stdin, stdout, stderr
+func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
+	rder := []byte{}
+	stdin := bytes.NewReader(rder)
+	stdout := bufio.NewWriter(&buf)
+	stderr := bufio.NewWriter(&buf)
+	return stdin, stdout, stderr
+}
+
+func TestImportCommand_BugOverwriteValue(t *testing.T) {
+	cmd := test.MustRunCluster(t, 1)[0]
+
+	buf := bytes.Buffer{}
+	stdin, stdout, stderr := GetIO(buf)
+	cm := NewImportCommand(stdin, stdout, stderr)
+	file, err := ioutil.TempFile("", "import-value.csv")
+	file.Write([]byte("0,17\n"))
+	ctx := context.Background()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cm.Host = cmd.API.Node().URI.HostPort()
+
+	http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
+	http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`)))
+
+	cm.Index = "i"
+	cm.Field = "f"
+	cm.Paths = []string{file.Name()}
+	err = cm.Run(ctx)
+	if err != nil {
+		t.Fatalf("Import Run with values doesn't work: %s", err)
+	}
+
+	file.Close()
+	file, err = ioutil.TempFile("", "import-value2.csv")
+	file.Write([]byte("0,16\n"))
+	cm.Paths = []string{file.Name()}
+	err = cm.Run(ctx)
+	if err != nil {
+		t.Fatalf("Import Run with values doesn't work: %s", err)
+	}
+
+	file.Close()
+	file, err = ioutil.TempFile("", "import-value3.csv")
+	file.Write([]byte("0,19\n"))
+	cm.Paths = []string{file.Name()}
+	err = cm.Run(ctx)
+	if err != nil {
+		t.Fatalf("Import Run with values doesn't work: %s", err)
+	}
+}

ctl/server.go

@@ -43,6 +43,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.")
 	flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.")
 
+	// Translation
+	flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "URL for primary translation node for replication.")
+
 	// Gossip
 	flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.")
 	flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.")

diagnostics.go

@@ -37,8 +37,8 @@ type versionResponse struct {
 	Message string `json:"message"`
 }
 
-// DiagnosticsCollector represents a collector/sender of diagnostics data.
-type DiagnosticsCollector struct {
+// diagnosticsCollector represents a collector/sender of diagnostics data.
+type diagnosticsCollector struct {
 	mu          sync.Mutex
 	host        string
 	VersionURL  string
@@ -56,9 +56,9 @@ type DiagnosticsCollector struct {
 	server *Server
 }
 
-// NewDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port".
-func NewDiagnosticsCollector(host string) *DiagnosticsCollector {
-	return &DiagnosticsCollector{
+// newDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port".
+func newDiagnosticsCollector(host string) *diagnosticsCollector {
+	return &diagnosticsCollector{
 		host:       host,
 		VersionURL: defaultVersionCheckURL,
 		startTime:  time.Now().Unix(),
@@ -70,13 +70,13 @@ func NewDiagnosticsCollector(host string) *DiagnosticsCollector {
 }
 
 // SetVersion of locally running Pilosa Cluster to check against master.
-func (d *DiagnosticsCollector) SetVersion(v string) {
+func (d *diagnosticsCollector) SetVersion(v string) {
 	d.version = v
 	d.Set("Version", v)
 }
 
 // Flush sends the current metrics.
-func (d *DiagnosticsCollector) Flush() error {
+func (d *diagnosticsCollector) Flush() error {
 	d.mu.Lock()
 	defer d.mu.Unlock()
 	d.metrics["Uptime"] = (time.Now().Unix() - d.startTime)
@@ -99,7 +99,7 @@ func (d *DiagnosticsCollector) Flush() error {
 }
 
 // CheckVersion of the local build against Pilosa master.
-func (d *DiagnosticsCollector) CheckVersion() error {
+func (d *diagnosticsCollector) CheckVersion() error {
 	var rsp versionResponse
 	req, err := http.NewRequest("GET", d.VersionURL, nil)
 	if err != nil {
@@ -131,7 +131,7 @@ func (d *DiagnosticsCollector) CheckVersion() error {
 }
 
 // compareVersion check version strings.
-func (d *DiagnosticsCollector) compareVersion(value string) error {
+func (d *diagnosticsCollector) compareVersion(value string) error {
 	currentVersion := versionSegments(value)
 	localVersion := versionSegments(d.version)
 
@@ -147,12 +147,12 @@ func (d *DiagnosticsCollector) compareVersion(value string) error {
 }
 
 // Encode metrics maps into the json message format.
-func (d *DiagnosticsCollector) encode() ([]byte, error) {
+func (d *diagnosticsCollector) encode() ([]byte, error) {
 	return json.Marshal(d.metrics)
 }
 
 // Set adds a key value metric.
-func (d *DiagnosticsCollector) Set(name string, value interface{}) {
+func (d *diagnosticsCollector) Set(name string, value interface{}) {
 	switch v := value.(type) {
 	case string:
 		if v == "" {
@@ -166,7 +166,7 @@ func (d *DiagnosticsCollector) Set(name string, value interface{}) {
 }
 
 // logErr logs the error and returns true if an error exists
-func (d *DiagnosticsCollector) logErr(err error) bool {
+func (d *diagnosticsCollector) logErr(err error) bool {
 	if err != nil {
 		d.Logger.Printf("%v", err)
 		return true
@@ -175,7 +175,7 @@ func (d *DiagnosticsCollector) logErr(err error) bool {
 }
 
 // EnrichWithOSInfo adds OS information to the diagnostics payload.
-func (d *DiagnosticsCollector) EnrichWithOSInfo() {
+func (d *diagnosticsCollector) EnrichWithOSInfo() {
 	uptime, err := d.server.systemInfo.Uptime()
 	if !d.logErr(err) {
 		d.Set("HostUptime", uptime)
@@ -199,7 +199,7 @@ func (d *DiagnosticsCollector) EnrichWithOSInfo() {
 }
 
 // EnrichWithMemoryInfo adds memory information to the diagnostics payload.
-func (d *DiagnosticsCollector) EnrichWithMemoryInfo() {
+func (d *diagnosticsCollector) EnrichWithMemoryInfo() {
 	memFree, err := d.server.systemInfo.MemFree()
 	if !d.logErr(err) {
 		d.Set("MemFree", memFree)
@@ -215,30 +215,30 @@ func (d *DiagnosticsCollector) EnrichWithMemoryInfo() {
 }
 
 // EnrichWithSchemaProperties adds schema info to the diagnostics payload.
-func (d *DiagnosticsCollector) EnrichWithSchemaProperties() {
-	var numSlices uint64
-	numFrames := 0
+func (d *diagnosticsCollector) EnrichWithSchemaProperties() {
+	var numShards uint64
+	numFields := 0
 	numIndexes := 0
 	bsiFieldCount := 0
 	timeQuantumEnabled := false
 
-	for _, index := range d.server.Holder.Indexes() {
-		numSlices += index.MaxSlice() + 1
+	for _, index := range d.server.holder.Indexes() {
+		numShards += index.maxShard() + 1
 		numIndexes += 1
-		for _, frame := range index.Frames() {
-			numFrames += 1
-			if fields, err := frame.GetFields(); err == nil {
-				bsiFieldCount += len(fields)
+		for _, field := range index.Fields() {
+			numFields += 1
+			if field.Type() == FieldTypeInt {
+				bsiFieldCount += 1
 			}
-			if frame.TimeQuantum() != "" {
+			if field.TimeQuantum() != "" {
 				timeQuantumEnabled = true
 			}
 		}
 	}
 
 	d.Set("NumIndexes", numIndexes)
-	d.Set("NumFrames", numFrames)
-	d.Set("NumSlices", numSlices)
+	d.Set("NumFields", numFields)
+	d.Set("NumShards", numShards)
 	d.Set("BSIFieldCount", bsiFieldCount)
 	d.Set("TimeQuantumEnabled", timeQuantumEnabled)
 }
@@ -267,51 +267,51 @@ type SystemInfo interface {
 	MemUsed() (uint64, error)
 }
 
-// NewNopSystemInfo creates a no-op implementation of SystemInfo.
-func NewNopSystemInfo() *NopSystemInfo {
-	return &NopSystemInfo{}
+// newNopSystemInfo creates a no-op implementation of SystemInfo.
+func newNopSystemInfo() *nopSystemInfo {
+	return &nopSystemInfo{}
 }
 
-// NopSystemInfo is a no-op implementation of SystemInfo.
-type NopSystemInfo struct {
+// nopSystemInfo is a no-op implementation of SystemInfo.
+type nopSystemInfo struct {
 }
 
 // Uptime is a no-op implementation of SystemInfo.Uptime.
-func (n *NopSystemInfo) Uptime() (uint64, error) {
+func (n *nopSystemInfo) Uptime() (uint64, error) {
 	return 0, nil
 }
 
 // Platform is a no-op implementation of SystemInfo.Platform.
-func (n *NopSystemInfo) Platform() (string, error) {
+func (n *nopSystemInfo) Platform() (string, error) {
 	return "", nil
 }
 
 // Family is a no-op implementation of SystemInfo.Family.
-func (n *NopSystemInfo) Family() (string, error) {
+func (n *nopSystemInfo) Family() (string, error) {
 	return "", nil
 }
 
 // OSVersion is a no-op implementation of SystemInfo.OSVersion.
-func (n *NopSystemInfo) OSVersion() (string, error) {
+func (n *nopSystemInfo) OSVersion() (string, error) {
 	return "", nil
 }
 
 // KernelVersion is a no-op implementation of SystemInfo.KernelVersion.
-func (n *NopSystemInfo) KernelVersion() (string, error) {
+func (n *nopSystemInfo) KernelVersion() (string, error) {
 	return "", nil
 }
 
 // MemFree is a no-op implementation of SystemInfo.MemFree.
-func (n *NopSystemInfo) MemFree() (uint64, error) {
+func (n *nopSystemInfo) MemFree() (uint64, error) {
 	return 0, nil
 }
 
 // MemTotal is a no-op implementation of SystemInfo.MemTotal.
-func (n *NopSystemInfo) MemTotal() (uint64, error) {
+func (n *nopSystemInfo) MemTotal() (uint64, error) {
 	return 0, nil
 }
 
 // MemUsed is a no-op implementation of SystemInfo.MemUsed.
-func (n *NopSystemInfo) MemUsed() (uint64, error) {
+func (n *nopSystemInfo) MemUsed() (uint64, error) {
 	return 0, nil
 }

diagnostics_internal_test.go

@@ -29,7 +29,7 @@ func TestDiagnosticsClient(t *testing.T) {
 	server := httptest.NewServer(nil)
 
 	// Create a new client.
-	d := NewDiagnosticsCollector(server.URL)
+	d := newDiagnosticsCollector(server.URL)
 
 	d.Set("gg", 10)
 	d.Set("ss", "ss")
@@ -76,7 +76,7 @@ func TestDiagnosticsVersion_Parse(t *testing.T) {
 }
 
 func TestDiagnosticsVersion_Compare(t *testing.T) {
-	d := NewDiagnosticsCollector("localhost:10101")
+	d := newDiagnosticsCollector("localhost:10101")
 
 	version := "v0.1.1"
 	d.SetVersion(version)
@@ -118,7 +118,7 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
 	}))
 
 	// Create a new client.
-	d := NewDiagnosticsCollector("localhost:10101")
+	d := newDiagnosticsCollector("localhost:10101")
 
 	version := "0.1.1"
 	d.SetVersion(version)
@@ -143,7 +143,7 @@ func BenchmarkDiagnostics(b *testing.B) {
 	server := httptest.NewServer(nil)
 
 	// Create a new client.
-	d := NewDiagnosticsCollector(server.URL)
+	d := newDiagnosticsCollector(server.URL)
 
 	prev := runtime.GOMAXPROCS(4)
 	defer runtime.GOMAXPROCS(prev)

docs/administration.md

@@ -24,19 +24,19 @@ Pilosa holds all row/column bitmap data in main memory. While this data is compr
 
 #### CPUs
 
-Pilosa is a concurrent application written in Go and can take full advantage of multicore machines. The main unit of parallelism is the [slice](../data-model/#slice), so a single query will only use a number of cores up to the number of slices stored on that host. Multiple queries can still take advantage of multiple cores as well though, so tuning in this area is dependent on the expected workload.
+Pilosa is a concurrent application written in Go and can take full advantage of multicore machines. The main unit of parallelism is the [shard](../data-model/#shard), so a single query will only use a number of cores up to the number of shards stored on that host. Multiple queries can still take advantage of multiple cores as well, so tuning in this area is dependent upon the expected workload.
 
 #### Disk
 
-Even though the main dataset is in memory Pilosa does back up to disk frequently. We recommend SSDs—especially if you have a write heavy application.
+Even though the main dataset is in memory Pilosa backs up to disk frequently. We recommend SSDs—especially if you have a write-heavy application.
 
 #### Network
 
-Pilosa is designed to be a distributed application, with data replication shared across the cluster. As such every write and read needs to communicate with several nodes. Therefore fast internode communication is essential. If using a service like AWS we recommend that all node exist in the same region and availability zone. The inherent latency of spreading a Pilosa cluster across physical regions it not usually worth the redundancy protection. Since Pilosa is designed to be an indexing service there already should be a system of record, or ability to rebuild a cluster quickly from backups.
+Pilosa is designed to be a distributed application, with data replication replicated across the cluster. As such, every write and read needs to communicate with several nodes. Therefore fast internode communication is essential. If using a service like AWS we recommend that all nodes exist in the same region and availability zone. The inherent latency of spreading a Pilosa cluster across physical regions is not usually worth the redundancy protection. Since Pilosa is designed to be an indexing service there should already be a system of record, or ability to rebuild a cluster quickly from backups.
 
 #### Overview
 
-While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines. The internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time.
+While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines, as the internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time.
 
 ### Open File Limits
 
@@ -56,23 +56,23 @@ When importing large datasets remember it is much faster to pre sort the data by
 pilosa import --sort -i project -f stargazer project-stargazer.csv
 ```
 
-##### Importing Field Values
+##### Importing Integer Values
 
-If you are using [BSI Range-Encoding](../data-model/#bsi-range-encoding) field values, you can import field values for a single frame and single field using `--field`. The CSV file should be in the format `Column,Value`.
+If you are using [integer](../data-model/#bsi-range-encoding) field values, the CSV file should be in the format `Column,Value`.
 
 ```
-pilosa import -i project -f stargazer --field star_count project-stargazer-counts.csv
+pilosa import -i project -f stargazer-counts project-stargazer-counts.csv
 ```
 
 <div class="note">
-    <p>Note that you must first create a frame and a field. View <a href="../api-reference/#create-frame">Create Frame</a> for more details.</p>
+    <p>Note that you must first create a field. View <a href="../api-reference/#create-field">Create Field</a> for more details. The `-e` flag can create the necessary schema when using a field of type "set".</p>
 </div>
 
 #### Exporting
 
-Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the frame. The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format `Row,Column` and sorted by column.
+Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the field. The API also expects the shard number, but the `pilosa export` sub command will export all shards within a field. The data will be in csv format `Row,Column` and sorted by column.
 ```request
-curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0" \
+curl "http://localhost:10101/export?index=repository&field=stargazer&shard=0" \
      --header "Accept: text/csv"
 ```
 ```response
@@ -122,7 +122,7 @@ Pilosa v0.9 introduces a few compatibility changes that need to be addressed.
 
 Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topology` file. Due to the way Pilosa internally shards indices, upgrading a Pilosa cluster will result in data loss if an existing cluster is brought up without these files. New clusters will generate them automatically, but you may migrate an existing cluster by using a tool we called [`topology-generator`](https://github.com/pilosa/upgrade-utils/tree/master/v0.9/topology-generator):
 
-1. Observe the `cluster.hosts` configuration value in Pilosa v0.8. The ordering of the nodes in the config file is significant, as it determines shard (AKA slice) ownership. Pilosa v0.9 uses UUIDs for each node, and the ordering is alphabetical.
+1. Observe the `cluster.hosts` configuration value in Pilosa v0.8. The ordering of the nodes in the config file is significant, as it determines shard ownership. Pilosa v0.9 uses UUIDs for each node, and the ordering is alphabetical.
 2. Install the `topology-generator`: `go get github.com/pilosa/upgrade-utils/v0.9/topology-generator`.
 3. Run the `topology-generator`. There are two arguments: the number of nodes and the output directory. For this example, we'll assume a 3-node cluster and place the files in the current working directory: `topology-generator 3 .`.
 4. This tool will generate a file, `topology`, and multiple id files, called `nodeX.id`, X being the node index position.
@@ -132,8 +132,8 @@ Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topo
 **Application changes**:
 
 1. Row and column labels were deprecated in Pilosa v0.8, and removed in Pilosa v0.9. Make sure that your application does not attempt to use a custom row or column label, as they are no longer supported.
-2. If your application relies on the implicit creation of [time quantums](../glossary/#time-quantum) by inheriting the time-quantum setting of the index, you must begin explicitly enabling the time quantum per-frame, as index-level time-quantums have been removed.
-3. Inverse frames have been deprecated, removed from docs, and will be unsupported in the next release.
+2. If your application relies on the implicit creation of [time quantums](../glossary/#time-quantum) by inheriting the time-quantum setting of the index, you must begin explicitly enabling the time quantum per-field, as index-level time-quantums have been removed.
+3. Inverse fields have been deprecated, removed from docs, and will be unsupported in the next release.
 
 ### Resizing the Cluster
 
@@ -211,7 +211,7 @@ curl localhost:10101/cluster/resize/set-coordinator \
 
 ### Backup/restore
 
-Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Frame->Views->Fragment->numbered slice files. These data files can be routinely backed up to restore nodes in a cluster.
+Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Field->Views->Fragment->numbered shard files. These data files can be routinely backed up to restore nodes in a cluster.
 
 Depending on the size of your data you have two options. For a small dataset you can rely on the periodic anti-entropy sync process to replicate existing data back to this node.
 
@@ -230,12 +230,12 @@ Note: This will only work when the replication factor is >= 2
 
 - To accomplish this you will first need:
   - List of all indexes on your cluster
-  - List of all frames in your indexes
-  - Max slice per index, listed in the `/slices/max` endpoint
-- With this information you can query the `/fragment/nodes` endpoint and iterate over each slice
-- Using the list of slices owned by this node you will then need to manually:
-  - setup a directory structure similar to the other nodes with a path for each Index/Frame
-  - copy each owned slice for an existing node to this new node
+  - List of all fields in your indexes
+  - Max shard per index, listed in the `/internal/shards/max` endpoint
+- With this information you can query the `/internal/fragment/nodes` endpoint and iterate over each shard
+- Using the list of shards owned by this node you will then need to manually:
+  - setup a directory structure similar to the other nodes with a path for each Index/Field
+  - copy each owned shard for an existing node to this new node
 - Modify the cluster config file to replace the previous node address with the new node address.
 - Restart the cluster
 - Wait for the first sync (10 minutes) to validate Index connections
@@ -249,11 +249,11 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi
 - **Cluster:** List of nodes in the cluster.
 - **NumNodes:** Number of nodes in the cluster.
 - **NumCPU:** Number of cores per node
-- **BSIEnabled:** Bit Slice Index Frames in use.
-- **TimeQuantumEnabled:** Time Quantum Frames in use.
+- **BSIEnabled:** Bit Sliced Index Fields in use.
+- **TimeQuantumEnabled:** Time Quantum Fields in use.
 - **NumIndexes:** Number of indexes in the Cluster.
-- **NumFrames:** Number of frames in the Cluster.
-- **NumSlices:** Number of slices in the Cluster.
+- **NumFields:** Number of fields in the Cluster.
+- **NumShards:** Number of shards in the Cluster.
 - **NumViews:** Number of views in the Cluster.
 - **OpenFiles:** Open file handle count.
 - **GoRoutines:** Go routine count.
@@ -274,16 +274,16 @@ StatsD Tags adhere to the DataDog format (key:value), and we tag the following:
 
 - NodeID
 - Index
-- Frame
+- Field
 - View
-- Slice
+- Shard
 
 #### Events
 We currently track the following events
 
 - **Index:** The creation of a new index.
-- **Frame:** The creation of a new frame.
-- **MaxSlice:** The creation of a new Slice.
+- **Field:** The creation of a new field.
+- **MaxShard:** The creation of a new Shard.
 - **SetBit:** Count of set bits.
 - **ClearBit:** Count of cleared bits.
 - **ImportBit:** During a bulk data import this represents the count of bits created.

docs/api-reference.md

@@ -17,7 +17,7 @@ Returns the schema of all indexes in JSON.
 curl -XGET localhost:10101/index
 ```
 ``` response
-{"indexes":[{"name":"user","frames":[{"name":"collab"}]}]}
+{"indexes":[{"name":"user","fields":[{"name":"collab"}]}]}
 ```
 
 ### List index schema
@@ -30,7 +30,7 @@ Returns the schema of the specified index in JSON.
 curl -XGET localhost:10101/index/user
 ```
 ``` response
-{"index":{"name":"user"}, "frames":[{"name":"collab"}]}]}
+{"name":"user", "fields":[{"name":"collab"}]}
 ```
 
 ### Create index
@@ -43,7 +43,7 @@ Creates an index with the given name.
 curl -XPOST localhost:10101/index/user
 ```
 ``` response
-{}
+{"success":true}
 ```
 
 ### Remove index
@@ -56,7 +56,7 @@ Removes the given index.
 curl -XDELETE localhost:10101/index/user
 ```
 ``` response
-{}
+{"success":true}
 ```
 
 ### Query index
@@ -68,102 +68,81 @@ Sends a [query](../query-language/) to the Pilosa server with the given index. T
 ``` request
 curl localhost:10101/index/user/query \
      -X POST \
-     -d 'Bitmap(frame="language", row=5)'
+     -d 'Row(language=5)'
 ```
 ``` response
-{"results":[{"attrs":{},"bits":[100]}]}
+{"results":[{"attrs":{},"columns":[100]}]}
 ```
 
 In order to send protobuf binaries in the request and response, set `Content-Type` and `Accept` headers to: `application/x-protobuf`.
 
 The response doesn't include column attributes by default. To return them, set the `columnAttrs` query argument to `true`.
 
-The query is executed for all [slices](../data-model/#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices.
+The query is executed for all [shards](../data-model/#shard) by default. To use specified shards only, set the `shards` query argument to a comma-separated list of slice indices.
 
 ``` request
-curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \
+curl "localhost:10101/index/user/query?columnAttrs=true&shards=0,1" \
      -X POST \
-     -d 'Bitmap(frame="language", row=5)'
+     -d 'Row(language=5)'
 ```
 ``` response
 {
-  "results":[{"attrs":{},"bits":[100]}],
+  "results":[{"attrs":{},"columns":[100]}],
   "columnAttrs":[{"id":100,"attrs":{"name":"Klingon"}}]
 }
 ```
 
-By default, all bits and attributes (*for `Bitmap` queries only*) are returned. In order to suppress returning bits, set `excludeBits` query argument to `true`; to suppress returning attributes, set `excludeAttrs` query argument to `true`.
+By default, all bits and attributes (*for `Row` queries only*) are returned. In order to suppress returning bits, set `excludeBits` query argument to `true`; to suppress returning attributes, set `excludeAttrs` query argument to `true`.
 
-### Create frame
+### Create field
 
-`POST /index/<index-name>/frame/<frame-name>`
+`POST /index/<index-name>/field/<field-name>`
 
-Creates a frame in the given index with the given name.
+Creates a field in the given index with the given name.
 
 The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields:
 
-* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame.
-* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
+* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field.
+* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this field. Default is `lru`.
 * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
 * `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding).
 
 Each individual `field` contains the following:
 
 * `name` (string): Field name.
-* `type` (string): Field type, currently only "int" is supported.
+* `type` (string): Field type, "set", "int" or "time".
 * `min` (int): Minimum value allowed for this field.
 * `max` (int): Maximum value allowed for this field.
 
 Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
 
 ``` request
-curl localhost:10101/index/user/frame/language -X POST
+curl localhost:10101/index/user/field/language -X POST
 ```
 ``` response
-{}
+{"success":true}
 ```
 
 ``` request
-curl localhost:10101/index/repository/frame/stats \
+curl localhost:10101/index/repository/field/stats \
     -X POST \
     -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
 ```
 ``` response
-{}
+{"success":true}
 ```
 
-### Remove frame
+### Remove field
 
-`DELETE /index/<index-name>/frame/<frame-name>`
+`DELETE /index/<index-name>/field/<field-name>`
 
-Removes the given frame.
+Removes the given field.
 
 ``` request
-curl -XDELETE localhost:10101/index/user/frame/language
+curl -XDELETE localhost:10101/index/user/field/language
 ```
 ``` response
-{}
-```
-
-### Create Field
-
-`POST /index/<index-name>/frame/<frame-name>/field/<field-name>`
-
-Creates a new field to store integer values in the given frame.
-
-The request payload is JSON, and it must contain the fields `type`, `min`, `max`.
-
-* `type` (string): Field type, currently only "int" is supported.
-* `min` (int): Minimum value allowed for this field.
-* `max` (int): Maximum value allowed for this field.
-
-``` request
-curl localhost:10101/index/repository/frame/stats/field/pullrequests \
-    -X POST \
-    -d '{"type": "int", "min": 0, "max": 1000000}'
-```
-``` response
-{}
+{"success":true}
 ```
 
 ### Get version
@@ -191,7 +170,7 @@ in a multi-node cluster, the cache is only recalculated on the node
 that receives the request.
 
 ``` request
-curl -XGET localhost:10101/recalculate-caches
+curl -XPOST localhost:10101/recalculate-caches
 ```
 
 Response: `204 No Content`

docs/client-libraries.md

@@ -46,16 +46,16 @@ func main() {
 		panic(err)
 	}
 
-	// We need to refer to indexes and frames before we can use them in a query.
+	// We need to refer to indexes and fields before we can use them in a query.
 	repository, _ := schema.Index("repository")
-	stargazer, _ := repository.Frame("stargazer")
-	language, _ := repository.Frame("language")
+	stargazer, _ := repository.Field("stargazer")
+	language, _ := repository.Field("language")
 
 	var response *pilosa.QueryResponse
 
 	// Which repositories did user 14 star:
-	response, _ = client.Query(stargazer.Bitmap(14))
-	fmt.Println("User 14 starred: ", response.Result().Bitmap().Bits)
+	response, _ = client.Query(stargazer.Row(14))
+	fmt.Println("User 14 starred: ", response.Result().Row().Columns)
 
 	// What are the top 5 languages in the sample data?
 	response, err = client.Query(language.TopN(5))
@@ -68,29 +68,29 @@ func main() {
 	// Which repositories were starred by both user 14 and 19:
 	response, _ = client.Query(
 		repository.Intersect(
-			stargazer.Bitmap(14),
-			stargazer.Bitmap(19)))
-	fmt.Println("Both user 14 and 19 starred:", response.Result().Bitmap().Bits)
+			stargazer.Row(14),
+			stargazer.Row(19)))
+	fmt.Println("Both user 14 and 19 starred:", response.Result().Row().Columns)
 
 	// Which repositories were starred by user 14 or 19:
 	response, _ = client.Query(
 		repository.Union(
-			stargazer.Bitmap(14),
-			stargazer.Bitmap(19)))
-	fmt.Println("User 14 or 19 starred:", response.Result().Bitmap().Bits)
+			stargazer.Row(14),
+			stargazer.Row(19)))
+	fmt.Println("User 14 or 19 starred:", response.Result().Row().Columns)
 
 	// Which repositories were starred by user 14 or 19 and were written in language 1:
 	response, _ = client.Query(
 		repository.Intersect(
 			repository.Union(
-				stargazer.Bitmap(14),
-				stargazer.Bitmap(19),
+				stargazer.Row(14),
+				stargazer.Row(19),
 			),
-			language.Bitmap(1)))
-	fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Bitmap().Bits)
+			language.Row(1)))
+	fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Row().Columns)
 
 	// Set user 99999 as a stargazer for repository 77777?
-	client.Query(stargazer.SetBit(99999, 77777))
+	client.Query(stargazer.Set(99999, 77777))
 }
 ```
 
@@ -112,6 +112,7 @@ We are going to use the index you have created in the [Getting Started](../getti
 Error handling has been omitted in the example below for brevity.
 
 ```python
+from __future__ import print_function
 from pilosa import Index, Client, PilosaError, TimeQuantum
 
 # We will just use the default client which assumes the server is at http://localhost:10101
@@ -122,8 +123,8 @@ client = Client()
 # and the stargazer data should be imported.
 # See the Getting Started repository: https://github.com/pilosa/getting-started/
 
-# Let's create Index and Frame objects, which will contain the settings
-# for the corresponding indexes and frames.
+# Let's create Index and Field objects, which will contain the settings
+# for the corresponding indexes and fields.
 try:
     schema = client.schema()
 except PilosaError as e:
@@ -132,13 +133,13 @@ except PilosaError as e:
     # We will just terminate the program in this case.
     raise SystemExit(e)
 
-# We need to refer to indexes and frames before we can use them in a query.
+# We need to refer to indexes and fields before we can use them in a query.
 repository = schema.index("repository")
-stargazer = repository.frame("stargazer")
-language = repository.frame("language")
+stargazer = repository.field("stargazer")
+language = repository.field("language")
 
 # Which repositories did user 8 star:
-repository_ids = client.query(stargazer.bitmap(14)).result.bitmap.bits
+repository_ids = client.query(stargazer.row(14)).result.row.columns
 print("User 8 starred: ", repository_ids)
 
 # What are the top 5 languages in the sample data:
@@ -147,33 +148,33 @@ print("Top 5 languages: ", [item.id for item in top_languages])
 
 # Which repositories were starred by both user 14 and 19:
 query = repository.intersect(
-    stargazer.bitmap(14),
-    stargazer.bitmap(19)
+    stargazer.row(14),
+    stargazer.row(19)
 )
-mutually_starred = client.query(query).result.bitmap.bits
+mutually_starred = client.query(query).result.row.columns
 print("Both user 14 and 19 starred:", mutually_starred)
 
 # Which repositories were starred by user 14 or 19:
 query = repository.union(
-    stargazer.bitmap(14),
-    stargazer.bitmap(19)
+    stargazer.row(14),
+    stargazer.row(19)
 )
-either_starred = client.query(query).result.bitmap.bits
+either_starred = client.query(query).result.row.columns
 print("User 14 or 19 starred:", either_starred)
 
 # Which repositories were starred by user 14 or 19 and were written in language 1:
 query = repository.intersect(
     repository.union(
-        stargazer.bitmap(14),
-        stargazer.bitmap(19)
+        stargazer.row(14),
+        stargazer.row(19)
     ),
-    language.bitmap(1)
+    language.row(1)
 )
-mutually_starred = client.query(query).result.bitmap.bits
+mutually_starred = client.query(query).result.row.columns
 print("User 14 or 19 starred, written in language 1:", mutually_starred)
 
 # Set user 99999 as a stargazer for repository 77777
-client.query(stargazer.setbit(99999, 77777))
+client.query(stargazer.set(99999, 77777))
 ```
 
 Running the above program should produce output like this:
@@ -218,10 +219,10 @@ public class StarTrace {
             throw new RuntimeException(ex);
         }
 
-        // We need to refer to indexes and frames before we can use them in a query.
+        // We need to refer to indexes and fields before we can use them in a query.
         Index repository = schema.index("repository");
-        Frame stargazer = repository.frame("stargazer");
-        Frame language = repository.frame("language");
+        Field stargazer = repository.field("stargazer");
+        Field language = repository.field("language");
 
         QueryResponse response;
         QueryResult result;
@@ -229,8 +230,8 @@ public class StarTrace {
         List<Long> repositoryIDs;
 
         // Which repositories did user 14 star:
-        response = client.query(stargazer.bitmap(14));
-        repositoryIDs = response.getResult().getBitmap().getBits();
+        response = client.query(stargazer.row(14));
+        repositoryIDs = response.getResult().getRow().getColumns();
         System.out.println("User 14 starred: " + repositoryIDs);
 
         // What are the top 5 languages in the sample data:
@@ -245,36 +246,36 @@ public class StarTrace {
 
         // Which repositories were starred by both user 14 and 19:
         query = repository.intersect(
-                stargazer.bitmap(14),
-                stargazer.bitmap(19)
+                stargazer.row(14),
+                stargazer.row(19)
         );
         response = client.query(query);
-        repositoryIDs = response.getResult().getBitmap().getBits();
+        repositoryIDs = response.getResult().getRow().getColumns();
         System.out.println("Both user 14 and 19 starred: " + repositoryIDs);
 
         // Which repositories were starred by user 14 or 19:
         query = repository.union(
-                stargazer.bitmap(14),
-                stargazer.bitmap(19)
+                stargazer.row(14),
+                stargazer.row(19)
         );
         response = client.query(query);
-        repositoryIDs = response.getResult().getBitmap().getBits();
+        repositoryIDs = response.getResult().getRow().getColumns();
         System.out.println("User 14 or 19 starred: " + repositoryIDs);
 
         // Which repositories were starred by user 14 or 19 and were written in language 1:
         query = repository.intersect(
                 repository.union(
-                        stargazer.bitmap(14),
-                        stargazer.bitmap(19)
+                        stargazer.row(14),
+                        stargazer.row(19)
                 ),
-                language.bitmap(1)
+                language.row(1)
         );
         response = client.query(query);
-        repositoryIDs = response.getResult().getBitmap().getBits();
+        repositoryIDs = response.getResult().getRow().getColumns();
         System.out.println("User 14 or 19 starred, written in language 1: " + repositoryIDs);
 
         // Set user 99999 as a stargazer for repository 77777:
-        client.query(stargazer.setBit(99999, 77777));
+        client.query(stargazer.set(99999, 77777));
     }
 }
 ```

docs/configuration.md

@@ -106,7 +106,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
 
 #### Max Writes Per Request
 
-* Description: Maximum number of mutating commands allowed per request. This includes SetBit, ClearBit, SetRowAttrs, SetColumnAttrs, and SetFieldValue.
+* Description: Maximum number of mutating commands allowed per request. This includes Set, Clear, SetRowAttrs, and SetColumnAttrs.
 * Flag: `--max-writes-per-request=5000`
 * Env: `PILOSA_MAX_WRITES_PER_REQUEST=5000`
 * Config:

docs/data-model.md

@@ -6,10 +6,10 @@ nav = [
     "Index",
     "Column",
     "Row",
-    "Frame",
+    "Field",
     "Time Quantum",
     "Attribute",
-    "Slice",
+    "Shard",
     "View",
 ]
 +++
@@ -22,7 +22,7 @@ The central component of Pilosa's data model is a boolean matrix. Each cell in t
 
 Rows and columns can represent anything (they could even represent the same set of things - a [bigraph](https://en.wikipedia.org/wiki/Bigraph)). Pilosa can associate arbitrary key/value pairs (referred to as attributes) to rows and columns, but queries and storage are optimized around the core matrix.
 
-Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa categorizes rows into different *frames* and quickly retrieves the top rows in a frame sorted by the number of bits set in each row.
+Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa categorizes rows into different *fields* and quickly retrieves the top rows in a field sorted by the number of columns set in each row.
 
 Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but setting a bit with column ID 2<sup>63</sup> on a single-node cluster, for example, will not work well due to memory limitations.
 
@@ -35,20 +35,22 @@ The purpose of the Index is to represent a data namespace. You cannot perform cr
 
 ### Column
 
-Column ids are sequential increasing integers and are common to all Frames within an Index. A single column often corresponds to a record in a relational table, although other configurations are possible, and sometimes preferable.
+Column ids are sequential increasing integers and are common to all Fields within an Index. A single column often corresponds to a record in a relational table, although other configurations are possible, and sometimes preferable.
 
 ### Row
 
-Row ids are sequential increasing integers namespaced to each Frame within an Index.
+Row ids are sequential increasing integers namespaced to each Field within an Index.
 
-### Frame
+### Field
 
-Frames are used to segment rows within an index, for example to define different functional groups. A frame might correspond to a single field in a relational table, where each row in a standard frame represents a single possible value of the field. Similarly, a frame with BSI values could represent all possible integer values of a field .
+Fields are used to segment rows within an index, for example to define different functional groups. A Pilosa field might correspond to a single field in a relational table, where each row in a standard Pilosa field represents a single possible value of the relational field. Similarly, an integer field could represent all possible integer values of a relational field.
 
 #### Relational Analogy
 
 The Pilosa index is a flexible structure; it can represent any sort of high-cardinality binary matrix. We have explored a number of modeling patterns in Pilosa use cases; one accessible example is a direct analogy to the relational model, summarized here.
 
+TODO diagram showing a few rows of a relational table and corresponding pilosa index
+
 Entities:
 
  Relational  | Pilosa
@@ -56,19 +58,19 @@ Entities:
  Database    | N/A *(internal: Holder)*
  Table       | Index
  Row         | Column
- Column      | Frame
+ Column      | Field
  Value       | Row
  Value (int) | Field.Value (see [BSI](#bsi-range-encoding))
 
 Simple queries:
 
- Relational                                  | Pilosa
----------------------------------------------|------------------------------------
- `select ID from People where Name = 'Bob'`  | `Bitmap(frame=Name, row=[Bob])`
- `select ID from People where Age > 30`      | `Range(frame=Default, Age > 30)`
- `select ID from People where Member = true` | `Bitmap(frame=Member, row=[true])`
+ Relational                                    | Pilosa
+-----------------------------------------------|------------------------------------
+ `select ID from People where Name = 'Bob'`    | `Row(Name="Bob")`
+ `select ID from People where Age > 30`        | `Range(Age > 30)`
+ `select ID from People where Member = true`   | `Row(Member=0)`
 
-In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple frames. For example, this SQL join:
+Note that `Row(Member=0)` selects all entities with a bit set in row 0 of the Member field. We could just as well use row 1 to store this, in which case we would use `Row(Member=1)`, which looks a bit more intuitive. In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple fields. For example, this SQL join:
 
 ```sql
 select AVG(p.Age) from People p
@@ -80,44 +82,44 @@ where c.Make = 'Ford'
 can be accomplished with a Pilosa query like this (note that [Sum](../query-language/#sum) returns a json object containing both the sum and count, from which the average is easily computed):
 
 ```pql
-Sum(Bitmap(frame="Car-Make", row=[Ford]), frame=Default, field=Age)
+Sum(Row(Car-Make="Ford"), field=Age)
 ```
 
 This is one major component of Pilosa's ability to combine relationships from multiple data stores.
 
 #### Ranked
 
-Ranked Frames maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Frame creation.
+Ranked Fields maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Field creation.
 
-![ranked frame diagram](/img/docs/frame-ranked.svg)
-*Ranked frame diagram*
+![ranked field diagram](/img/docs/field-ranked.svg)
+*Ranked field diagram*
 
 #### LRU
 
 The LRU cache maintains the most recently accessed Rows.
 
-![lru frame diagram](/img/docs/frame-lru.svg)
-*LRU frame diagram*
+![lru field diagram](/img/docs/field-lru.svg)
+*LRU field diagram*
 
 ### Time Quantum
 
-Setting a time quantum on a frame creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
+Setting a time quantum on a field creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
 
 ### Attribute
 
 Attributes are arbitrary key/value pairs that can be associated with either rows or columns. This metadata is stored in a separate BoltDB data structure.
 
-Column-level attributes are common across an index. That is, each column attribute applies to all bits in the corresponding column, across all frames in an index. Row attributes apply to all bits in the corresponding row.
+Column-level attributes are common across an index. That is, each column attribute applies to all bits in the corresponding column, across all fields in an index. Row attributes apply to all bits in the corresponding row.
 
-### Slice
+### Shard
 
-Indexes are sharded into groups of columns called Slices. Each Slice contains a fixed number of columns, which is the SliceWidth. SliceWidth is a constant that can only be modified at compile time, and before ingesting data. The default value is 2<sup>20</sup>.
+Indexes are segmented into groups of columns called shards (previously known as slices). Each shard contains a fixed number of columns, which is the ShardWidth. ShardWidth is a constant that can only be modified at compile time, and before ingesting data. The default value is 2<sup>20</sup>.
 
 Query operations run in parallel, and they are evenly distributed across a cluster via a consistent hash algorithm.
 
 ### View
 
-Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API.
+Views represent the various data layouts within a Field. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API.
 
 #### Standard
 
@@ -125,34 +127,34 @@ The standard View contains the same Row/Column format as the input data.
 
 #### Time Quantums
 
-If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the diagram below:
+If a Field has a time quantum, then Views are generated for each of the defined time segments. For example, for a field with a time quantum of `YMD`, the following `Set()` queries will result in the data described in the diagram below:
 
 ```
-SetBit(frame="A", row=8, col=3, timestamp="2017-05-18T00:00")
-SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00")
+Set(3, A=8, 2017-05-18T00:00)
+Set(3, A=8, 2017-05-19T00:00)
 ```
 
-![time quantum frame diagram](/img/docs/frame-time-quantum.svg)
-*Time quantum frame diagram*
+![time quantum field diagram](/img/docs/field-time-quantum.svg)
+*Time quantum fueld diagram*
 
 #### BSI Range-Encoding
 
-Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead.
+Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead.
 
-Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
+Internally Pilosa stores each BSI `field` as a `view`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
 
-For example, the following `SetFieldValue()` queries will result in the data described in the diagram below:
+For example, the following `Set()` queries executed against BSI fields will result in the data described in the diagram below:
 
 ```
-SetFieldValue(col=1, frame="A", field0=1)
-SetFieldValue(col=2, frame="A", field0=2)
-SetFieldValue(col=3, frame="A", field0=3)
-SetFieldValue(col=4, frame="A", field0=7)
-SetFieldValue(col=2, frame="A", field1=1)
-SetFieldValue(col=3, frame="A", field1=6)
+Set(1, A=1)
+Set(2, A=2)
+Set(3, A=3)
+Set(4, A=7)
+Set(2, B=1)
+Set(3, B=6)
 ```
 
-![BSI frame diagram](/img/docs/frame-bsi.svg)
-*BSI frame diagram*
+![BSI field diagram](/img/docs/field-bsi.svg)
+*BSI field diagram*
 
 Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa.

docs/examples.md

@@ -3,7 +3,6 @@ title = "Examples"
 weight = 4
 nav = [
     "Transportation",
-    "Chemical similarity search",
 ]
 +++
 
@@ -33,9 +32,9 @@ The NYC taxi data is comprised of a number of csv files listed here: http://www.
 * Dropoff time: timestamp
 * Pickup time: timestamp
 
-We import these fields, creating one or more Pilosa frames from each of them:
+We import these fields, creating one or more Pilosa fields from each of them:
 
-frame	|mapping
+field	|mapping
 ------------|---------------------
 cab_type	|direct map of enum int → row ID
 dist_miles	|round(dist) → row ID
@@ -52,24 +51,24 @@ pickup_month	|month(timestamp) → row ID
 pickup_day	|day(timestamp) → row ID
 pickup_time	|time of day mapped to one of 48 half-hour buckets → row ID
 
-We also created two extra frames that represent the duration and average speed of each ride:
+We also created two extra fields that represent the duration and average speed of each ride:
 
-frame	|mapping
+field	|mapping
 --------------------|-------------
 duration_minutes	|round(drop_timestamp - pickup_timestamp) → row ID
 speed_mph	|round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID
 
 #### Mapping
 
-Each column that we want to use must be mapped to a combination of frames and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities.
+Each column that we want to use must be mapped to a combination of fields and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities.
 
-##### 0 columns → 1 frame
+##### 0 columns → 1 field
 
-**cab_type**: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this frame. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this frame are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type frame is constant.
+**cab_type**: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this field. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this field are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type field is constant.
 
-##### 1 column → 1 frame
+##### 1 column → 1 field
 
-The following three frames are mapped in a simple direct way from single columns of the original data.
+The following three fields are mapped in a simple direct way from single columns of the original data.
 
 **dist_miles:** each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times.
 
@@ -84,7 +83,7 @@ lfm := pdk.LinearFloatMapper{
 
 `Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three.
 
-This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the frame to use (`Frame`).
+This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the field to use (`Frame`).  TODO update so this makes sense
 ```go
 pdk.BitMapper{
     Frame:   "dist_miles",
@@ -129,27 +128,27 @@ Here, we define a list of Mappers, each including a name, which we use to refer
 
 **passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID.
 
-##### 1 column → multiple frames
+##### 1 column → multiple fields
 
 When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis.
 
-We do this by storing time data in four separate frames for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of frame "year", row 6 of frame "month", and row 24 of frame "day". 
+We do this by storing time data in four separate fields for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of field "year", row 6 of field "month", and row 24 of field "day". 
 
-We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of frame "time_of_day".
+We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of field "time_of_day".
 
-We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total frames for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time.
+We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total fields for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time.
 
-##### Multiple columns → 1 frame
+##### Multiple columns → 1 field
 
 The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID.
 
-We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two frames for two locations: pickup_grid_id, drop_grid_id.
+We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two fields for two locations: pickup_grid_id, drop_grid_id.
 
 Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient.
 
 ##### Complex mappings
 
-We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the frame `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the frame `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work:
+We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the field `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the field `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work:
 ```go
 durm := pdk.CustomMapper{
     Func: func(fields ...interface{}) interface{} {
@@ -172,7 +171,7 @@ Now we can run some example queries.
 Count per cab type can be retrieved, sorted, with a single PQL call.
 
 ```request
-TopN(frame=cab_type)
+TopN(cab_type)
 ```
 ```response
 {"results":[[{"id":1,"count":1992943},{"id":0,"count":7057}]]}
@@ -181,7 +180,7 @@ TopN(frame=cab_type)
 High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs.
 
 ```request
-TopN(frame=pickup_grid_id)
+TopN(pickup_grid_id)
 ```
 ```response
 {"results":[[{"id":5060,"count":40620},{"id":4861,"count":38145},{"id":4962,"count":35268},...]]}
@@ -193,7 +192,7 @@ Average of `total_amount` per `passenger_count` can be computed with some postpr
 queries = ''
 pcounts = range(10)
 for i in pcounts:
-    queries += "TopN(Bitmap(id=%d, frame='passenger_count'), frame=total_amount_dollars)" % i
+    queries += "TopN(Row(passenger_count=%d), total_amount_dollars)" % i
 resp = requests.post(qurl, data=queries)
 
 average_amounts = []
@@ -209,6 +208,8 @@ Note that the <a href="../data-model/#bsi-range-encoding">BSI</a>-powered <a hre
 
 For more examples and details, see this [ipython notebook](https://github.com/pilosa/notebooks/blob/master/taxi-use-case.ipynb).
 
+<!--
+
 ### Chemical similarity search
 
 <div class="warning">
@@ -326,3 +327,6 @@ python benchmarks.py -id 6223
 As Matt Swain’s blog post also did a great job using mongoDB for chemical similarity search, we compared benchmark on 500000 molecules between mongoDB aggregation framework with Pilosa.
 
 Both using the same molecule, Morgan fingerprint folded to fixed lengths of 4096 bits and were run on a MacBook Pro with a 2.8 GHz 2-core Intel Core i7 processor, memory of 16 GB 1600 MHz DDR3, single host cluster
+
+
+-->

docs/getting-started.md

@@ -34,19 +34,20 @@ Let's make sure Pilosa is running:
 curl localhost:10101/status
 ```
 ``` response
-{"state":"NORMAL","nodes":[{"id":"18eb5546-5a1a-4ba4-9c52-b53fbe22317e","uri":{"scheme":"http","host":"localhost","port":10101}}]}
+{"state":"NORMAL","nodes":[{"id":"91715a50-7d50-4c54-9a03-873801da1cd1","uri":{"scheme":"http","host":"localhost","port
+":10101},"isCoordinator":true}],"localID":"91715a50-7d50-4c54-9a03-873801da1cd1"}
 ```
 
 ### Sample Project
 
 In order to better understand Pilosa's capabilities, we will create a sample project called "Star Trace" containing information about 1,000 popular Github repositories which have "go" in their name. The Star Trace index will include data points such as programming language, tags, and stargazers—people who have starred a project.
 
-Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model/) section of the documentation.
+Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Fields. So the "repository" index might have a "languages" field as well as a "tags" field. You can learn more about indexes and fields in the [Data Model](../data-model/) section of the documentation.
 
 #### Create the Schema
 
 Note:
-The queries in this section which are used to set up the indexes in Pilosa just return the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows:
+If at any time you want to verify the data structure, you can request the schema as follows:
 
 ``` request
 curl localhost:10101/schema
@@ -55,35 +56,37 @@ curl localhost:10101/schema
 {"indexes":null}
 ```
 
-Before we can import data or run queries, we need to create our indexes and the frames within them. Let's create the repository index first:
+Before we can import data or run queries, we need to create our indexes and the fields within them. Let's create the repository index first:
 ``` request
 curl localhost:10101/index/repository -X POST
 ```
 ``` response
-{}
+{"success":true}
 ```
 
-Let's create the `stargazer` frame which has user IDs of stargazers as its rows:
+Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ``` request
-curl localhost:10101/index/repository/frame/stargazer \
+curl localhost:10101/index/repository/field/stargazer \
      -X POST \
-     -d '{"options": {"timeQuantum": "YMD"}}'
+     -d '{"options": {"type": "time", "timeQuantum": "YMD"}}'
 ```
 ``` response
-{}
+{"success":true}
 ```
 
-Since our data contains time stamps for the time users starred repos, we set the *time quantum* for the `stargazer` frame in the options as well. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`.
+Since our data contains time stamps whcih represent the time users starred repos, we set the field type to `time`. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`.
 
-Next up is the `language` frame, which will contain IDs for programming languages:
+Next up is the `language` field, which will contain IDs for programming languages:
 ``` request
-curl localhost:10101/index/repository/frame/language \
+curl localhost:10101/index/repository/field/language \
      -X POST
 ```
 ``` response
-{}
+{"success":true}
 ```
 
+The `language` is a `set` field, but since the default field type is `set`, we didn't specify it in field options.
+
 #### Import Data From CSV Files
 
 Download the `stargazer.csv` and `language.csv` files here:
@@ -112,22 +115,18 @@ Note that both the user IDs and the repository IDs were remapped to sequential i
 
 #### Make Some Queries
 
-<div class="note">
-    <p>Note the Pilosa server comes with a <a href="../webui/">WebUI</a> for constructing queries in a browser. In local development, it is available at <a href="http://localhost:10101">localhost:10101</a>.</p>
-</div>
-
 Which repositories did user 14 star:
 ``` request
 curl localhost:10101/index/repository/query \
      -X POST \
-     -d 'Bitmap(frame="stargazer", row=14)'
+     -d 'Row(stargazer=14)'
 ```
 ``` response
 {
     "results":[
         {
             "attrs":{},
-            "bits":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514]
+            "columns":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514]
         }
     ]
 }
@@ -137,7 +136,7 @@ What are the top 5 languages in the sample data:
 ``` request
 curl localhost:10101/index/repository/query \
      -X POST \
-     -d 'TopN(frame="language", n=5)'
+     -d 'TopN(language, n=5)'
 ```
 ``` response
 {
@@ -158,8 +157,8 @@ Which repositories were starred by user 14 and 19:
 curl localhost:10101/index/repository/query \
      -X POST \
      -d 'Intersect(
-            Bitmap(frame="stargazer", row=14), 
-            Bitmap(frame="stargazer", row=19)
+            Row(stargazer=14), 
+            Row(stargazer=19)
         )'
 ```
 ``` response
@@ -167,7 +166,7 @@ curl localhost:10101/index/repository/query \
     "results":[
         {
             "attrs":{},
-            "bits":[2,3,362,396,416,461,464,466,470,486]
+            "columns":[2,3,362,396,416,461,464,466,470,486]
         }
     ]
 }
@@ -178,8 +177,8 @@ Which repositories were starred by user 14 or 19:
 curl localhost:10101/index/repository/query \
      -X POST \
      -d 'Union(
-            Bitmap(frame="stargazer", row=14),
-            Bitmap(frame="stargazer", row=19)
+            Row(stargazer=14), 
+            Row(stargazer=19)
         )'
 ```
 ``` response
@@ -187,7 +186,7 @@ curl localhost:10101/index/repository/query \
     "results":[
         {
             "attrs":{},
-            "bits":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514]
+            "columns":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514]
         }
     ]
 }
@@ -198,9 +197,9 @@ Which repositories were starred by user 14 and 19 and also were written in langu
 curl localhost:10101/index/repository/query \
      -X POST \
      -d 'Intersect(
-            Bitmap(frame="stargazer", row=14),
-            Bitmap(frame="stargazer", row=19),
-            Bitmap(frame="language", row=1)
+            Row(stargazer=14), 
+            Row(stargazer=19),
+            Row(language=1)
         )'
 ```
 ``` response
@@ -208,7 +207,7 @@ curl localhost:10101/index/repository/query \
     "results":[
         {
             "attrs":{},
-            "bits":[2,362,416,461]
+            "columns":[2,362,416,461]
         }
     ]
 }
@@ -218,7 +217,7 @@ Set user 99999 as a stargazer for repository 77777:
 ``` request
 curl localhost:10101/index/repository/query \
      -X POST \
-     -d 'SetBit(frame="stargazer", col=77777, row=99999)'
+     -d 'Set(77777, stargazer=99999)'
 ```
 ``` response
 {"results":[true]}

docs/glossary.md

@@ -6,25 +6,25 @@ nav = []
 
 ## Glossary
 
-<strong id="anti-entropy">[Anti-entropy](../configuration/#anti-entropy-interval):</strong> A periodic process that compares each [slice](#slice) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies.
+<strong id="anti-entropy">[Anti-entropy](../configuration/#anti-entropy-interval):</strong> A periodic process that compares each [shard](#shard) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies.
 
 <strong id="attribute">[Attribute](../data-model/#attribute):</strong> Attributes can be associated to both [rows](#row) and [columns](#column). This metadata is kept separately from the core binary matrix in a [BoltDB](https://github.com/boltdb/bolt) store.
 
-<strong id="bit">[Bit](../data-model/#overview):</strong> Bits are the fundamental unit of data in Pilosa. A bit lives in a [frame](#frame), at the intersection of a [row](#row) and [column](#column).
+<strong id="bit">[Bit](../data-model/#overview):</strong> Bits are the fundamental unit of data in Pilosa. A bit lives in a [field](#field), at the intersection of a [row](#row) and [column](#column).
 
-<strong id="bitmap">[Bitmap](../data-model/#overview):</strong> The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap.
+<strong id="bitmap">[Bitmap](../data-model/#overview):</strong> The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap).
 
 <strong id="bsi">[BSI](../data-model/#bsi-range-encoding)</strong> Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries.
 
-<strong id="cluster">Cluster:</strong> A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
+<strong id="cluster">Cluster:</strong> A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
 
-<strong id="column">[Column](../data-model/#column):</strong> Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [frames](#frame) within an [index](#index).
+<strong id="column">[Column](../data-model/#column):</strong> Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [fields](#field) within an [index](#index).
 
-<strong id="field">[Field](../data-model/#bsi-range-encoding):</strong> A group of rows used to store integer values with [BSI](#bsi), for use in [Range](#range-bsi) and [Sum](#sum) queries.
+<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [field](#field) and a [shard](#shard) in an [index](#index).
 
-<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index).
+<strong id="field">[Field](../data-model/#field):</strong> Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of three types: set, [int](#bsi), and time. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field).
 
-<strong id="frame">[Frame](../data-model/#frame):</strong> Frames are used to group [rows](#row) into different categories. Row IDs are namespaced by frame such that the same row ID in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame.
+<strong id="frame">[Frame](../data-model/#field):</strong> Prior to Pilosa 1.0, fields were known as frames.
 
 <strong id="gossip">[Gossip](https://en.wikipedia.org/wiki/Gossip_protocol):</strong> A protocol used by Pilosa for internal communication.
 
@@ -32,11 +32,11 @@ nav = []
 
 <strong id="jump-consistent-hash">[Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf):</strong> A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes.
 
-<strong id="max">[Max](../query-language/#max):</strong> A [PQL](#pql) query that returns the maximum integer value stored in [BSI](#bsi) [fields](#field).
+<strong id="max">[Max](../query-language/#max):</strong> A [PQL](#pql) query that returns the maximum integer value stored in an [integer](#bsi) [field](#field).
 
-<strong id="maxslice">MaxSlice:</strong> The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries.
+<strong id="maxshard">MaxShard:</strong> The total number of [shards](#shard) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. MaxShard is zero-indexed, so if an index contains six shards, its MaxShard will be 5.
 
-<strong id="min">[Min](../query-language/#min):</strong> A [PQL](#pql) query that returns the minimum integer value stored in [BSI](#bsi) [fields](#field).
+<strong id="min">[Min](../query-language/#min):</strong> A [PQL](#pql) query that returns the minimum integer value stored in an [integer](#bsi) [field](#field).
 
 <strong id="node">Node:</strong> An individual running instance of Pilosa server which belongs to a [cluster](#cluster).
 
@@ -54,20 +54,20 @@ nav = []
 
 <strong id="roaring-bitmap">[Roaring Bitmap](http://roaringbitmap.org):</strong> the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations.
 
-<strong id="row">[Row](../data-model/#row):</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [frame](#frame) within an [index](#index). Represented as a [Bitmap](#bitmap).
+<strong id="row">[Row](../data-model/#row):</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [field](#field) within an [index](#index). Represented as a [Bitmap](#bitmap).
 
-<strong id="slice">[Slice](../data-model/#slice):</strong> [Columns](#column) are sharded on a preset [width](#slicewidth). Each shard is referred to as a slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash).
+<strong id="slice">[Slice](../data-model/#shard):</strong> Prior to Pilosa 1.0, shards were known as slices.
 
-<strong id="slicewidth">SliceWidth:</strong> This is the number of [columns](#column) in a [slice](#slice). `SliceWidth` defaults to 2<sup>20</sup> or about one million. It can be modified, but only at compile time, and before ingesting any data.
+<strong id="shard">[Shard](../data-model/#shard):</strong> [Columns](#column) are [sharded](https://en.wikipedia.org/wiki/Shard_(database_architecture)) on a preset [width](#shardwidth). Shards are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash).
 
-<strong id="sum">[Sum](../query-language/#sum):</strong> A [PQL](#pql) query that returns the sum of integers stored in [BSI](#bsi) [fields](#field).
+<strong id="shardwidth">ShardWidth:</strong> This is the number of [columns](#column) in a [shard](#shard). `ShardWidth` defaults to 2<sup>20</sup> or about one million. It can be modified, but only at compile time, and before ingesting any data.
 
-<strong id="tanimoto">[Tanimoto](../examples/#chemical-similarity-search):</strong> Used for similarity queries on Pilosa data. The [Tanimoto Coefficient](https://en.wikipedia.org/wiki/Jaccard_index#Tanimoto_similarity_and_distance) between two [Bitmaps](#bitmap) A and B is the ratio of the size of their intersection to the size of their union (|A∩B|/|A∪B|).
+<strong id="sum">[Sum](../query-language/#sum):</strong> A [PQL](#pql) query that returns the sum of integers stored in an [integer](#bsi) [field](#field).
 
-<strong id="time-quantum">[Time quantum](../data-model/#time-quantum):</strong> Defines the granularity to be used for time [Range](#range) queries.
+<strong id="time-quantum">[Time quantum](../data-model/#time-quantum):</strong> Defines the granularity to be used for [Range](#range) queries on time [fields](#field).
 
 <strong id="toml">[TOML](https://github.com/toml-lang/toml):</strong> the language used for Pilosa's [configuration file](../configuration/).
 
-<strong id="topn">[TopN](../query-language/#topn):</strong> A [PQL](#pql) query that returns a list of row IDs, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame).
+<strong id="topn">[TopN](../query-language/#topn):</strong> A [PQL](#pql) query that returns a list of rows, sorted by the count of [columns](#column) set in the [row](#row), within a specified [field](#field).
 
-<strong id="view">[View](../data-model/#view):</strong> Views separate the different data layouts within a [Frame](#frame). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.
+<strong id="view">[View](../data-model/#view):</strong> Views separate the different data layouts within a [Field](#field). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based field views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.

docs/installation.md

@@ -49,7 +49,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
       pilosa [command]
 
     Available Commands:
-      bench           Benchmark operations.
       check           Do a consistency check on a pilosa data file.
       config          Print the current configuration.
       export          Export data from pilosa.
@@ -108,7 +107,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
       pilosa [command]
 
     Available Commands:
-      bench           Benchmark operations.
       check           Do a consistency check on a pilosa data file.
       config          Print the current configuration.
       export          Export data from pilosa.
@@ -144,11 +142,10 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
     git clone https://github.com/pilosa/pilosa.git
     ```
 
-3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
+3. Build the Pilosa repo:
     ```
     cd $GOPATH/src/github.com/pilosa/pilosa
     make install-build-deps
-    make generate-statik
     make install
     ```
 
@@ -173,7 +170,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
       pilosa [command]
 
     Available Commands:
-      bench           Benchmark operations.
       check           Do a consistency check on a pilosa data file.
       config          Print the current configuration.
       export          Export data from pilosa.
@@ -262,7 +258,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
       pilosa [command]
 
     Available Commands:
-      bench           Benchmark operations.
       check           Do a consistency check on a pilosa data file.
       config          Print the current configuration.
       export          Export data from pilosa.
@@ -298,11 +293,10 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
     git clone https://github.com/pilosa/pilosa.git
     ```
 
-3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
+3. Build the Pilosa repo:
     ```
     cd $GOPATH/src/github.com/pilosa/pilosa
     make install-build-deps
-    make generate-statik
     make install
     ```
 
@@ -327,7 +321,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
       pilosa [command]
 
     Available Commands:
-      bench           Benchmark operations.
       check           Do a consistency check on a pilosa data file.
       config          Print the current configuration.
       export          Export data from pilosa.

docs/pdk.md

@@ -18,7 +18,7 @@ Running `pdk -h` will give the most up to date list of all the tools and example
 `pdk kafka` reads either JSON or Avro encoded records from Kafka (using the
 Confluent Schema Registry in the case of Avro), and indexes them in Pilosa. Each
 record from Kafka is assigned a Pilosa column, and each value in a record is
-assigned a row or field. Frame and field names are built from the "path" through
+assigned a row or field. Pilosa field names are built from the "path" through
 the record to arrive at that field. For example:
 
 ```json
@@ -38,30 +38,30 @@ the record to arrive at that field. For example:
 
 This JSON object would result in the following Pilosa schema:
 
-| Name           | Field     | Type   | Min |        Max |   Size |
-|----------------|-----------|--------|-----|------------|--------|
-| name           |           | ranked |     |            | 100000 |
-| favorite_foods |           | ranked |     |            | 100000 |
-| default        |           | ranked |     |            | 100000 |
-|                | age       | int    |   0 | 2147483647 |        |
-| location       |           | ranked |     |            |   1000 |
-|                | latitude  | int    |   0 | 2147483647 |        |
-|                | longitude | int    |   0 | 2147483647 |        |
-| location-city  |           | ranked |     |            | 100000 |
-| location-state |           | ranked |     |            | 100000 |
+| Field          | Type   | Min |        Max |   Size |
+|----------------|--------|-----|------------|--------|
+| name           | ranked |     |            | 100000 |
+| favorite_foods | ranked |     |            | 100000 |
+| default        | ranked |     |            | 100000 |
+| age            | int    |   0 | 2147483647 |        |
+| location       | ranked |     |            |   1000 |
+| latitude       | int    |   0 | 2147483647 |        |
+| longitude      | int    |   0 | 2147483647 |        |
+| location-city  | ranked |     |            | 100000 |
+| location-state | ranked |     |            | 100000 |
 
-All frames are created as ranked frames by default, with the cache size listed above. Fields are created with
-a minimum size of zero and a fixed maximum of 2147483647. Fields at the top level
-are created in the default frame. Frames are a dash-separated concatenation of
-all key values in the path - you can see this with frames like location-city.
+All set fields are created as ranked fields by default, with the cache size 
+listed above. Integer fields are created with a minimum size of zero and a 
+fixed maximum of 2147483647. Field names are a dash-separated concatenation of
+all key values in the path - you can see this with fields like location-city.
 
 
 Most of the options to `pdk kafka` are self-explanatory (kafka hosts, pilosa hosts,
 kafka topics, kafka group, etc.), but there are a few options that give some
 control over the way data is indexed, and ingestion performance.
 
-* `--batch-size`: The batch size controls how many set bits or values are batched up to be imported *per frame*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner.
-* `--framer.collapse`: This is a list of strings which will be removed from the frame names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be frames named "city" and "state" rather than "location-city" and "location-state".
+* `--batch-size`: The batch size controls how many set bits or values are batched up to be imported *per field*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner.
+* `--framer.collapse`: This is a list of strings which will be removed from the field names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be fields named "city" and "state" rather than "location-city" and "location-state".
 * `--framer.ignore`: This allows you to skip indexing on any path containing these strings. If you have a field like email address or some other unique ID, you might not want to index it.
 * `--subject-path`: If nothing is passed for this option, then each record will be assigned a unique sequential column ID. If `subject-path` is specified, then the value at this path in the record will be mapped to a column ID. If the same value appears in another record, the same column ID will be used.
 * `--proxy`: The PDK ingests data, but also keeps a mapping for string values to row IDs, and from subjects to column ids. Because of this, querying Pilosa directly may not be useful, since it only returns integer row and column ids. The PDK will start a proxy server which intercepts requests to Pilosa using strings for row and column ids, and translates them to the integers that Pilosa understands. It will also translate responses so that (e.g.) a TopN query will return `{"results":[[{"Key":"chipotle dip","Count":1},{"Key":"corn chips","Count":1}]]}`. By default, the mapping is stored in an embedded leveldb.

docs/query-language.md

@@ -1,4 +1,4 @@
-+++
+v+++
 title = "Query Language"
 weight = 6
 nav = [
@@ -29,13 +29,13 @@ There will be one item in the `results` array for each PQL query in the request.
 
 ##### Examples
 
-Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index, frames, and populate them with some data.
+Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index and fields, and to populate them with some data.
 
-The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", col=10, row=1)` against a server using curl, you would:
+The examples just show the PQL quer(ies) needed - to run the query `Set(10, stargazer=1)` against a server using curl, you would:
 ``` request
 curl localhost:10101/index/repository/query \
      -X POST \
-     -d 'SetBit(frame="stargazer", col=10, row=1)'
+     -d 'Set(10, stargazer=1)'
 ```
 ``` response
 {"results":[true]}
@@ -43,28 +43,27 @@ curl localhost:10101/index/repository/query \
 
 #### Arguments and Types
 
-* `frame` The frame specifies on which Pilosa [frame](../glossary/#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
-* `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04")
+* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
+* `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04)
 * `UINT` An unsigned integer (e.g. 42839)
 * `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*`
 * `ATTR_VALUE` Can be a string, float, integer, or bool.
-* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Xor`, `Intersect`, `Range`
+* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Range`
 * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`)
 
 ### Write Operations
 
-#### SetBit
+#### Set
 
 **Spec:**
 
 ```
-SetBit(<frame=STRING>, <row=UINT>, <col=UINT>, 
-       [timestamp=TIMESTAMP])
+Set(<COLUMN>, <FIELD>=<ROW>, [TIMESTAMP])
 ```
 
 **Description:**
 
-`SetBit` assigns a value of 1 to a bit in the binary matrix, thus associating the given row in the given frame with the given column.
+`Set` assigns a value of 1 to a bit in the binary matrix, thus associating the given row (the `<ROW>` value) in the given field with the given column.
 
 **Result Type:** boolean
 
@@ -77,17 +76,17 @@ A return value of `false` indicates that the bit was already set to 1 and nothin
 
 Set the bit at row 1, column 10:
 ```request
-SetBit(frame="stargazer", col=10, row=1)
+Set(10, stargazer=1)
 ```
 ```response
 {"results":[true]}
 ```
 
-This sets a bit in the stargazer frame, representing that the user with id=1 has starred the repository with id=10.
+This sets a bit in the stargazer field, representing that the user with id=1 has starred the repository with id=10.
 
-SetBit also supports providing a timestamp. To write the date that a user starred a repository:
+Set also supports providing a timestamp. To write the date that a user starred a repository:
 ```request
-SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00")
+Set(10, stargazer=1, 2016-01-01T00:00)
 ```
 ```response
 {"results":[true]}
@@ -95,24 +94,32 @@ SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00")
 
 Set multiple bits in a single request:
 ```request
-SetBit(frame="stargazer", col=10, row=1) SetBit(frame="stargazer", col=10, row=2) SetBit(frame="stargazer", col=20, row=1) SetBit(frame="stargazer", col=30, row=2)
+Set(1, stargazer=10) Set(2, stargazer=10) Set(1, stargazer=20) Set(2, stargazer=30)
 ```
 ```response
 {"results":[false,true,true,true]}
 ```
 
+Set the field "pullrequests" to integer value 2 at column 10:
+```request
+Set(10, pullrequests=2)
+```
+```response
+{"results":[true]}
+```
+
 #### SetRowAttrs
 **Spec:**
 
 ```
-SetRowAttrs(<frame=STRING>, <row=UINT>, 
+SetRowAttrs(<FIELD>, <ROW>, 
             <ATTR_NAME=ATTR_VALUE>, 
             [ATTR_NAME=ATTR_VALUE ...])
 ```
 
 **Description:**
 
-`SetRowAttrs` associates arbitrary key/value pairs with a row in a frame. Setting a value of `null`, without quotes, deletes an attribute.
+`SetRowAttrs` associates arbitrary key/value pairs with a row in a field. Setting a value of `null`, without quotes, deletes an attribute.
 
 **Result Type:** null
 
@@ -122,17 +129,17 @@ SetRowAttrs queries always return `null` upon success.
 
 Set attributes `username` and `active` on row 10:
 ```request
-SetRowAttrs(frame="stargazer", row=10, username="mrpi", active=true)
+SetRowAttrs(stargazer, 10, username="mrpi", active=true)
 ```
 ```response
 {"results":[null]}
 ```
 
-Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", row=10)`.
+Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Row](../query-language/#row) query like so `Row(stargazer=10)`.
 
 Delete attribute `username` on row 10:
 ```request
-SetRowAttrs(frame="stargazer", row=10, username=null)
+SetRowAttrs(stargazer, 10, username=null)
 ```
 ```response
 {"results":[null]}
@@ -143,7 +150,7 @@ SetRowAttrs(frame="stargazer", row=10, username=null)
 **Spec:**
 
 ```
-SetColumnAttrs(<frame=STRING>, <row=UINT>, 
+SetColumnAttrs(<COLUMN>, 
                <ATTR_NAME=ATTR_VALUE>, 
                [ATTR_NAME=ATTR_VALUE ...])
 ```
@@ -154,13 +161,13 @@ SetColumnAttrs(<frame=STRING>, <row=UINT>,
 
 **Result Type:** null
 
-SetColumnAttrs queries always return `null` upon success. Setting a value of `null`, without quotes, deletes an attribute. To avoid confusion, `frame` cannot be used as an attribute name.
+SetColumnAttrs queries always return `null` upon success. Setting a value of `null`, without quotes, deletes an attribute.
 
 **Examples:**
 
 Set attributes `stars`, `url`, and `active` on column 10:
 ```request
-SetColumnAttrs(col=10, stars=123, url="http://projects.pilosa.com/10", active=true)
+SetColumnAttrs(10, stars=123, url="http://projects.pilosa.com/10", active=true)
 ```
 ```response
 {"results":[null]}
@@ -170,13 +177,13 @@ Set url value and active status for project 10. These are arbitrary key/value pa
 
 ColumnAttrs can be requested by adding the URL parameter `columnAttrs=true` to a query. For example:
 ```request
-curl localhost:10101/index/repository/query?columnAttrs=true -XPOST -d 'Bitmap(frame="stargazer", row=1)Bitmap(frame="stargazer", row=2)'
+curl localhost:10101/index/repository/query?columnAttrs=true -XPOST -d 'Row(stargazer=1) Row(stargazer=2)'
 ```
 ```response
 {
   "results":[
-    {"attrs":{},"bits":[10,20]},
-    {"attrs":{},"bits":[10,30]}
+    {"attrs":{},"cols":[10,20]},
+    {"attrs":{},"cols":[10,30]}
   ],
   "columnAttrs":[
     {"id":10,"attrs":{"active":true,"stars":123,"url":"http://projects.pilosa.com/10"}},
@@ -189,25 +196,25 @@ In this example, ColumnAttrs have been set on columns 10 and 20, but not column
 
 Delete the `url` attribute on column 10:
 ```request
-SetColumnAttrs(col=10, url=null)
+SetColumnAttrs(10, url=null)
 ```
 ```response
 {"results":[null]}
 ```
 
-#### ClearBit
+#### Clear
 
 **Spec:**
 
 ```
-ClearBit(<frame=STRING>, <row=UINT>, <col=UINT>)
+Clear(<COLUMN>, <FIELD>=<ROW>)
 ```
 
 **Description:**
 
-`ClearBit` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given frame from the given column.
+`Clear` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given field from the given column.
 
-Note that clearing bits from time views is not supported.
+Note that clearing a column on a time field will remove all data for that column.
 
 **Result Type:** boolean
 
@@ -217,9 +224,9 @@ A return value of `false` indicates that the bit was already set to 0 and nothin
 
 **Examples:**
 
-Clear the bit at row 1 and column 10 in the stargazer frame:
+Clear the bit at row 1 and column 10 in the stargazer field:
 ```request
-ClearBit(frame="stargazer", col=10, row=1)
+Clear(10, stargazer=1)
 ```
 ```response
 {"results":[true]}
@@ -227,79 +234,48 @@ ClearBit(frame="stargazer", col=10, row=1)
 
 This represents removing the relationship between the user with id=1 and the repository with id=10.
 
-#### SetFieldValue
-
-**Spec:**
-
-```
-SetFieldValue(<col=UINT>, <frame=STRING>, <FIELD_NAME=INT>)
-```
-
-**Description:**
-
-`SetFieldValue` assigns an integer value with the specified field name to the `col` in the given `frame`.
-
-**Result Type:** null
-
-SetFieldValue returns `null` upon success.
-
-**Examples:**
-
-Set the field value `pullrequest` to the value 2, on column 10 in frame `stats`:
-```request
-SetFieldValue(col=10, frame="stats", pullrequests=2)
-```
-```response
-{"results":[null]}
-```
-
-This represents setting the number of pull requests of repository 10 to 2.
-
-This example assumes the existence of the frame `stats` and the field `pullrequests`. See [frame creation](../api-reference/#create-frame) and [field creation](../api-reference/#create-field) for more information.
-
-
 ### Read Operations
 
-#### Bitmap
+#### Row
 
 **Spec:**
 
 ```
-Bitmap(<frame=STRING>, (<rowL=UINT> | <col>=UINT))
+Row(<FIELD>=<ROW>)
 ```
 
 **Description:**
 
-`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row or column argument is provided in the query. It also retrieves any attributes set on that row or column.
+`Row` retrieves the indices of all the columns in a row. It also retrieves any attributes set on that row.
 
-**Result Type:** object with attrs and bits.
+**Result Type:** object with attrs and columns.
 
-e.g. `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}`
+e.g. `{"attrs":{"username":"mrpi","active":true},"columns":[10, 20]}`
 
 **Examples:**
 
-Query all columns with a bit set in row 1 of the frame `stargazer` (repositories that are starred by user 1):
+Query all columns with a bit set in row 1 of the field `stargazer` (repositories that are starred by user 1):
 ```request
-Bitmap(frame="stargazer", row=1)
+Row(stargazer=1)
 ```
 ```response
-{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}
+{"attrs":{"username":"mrpi","active":true},"columns":[10, 20]}
 ```
 
 * attrs are the attributes for user 1 
-* bits are the repositories which user 1 has starred.
+* columns are the repositories which user 1 has starred.
 
 #### Union
 
 **Spec:**
 
 ```
-Union([BITMAP_CALL ...])
+Union([ROW_CALL ...])
 ```
 
 **Description:**
 
-Union performs a logical OR on the results of all `BITMAP_CALL` queries passed to it.
+Union performs a logical OR on the results of all `ROW_CALL` queries passed to it.
 
 **Result Type:** object with attrs and bits
 
@@ -309,28 +285,27 @@ attrs will always be empty
 
 Query columns with a bit set in either of two rows (repositories that are starred by either of two users):
 ```request
-Union(Bitmap(frame="stargazer", stargazer_id=1), Bitmap(frame="stargazer", stargazer_id=2))
+Union(Row(stargazer=1), Row(stargazer=2))
 ```
 ```response
-{"attrs":{},"bits":[10, 20, 30]}
+{"attrs":{},"columns":[10, 20, 30]}
 ```
 
-* bits are repositories that were starred by user 1 OR user 2
+* columns are repositories that were starred by user 1 OR user 2
 
 #### Intersect
 
-
 **Spec:**
 
 ```
-Intersect(<BITMAP_CALL>, [BITMAP_CALL ...])
+Intersect(<ROW_CALL>, [ROW_CALL ...])
 ```
 
 **Description:**
 
-Intersect performs a logical AND on the results of all `BITMAP_CALL` queries passed to it.
+Intersect performs a logical AND on the results of all `ROW_CALL` queries passed to it.
 
-**Result Type:** object with attrs and bits
+**Result Type:** object with attrs and columns
 
 attrs will always be empty
 
@@ -339,27 +314,27 @@ attrs will always be empty
 Query columns with a bit set in both of two rows (repositories that are starred by both of two users):
 
 ```request
-Intersect(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2))
+Intersect(Row(stargazer=1), Row(stargazer=2))
 ```
 ```response
-{"attrs":{},"bits":[10]}
+{"attrs":{},"columns":[10]}
 ```
 
-* bits are repositories that were starred by user 1 AND user 2
+* columns are repositories that were starred by user 1 AND user 2
 
 #### Difference
 
 **Spec:**
 
 ```
-Difference(<BITMAP_CALL>, [BITMAP_CALL ...])
+Difference(<ROW_CALL>, [ROW_CALL ...])
 ```
 
 **Description:**
 
-Difference returns all of the bits from the first `BITMAP_CALL` argument passed to it, without the bits from each subsequent `BITMAP_CALL`.
+Difference returns all of the bits from the first `ROW_CALL` argument passed to it, without the bits from each subsequent `ROW_CALL`.
 
-**Result Type:** object with attrs and bits
+**Result Type:** object with attrs and columns
 
 attrs will always be empty
 
@@ -367,37 +342,37 @@ attrs will always be empty
 
 Query columns with a bit set in one row and not another (repositories that are starred by one user and not another):
 ```request
-Difference(Bitmap(frame="stargazer", row=1), Bitmap( frame="stargazer", row=2))
+Difference(Row(stargazer=1), Row(stargazer=2))
 ```
 ```response
-{"results":[{"attrs":{},"bits":[20]}]}
+{"results":[{"attrs":{},"columns":[20]}]}
 ```
 
-* bits are repositories that were starred by user 1 BUT NOT user 2
+* columns are repositories that were starred by user 1 BUT NOT user 2
 
 Query for the opposite difference:
 ```request
-Difference(Bitmap(frame="stargazer", row=2), Bitmap( frame="stargazer", row=1))
+Difference(Row(stargazer=2), Row(stargazer=1))
 ```
 ```response
-{"attrs":{},"bits":[30]}
+{"attrs":{},"columns":[30]}
 ```
 
-* Bits are repositories that were starred by user 2 BUT NOT user 1
+* columnss are repositories that were starred by user 2 BUT NOT user 1
 
 #### Xor
 
 **Spec:**
 
 ```
-Xor(<BITMAP_CALL>, [BITMAP_CALL ...])
+Xor(<ROW_CALL>, [ROW_CALL ...])
 ```
 
 **Description:**
 
-Xor performs a logical XOR on the results of each `BITMAP_CALL` query passed to it.
+Xor performs a logical XOR on the results of each `ROW_CALL` query passed to it.
 
-**Result Type:** object with attrs and bits
+**Result Type:** object with attrs and columns
 
 attrs will always be empty
 
@@ -406,24 +381,24 @@ attrs will always be empty
 Query columns with a bit set in exactly one of two rows (repositories that are starred by only one of two users):
 
 ```request
-Xor(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2))
+Xor(Row(stargazer=2), Row(stargazer=1))
 ```
 ```response
-{"results":[{"attrs":{},"bits":[10,20,30]}]}
+{"results":[{"attrs":{},"columns":[10,20,30]}]}
 ```
 
-* bits are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both)
+* columns are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both)
 
 #### Count
 **Spec:**
 
 ```
-Count(<BITMAP_CALL>)
+Count(<ROW_CALL>)
 ```
 
 **Description:**
 
-Returns the number of set bits in the `BITMAP_CALL` passed in.
+Returns the number of set bits in the `ROW_CALL` passed in.
 
 **Result Type:** int
 
@@ -431,7 +406,7 @@ Returns the number of set bits in the `BITMAP_CALL` passed in.
 
 Query the number of bits set in a row (the number of repositories a user has starred):
 ```request
-Count(Bitmap(frame="stargazer", row=1))
+Count(Row(stargazer=1))
 ```
 ```response
 {"results":[1]}
@@ -444,34 +419,34 @@ Count(Bitmap(frame="stargazer", row=1))
 **Spec:**
 
 ```
-TopN([BITMAP_CALL], <frame=STRING>, [n=UINT],
-     [<field=ATTR_NAME>, <filters=[]ATTR_VALUE>])
+TopN([ROW_CALL], <FIELD>, [n=UINT],
+     [attrName=<ATTR_NAME>, attrValues=<[]ATTR_VALUE>])
 ```
 
 **Description:**
 
-Return the id and count of the top `n` bitmaps (by count of bits) in the frame.
-The `field` and `filters` arguments work together to only return Bitmaps which
-have the attribute specified by `field` with one of the values specified in
-`filters`.
+Return the id and count of the top `n` rows (by count of bits) in the field.
+The `attrName` and `attrValues` arguments work together to only return rows which
+have the attribute specified by `attrName` with one of the values specified in
+`attrValues`.
 
 **Result Type:** array of key/count objects
 
 **Caveats:**
 
-* Performing a TopN() query on a frame with cache type ranked will return the top bitmaps sorted by count in descending order.
-* Frames with cache type lru will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of frame will return bitmaps sorted in order of most recently set bit.
-* The frame's cache size determines the number of sorted bitmaps to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance.
-* Once full, the cache will truncate the set of bitmaps according to the frame option CacheSize. Bitmaps that straddle the limit and have the same count will be truncated in no particular order.
-* The TopN query's attribute filter is applied to the existing sorted cache of bitmaps. Bitmaps that fall outside of the sorted cache range, even if they would normally pass the filter, are ignored.
+* Performing a TopN() query on a field with cache type ranked will return the top rows sorted by count in descending order.
+* Fields with cache type lru will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of field will return rows sorted in order of most recently set bit.
+* The field's cache size determines the number of sorted rows to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance.
+* Once full, the cache will truncate the set of rows according to the field option CacheSize. Rows that straddle the limit and have the same count will be truncated in no particular order.
+* The TopN query's attribute filter is applied to the existing sorted cache of rows. Rows that fall outside of the sorted cache range, even if they would normally pass the filter, are ignored.
 
-See [frame creation](../api-reference/#create-frame) for more information about the cache.
+See [field creation](../api-reference/#create-field) for more information about the cache.
 
 **Examples:**
 
 Basic TopN query:
 ```request
-TopN(frame="stargazer")
+TopN(stargazer)
 ```
 ```response
 {"results":[[{"id":1240,"count":102},{"id":4734,"count":100},{"id":12709,"count":93},...]]}
@@ -479,11 +454,11 @@ TopN(frame="stargazer")
 
 * `id` is a row ID (user ID)
 * `count` is a count of columns (repositories)
-* Results are the number of bits set in the corresponding row (repositories that each user starred) in descending order for all rows (users) in the stargazer frame. For example user 1240 starred 102 repositories, user 4734 starred 100 repositories, user 12709 starred 93 repository.
+* Results are the number of bits set in the corresponding row (repositories that each user starred) in descending order for all rows (users) in the stargazer field. For example user 1240 starred 102 repositories, user 4734 starred 100 repositories, user 12709 starred 93 repository.
 
 Limit the number of results:
 ```request
-TopN(frame="stargazer", n=2)
+TopN(stargazer, n=2)
 ```
 ```response
 {"results":[[{"id":1240,"count":102},{"id":4734,"count":100}]]}
@@ -491,19 +466,19 @@ TopN(frame="stargazer", n=2)
 
 * Results are the top two rows (users) sorted by number of bits set (repositories they've starred) in descending order.
 
-Filter based on an existing Bitmap:
+Filter based on an existing row:
 ```request
-TopN(Bitmap(frame="language", row=1), frame="stargazer", n=2)
+TopN(Row(language=1), stargazer, n=2)
 ```
 ```response
 {"results":[[{"id":1240,"count":35},{"id":7508,"count":32}]]}
 ```
 
-* Results are the top two users (rows) sorted by the number of bits set in the intersection with row 1 of the language frame (repositories that they've starred which are written in language 1).
+* Results are the top two users (rows) sorted by the number of bits set in the intersection with row 1 of the language field (repositories that they've starred which are written in language 1).
 
 Filter based on attributes:
 ```request
-TopN(frame="stargazer", n=2, field=active, filters=[true])
+TopN(stargazer, n=2, attrName=active, attrValues=[true])
 ```
 ```response
 {"results":[[{"id":10,"count":1},{"id":13,"count":1}]]}
@@ -516,31 +491,30 @@ TopN(frame="stargazer", n=2, field=active, filters=[true])
 **Spec:**
 
 ```
-Range(<frame=STRING>, <row=UINT>, 
-      <start=TIMESTAMP>, <end=TIMESTAMP>)
+Range(<FIELD>=<ROW>, <TIMESTAMP>, <TIMESTAMP>)
 ```
 
 **Description:**
 
-Similar to `Bitmap`, but only returns bits which were set with timestamps
-between the given `start` and `end` timestamps. 
+Similar to `Row`, but only returns bits which were set with timestamps
+between the given `start` (first) and `end` (second) timestamps. 
 
 **Result Type:** object with attrs and bits
 
 
 **Examples:**
 
-Query all columns with a bit set in row 1 of a frame (repositories that a user has starred), within a date range:
+Query all columns with a bit set in row 1 of a field (repositories that a user has starred), within a date range:
 ```request
-Range(frame="stargazer", row=1, start="2010-01-01T00:00", end="2017-03-02T03:00")
+Range(stargazer=1, 2010-01-01T00:00, 2017-03-02T03:00)
 ```
 ```response
-{{"attrs":{},"bits":[10]}
+{{"attrs":{},"columns":[10]}
 ```
 
 This example assumes timestamps have been set on some bits.
 
-* bits are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02.
+* columns are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02.
 
 
 #### Range (BSI)
@@ -548,16 +522,15 @@ This example assumes timestamps have been set on some bits.
 **Spec:**
 
 ```
-Range(<frame=STRING>, <FIELD_NAME, COMPARISON_OPERATOR, COMPARISON_VALUE> )
+Range([<COMPARISON_VALUE> <COMPARISON_OPERATOR>] <FIELD> <COMPARISON_OPERATOR> <COMPARISON_VALUE>)
 ```
 
 **Description:**
 
-The `Range` query is overloaded to work on `field` values as well as `timestamp` values.
+The `Range` query is overloaded to work on `integer` values as well as `timestamp` values.
 Returns bits that are true for the comparison operator.
 
-**Result Type:** object with attrs and bits
-
+**Result Type:** object with attrs and columns
 
 **Examples:**
 
@@ -565,13 +538,13 @@ In our source data, commitactivity was counted over the last year.
 The following greater-than `Range` query returns all columns with a field value greater than 100 (repositories having more than 100 commits):
 
 ```request
-Range(frame="stats", commitactivity > 100)
+Range(commitactivity > 100)
 ```
 ```response
-{{"attrs":{},"bits":[10]}
+{{"attrs":{},"columns":[10]}
 ```
 
-* bits are repositories which had at least 100 commits in the last year.
+* columns are repositories which had at least 100 commits in the last year.
 
 BSI range queries support the following operators:
 
@@ -583,35 +556,37 @@ BSI range queries support the following operators:
  `>=`     | greater-than-or-equal-to, GTE | integer            
  `==`     | equal-to, EQ                  | integer            
  `!=`     | not-equal-to, NEQ             | integer or `null`  
- `><`     | between, BETWEEN              | [integer, integer] 
 
-The `BETWEEN` form specifies an interval with both bounds, using the `><` operator, and a two-element list containing the lower and upper bounds of the interval:
+`<`, and `<=` can be chained together to represent a bounded interval. For example:
 
-```pql
-Range(frame="stats", commitactivity >< [100, 200])
+```request
+Range(50 < commitactivity < 150)
+```
+```response
+{{"attrs":{},"columns":[10]}
 ```
 
-This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way.
+As of Pilosa 1.0, the "between" syntax `Range(frame=stats, commitactivity >< [50, 150])` is no longer supported.
 
 #### Min
 
 **Spec:**
 
 ```
-Min([BITMAP_CALL], <frame=STRING>, <field=STRING>)
+Min([ROW_CALL], field=<FIELD>)
 ```
 
 **Description:**
 
-Returns the minimum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered.
+Returns the minimum value of all BSI integer values in this `field`. If the optional `Row` call is supplied, only columns with set bits are considered, otherwise all columns are considered.
 
 **Result Type:** object with the min and count of columns containing the min value.
 
 **Examples:**
 
-Query the minimum value of all fields in a frame (minimum size of all repositories):
+Query the minimum value of a field (minimum size of all repositories):
 ```request
-Min(frame="stats", field="diskusage")
+Min(field="diskusage")
 ```
 ```response
 {"value":4,"count":2}
@@ -624,20 +599,20 @@ Min(frame="stats", field="diskusage")
 **Spec:**
 
 ```
-Max([BITMAP_CALL], <frame=STRING>, <field=STRING>)
+Max([ROW_CALL], field=<FIELD>)
 ```
 
 **Description:**
 
-Returns the maximum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered.
+Returns the maximum value of all BSI integer values in this `field`. If the optional `Row` call is supplied, only columns with set bits are considered, otherwise all columns are considered.
 
 **Result Type:** object with the max and count of columns containing the max value.
 
 **Examples:**
 
-Query the maximum value of all fields in a frame (maximum size of all repositories):
+Query the maximum value of a field (maximum size of all repositories):
 ```request
-Max(frame="stats", field="diskusage")
+Max(field="diskusage")
 ```
 ```response
 {"value":88,"count":13}
@@ -650,20 +625,20 @@ Max(frame="stats", field="diskusage")
 **Spec:**
 
 ```
-Sum([BITMAP_CALL], <frame=STRING>, <field=STRING>)
+Sum([ROW_CALL], field=<FIELD>)
 ```
 
 **Description:**
 
-Returns the count and computed sum of all BSI integer values in the `field` and `frame`. If the optional `Bitmap` call is supplied, columns with set bits are summed, otherwise the sum is across all columns.
+Returns the count and computed sum of all BSI integer values in the `field`. If the optional `Row` call is supplied, columns with set bits are summed, otherwise the sum is across all columns.
 
-**Result Type:** object with the computed sum and count of the bitmap field.
+**Result Type:** object with the computed sum and count of the values in the integer field.
 
 **Examples:**
 
 Query the size of all repositories.
 ```request
-Sum(frame="stats", field="diskusage")
+Sum(field="diskusage")
 ```
 ```response
 {"value":10,"count":3}

docs/tutorials.md

@@ -407,37 +407,65 @@ curl localhost:10101/index/patients \
      -X POST 
 ```
 ``` response
-{}
+{"success":true}
 ```
 
-In addition to storing rows of bits, a frame can also contain fields that store integer values. The next step creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame.
+In addition to storing rows of bits, a frame can also contain fields that store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame.
 ``` request
-curl localhost:10101/index/patients/frame/measurements \
+curl localhost:10101/index/patients/field/age \
      -X POST \
-     -d '{"options":{
-              "fields": [
-                  {"name": "age", "type": "int", "min": 0, "max": 120},
-                  {"name": "weight", "type": "int", "min": 0, "max": 500},
-                  {"name": "tcells", "type": "int", "min": 0, "max": 2000}
-              ]
-         }}'
+     -d '{"options":{"type": "int", "min": 0, "max": 120}}'
 ```
 ``` response
-{}
+{"success":true}
 ```
 
-If you need to, you can add fields to an existing frame by posting to the [Create Field endpoint](../api-reference/#create-field).
+``` request
+curl localhost:10101/index/patients/field/weight \
+     -X POST \
+     -d '{"options":{"type": "int", "min": 0, "max": 500}}'
+```
+``` response
+{"success":true}
+```
+
+``` request
+curl localhost:10101/index/patients/field/tcells \
+     -X POST \
+     -d '{"options":{"type": "int", "min": 0, "max": 2000}}'
+```
+``` response
+{"success":true}
+```
 
 Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL.
 
-This query sets the age, weight, and t-cell count for the patient with ID `1` in our system:
+The following queries set the age, weight, and t-cell count for the patient with ID `1` in our system:
 ``` request
 curl localhost:10101/index/patients/query \
      -X POST \
-     -d 'SetFieldValue(col=1, frame="measurements", age=34, weight=128, tcells=1145)'
+     -d 'Set(1, age=34)'
 ```
 ``` response
-{"results":[null]}
+{"results":[true]}
+```
+
+``` request
+curl localhost:10101/index/patients/query \
+     -X POST \
+     -d 'Set(1, weight=128)'
+```
+``` response
+{"results":[true]}
+```
+
+``` request
+curl localhost:10101/index/patients/query \
+     -X POST \
+     -d 'Set(1, tcells=1145)'
+```
+``` response
+{"results":[true]}
 ```
 
 In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file.
@@ -454,7 +482,7 @@ Assuming we have a file called `ages.csv` that is structured like this:
 8,33
 9,63
 ```
-where the first column of the CSV represents the patient `ID` and the second column represents the patient's`age`, then we can import the data into our `age` field by running this command:
+where the first column of the CSV represents the patient `ID` and the second column represents the patient's `age`, then we can import the data into our `age` field by running this command:
 ```
 pilosa import -i patients -f measurements --field age ages.csv
 ```
@@ -465,10 +493,10 @@ In order to find all patients over the age of 40, then simply run a `Range` quer
 ``` request
 curl localhost:10101/index/patients/query \
      -X POST \
-     -d 'Range(frame="measurements", age > 40)'
+     -d 'Range(age > 40)'
 ```
 ``` response
-{"results":[{"attrs":{},"bits":[2,6,9]}]}
+{"results":[{"attrs":{},"columns":[2,6,9]}]}
 ```
 
 You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation.
@@ -477,21 +505,21 @@ To find the average age of all patients, run a `Sum` query:
 ``` request
 curl localhost:10101/index/patients/query \
      -X POST \
-     -d 'Sum(frame="measurements", field="age")'
+     -d 'Sum(field="age")'
 ```
 ``` response
-{"results":[{"sum":377,"count":9}]}
+{"results":[{"value":377,"count":9}]}
 ```
-The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`.
+The results you get from the `Sum` query contain the sum of all values as well as the `count` of columns with a value. To get the average you can just divide `value` by `count`.
 
 You can also provide a filter to the `Sum()` function to find the average age of all patients over 40.
 ``` request
 curl localhost:10101/index/patients/query \
      -X POST \
-     -d 'Sum(Range(frame="measurements", age > 40), frame="measurements", field="age")'
+     -d 'Sum(Range(age > 40), field="age")'
 ```
 ``` response
-{"results":[{"sum":191,"count":3}]}
+{"results":[{"value":191,"count":3}]}
 ```
 Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query.
 
@@ -499,42 +527,42 @@ To find the minimum age of all patients, run a `Min` query:
 ``` request
 curl localhost:10101/index/patients/query \
      -X POST \
-     -d 'Min(frame="measurements", field="age")'
+     -d 'Min(field="age")'
 ```
 ``` response
-{"results":[{"min":19,"count":1}]}
+{"results":[{"value":19,"count":1}]}
 ```
-The results you get from the `Min` query contain the `min` of all values as well as the `count` of columns with that value.
+The results you get from the `Min` query contain the minimum `value` of all values as well as the `count` of columns with that value.
 
 You can also provide a filter to the `Min()` function to find the minimum age of all patients over 40.
 ``` request
 curl localhost:10101/index/patients/query \
      -X POST \
-     -d 'Min(Range(frame="measurements", age > 40), frame="measurements", field="age")'
+     -d 'Min(Range(age > 40), field="age")'
 ```
 ``` response
-{"results":[{"min":57,"count":1}]}
+{"results":[{"value":57,"count":1}]}
 ```
 
 To find the maximum age of all patients, run a `Max` query:
 ``` request
 curl localhost:10101/index/patients/query \
      -X POST \
-     -d 'Max(frame="measurements", field="age")'
+     -d 'Max(field="age")'
 ```
 ``` response
-{"results":[{"max":71,"count":1}]}
+{"results":[{"value":71,"count":1}]}
 ```
-The results you get from the `Max` query contain the `max` of all values as well as the `count` of columns with that value.
+The results you get from the `Max` query contain the maximum `value` of all values as well as the `count` of columns with that value.
 
 You can also provide a filter to the `Max()` function to find the maximum age of all patients under 40.
 ``` request
 curl localhost:10101/index/patients/query \
      -X POST \
-     -d 'Max(Range(frame="measurements", age < 40), frame="measurements", field="age")'
+     -d 'Max(Range(age < 40), field="age")'
 ```
 ``` response
-{"results":[{"max":34,"count":1}]}
+{"results":[{"value":34,"count":1}]}
 ```
 
 ### Storing Row and Column Attributes
@@ -549,28 +577,28 @@ curl localhost:10101/index/books \
      -X POST
 ```
 ``` response
-{}
+{"success":true}
 ```
 
-Next, create a frame in the `books` index called `members` which will represent library members who have read books.
+Next, create a field in the `books` index called `members` which will represent library members who have read books.
 ``` request
-curl localhost:10101/index/books/frame/members \
+curl localhost:10101/index/books/field/members \
      -X POST \
      -d '{}'
 ```
 ``` response
-{}
+{"success":true}
 ```
 
 Now, let's add some books to our index.
 ``` request
 curl localhost:10101/index/books/query \
      -X POST \
-     -d 'SetColumnAttrs(col=1, name="To Kill a Mockingbird", year=1960)
-         SetColumnAttrs(col=2, name="No Name in the Street", year=1972)
-         SetColumnAttrs(col=3, name="The Tipping Point", year=2000)
-         SetColumnAttrs(col=4, name="Out Stealing Horses", year=2003)
-         SetColumnAttrs(col=5, name="The Forever War", year=2008)'
+     -d 'SetColumnAttrs(1, name="To Kill a Mockingbird", year=1960)
+         SetColumnAttrs(2, name="No Name in the Street", year=1972)
+         SetColumnAttrs(3, name="The Tipping Point", year=2000)
+         SetColumnAttrs(4, name="Out Stealing Horses", year=2003)
+         SetColumnAttrs(5, name="The Forever War", year=2008)'
 ```
 ``` response
 {"results":[null,null,null,null,null]}
@@ -580,11 +608,11 @@ And add some members.
 ``` request
 curl localhost:10101/index/books/query \
      -X POST \
-     -d 'SetRowAttrs(frame="members", row=10001, fullName="John Smith")
-         SetRowAttrs(frame="members", row=10002, fullName="Sue Perkins")
-         SetRowAttrs(frame="members", row=10003, fullName="Jennifer Hawks")
-         SetRowAttrs(frame="members", row=10004, fullName="Pedro Vazquez")
-         SetRowAttrs(frame="members", row=10005, fullName="Pat Washington")'
+     -d 'SetRowAttrs(members, 10001, fullName="John Smith")
+         SetRowAttrs(members, 10002, fullName="Sue Perkins")
+         SetRowAttrs(members, 10003, fullName="Jennifer Hawks")
+         SetRowAttrs(members, 10004, fullName="Pedro Vazquez")
+         SetRowAttrs(members, 10005, fullName="Pat Washington")'
 ```
 ``` response
 {"results":[null,null,null,null,null]}
@@ -594,29 +622,29 @@ At this point we can query one of the `member` records by querying that row.
 ``` request
 curl localhost:10101/index/books/query \
      -X POST \
-     -d 'Bitmap(frame="members", row=10002)'
+     -d 'Row(members=10002)'
 ```
 ``` response
-{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]}
+{"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[]}]}
 ```
 
 Now let's add some data to the matrix such that each pair represents a member who has read that book.
 ``` request
 curl localhost:10101/index/books/query \
      -X POST \
-     -d 'SetBit(frame="members", row=10001, col=3)
-         SetBit(frame="members", row=10001, col=5)
-         SetBit(frame="members", row=10002, col=1)
-         SetBit(frame="members", row=10002, col=2)
-         SetBit(frame="members", row=10002, col=4)
-         SetBit(frame="members", row=10003, col=3)
-         SetBit(frame="members", row=10004, col=4)
-         SetBit(frame="members", row=10004, col=5)
-         SetBit(frame="members", row=10005, col=1)
-         SetBit(frame="members", row=10005, col=2)
-         SetBit(frame="members", row=10005, col=3)
-         SetBit(frame="members", row=10005, col=4)
-         SetBit(frame="members", row=10005, col=5)'
+     -d 'Set(3, members=10001)
+         Set(5, members=10001)
+         Set(1, members=10002)
+         Set(2, members=10002)
+         Set(4, members=10002)
+         Set(3, members=10003)
+         Set(4, members=10004)
+         Set(5, members=10004)
+         Set(1, members=10005)
+         Set(2, members=10005)
+         Set(3, members=10005)
+         Set(4, members=10005)
+         Set(5, members=10005)'
 ```
 ``` response
 {"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]}
@@ -626,22 +654,22 @@ Now pull the record for `Sue Perkins` again.
 ``` request
 curl localhost:10101/index/books/query \
      -X POST \
-     -d 'Bitmap(frame="members", row=10002)'
+     -d 'Row(members=10002)'
 ```
 ``` response
-{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]}
+{"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[1,2,4]}]}
 ```
-Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read.
+Notice that the result set now contains a list of integers in the `columns` attribute. These integers match the column IDs of the books that Sue has read.
 
 In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query.
 ``` request
 curl localhost:10101/index/books/query?columnAttrs=true \
      -X POST \
-     -d 'Bitmap(frame="members", row=10002)'
+     -d 'Row(members=10002)'
 ```
 ``` response
 {
-  "results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}],
+  "results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[1,2,4]}],
   "columnAttrs":[
     {"id":1,"attrs":{"name":"To Kill a Mockingbird","year":1960}},
     {"id":2,"attrs":{"name":"No Name in the Street","year":1972}},
@@ -655,11 +683,11 @@ Finally, if we want to find out which books were read by both `Sue` and `Pedro`,
 ``` request
 curl localhost:10101/index/books/query?columnAttrs=true \
      -X POST \
-     -d 'Intersect(Bitmap(frame="members", row=10002), Bitmap(frame="members", row=10004))'
+     -d 'Intersect(Row(members=10002), Row(members=10004))'
 ```
 ``` response
 {
-  "results":[{"attrs":{},"bits":[4]}],
+  "results":[{"attrs":{},"columns":[4]}],
   "columnAttrs":[
     {"id":4,"attrs":{"name":"Out Stealing Horses","year":2003}}
   ]

docs/webui.md

@@ -9,13 +9,27 @@ nav = [
 
 ## WebUI
 
-The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101).
+A web-based app called Pilosa WebUI is available in a separate package. This can be used for constructing queries and viewing the cluster status.
 
-This can be used for constructing queries and viewing the cluster status.
+### Installation
+
+Releases are [available on Github](https://github.com/pilosa/webui/releases) as well as on [Homebrew](https://brew.sh/) for Mac.
+
+Installing on a Mac with Homebrew is simple; just run:
+
+```
+brew install pilosa-webui
+```
+
+You may also build from source by checking out the [repo on Github](https://github.com/pilosa/webui) and running:
+
+```
+make install
+```
 
 ### Console
 
-The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
+The Console view allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
 
 Each query's result will be displayed in the Output section along with the query time. 
 
@@ -29,14 +43,14 @@ In addition to standard PQL, the console supports a few special commands, prefix
 - `:create index <indexname>`
 - `:delete index <indexname>`
 - `:use <indexname>`
-- `:create frame <framename>`
-- `:delete frame <framename>`
+- `:create field <fieldname>`
+- `:delete field <fieldname>`
 
-Frame creation also supports options like `timeQuantum`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame).
+Field creation also supports options like `timeQuantum`. When creating a new field, add options by using the keys documented in [API reference](../api-reference/#create-field).
 
-- `:create frame <framename> cacheSize=10000`
+- `:create field <fieldname> cacheSize=10000`
 
 
 ### Cluster Admin
 
-Use the [Cluster Admin tab](http://localhost:10101/#admin) to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames.
+Use the Cluster Admin tab to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Fields.

enterprise/b/btree.go

@@ -56,23 +56,23 @@ func init() {
 
 var (
 	btDPool = sync.Pool{New: func() interface{} { return &d{} }}
-	btEPool = btEpool{sync.Pool{New: func() interface{} { return &Enumerator{} }}}
-	btTPool = btTpool{sync.Pool{New: func() interface{} { return &Tree{} }}}
+	btEPool = btEpool{sync.Pool{New: func() interface{} { return &enumerator{} }}}
+	btTPool = btTpool{sync.Pool{New: func() interface{} { return &tree{} }}}
 	btXPool = sync.Pool{New: func() interface{} { return &x{} }}
 )
 
 type btTpool struct{ sync.Pool }
 
-func (p *btTpool) get(cmp Cmp) *Tree {
-	x := p.Get().(*Tree)
+func (p *btTpool) get(cmp Cmp) *tree {
+	x := p.Get().(*tree)
 	x.cmp = cmp
 	return x
 }
 
 type btEpool struct{ sync.Pool }
 
-func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *Tree, ver int64) *Enumerator {
-	x := p.Get().(*Enumerator)
+func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *tree, ver int64) *enumerator {
+	x := p.Get().(*enumerator)
 	x.err, x.hit, x.i, x.k, x.q, x.t, x.ver = err, hit, i, k, q, t, ver
 	return x
 }
@@ -98,26 +98,26 @@ type (
 		v *roaring.Container
 	}
 
-	// Enumerator captures the state of enumerating a tree. It is returned
+	// enumerator captures the state of enumerating a tree. It is returned
 	// from the Seek* methods. The enumerator is aware of any mutations
 	// made to the tree in the process of enumerating it and automatically
 	// resumes the enumeration at the proper key, if possible.
 	//
-	// However, once an Enumerator returns io.EOF to signal "no more
+	// However, once an enumerator returns io.EOF to signal "no more
 	// items", it does no more attempt to "resync" on tree mutation(s).  In
-	// other words, io.EOF from an Enumerator is "sticky" (idempotent).
-	Enumerator struct {
+	// other words, io.EOF from an enumerator is "sticky" (idempotent).
+	enumerator struct {
 		err error
 		hit bool
 		i   int
 		k   uint64
 		q   *d
-		t   *Tree
+		t   *tree
 		ver int64
 	}
 
-	// Tree is a B+tree.
-	Tree struct {
+	// tree is a B+tree.
+	tree struct {
 		c     int
 		cmp   Cmp
 		first *d
@@ -140,9 +140,9 @@ type (
 var ( // R/O zero values
 	zd  d
 	zde de
-	ze  Enumerator
+	ze  enumerator
 	zk  uint64
-	zt  Tree
+	zt  tree
 	zx  x
 	zxe xe
 )
@@ -233,14 +233,14 @@ func (l *d) mvR(r *d, c int) {
 
 // ----------------------------------------------------------------------- Tree
 
-// TreeNew returns a newly created, empty Tree. The compare function is used
+// treeNew returns a newly created, empty Tree. The compare function is used
 // for key collation.
-func TreeNew(cmp Cmp) *Tree {
+func treeNew(cmp Cmp) *tree {
 	return btTPool.get(cmp)
 }
 
 // Clear removes all K/V pairs from the tree.
-func (t *Tree) Clear() {
+func (t *tree) Clear() {
 	if t.r == nil {
 		return
 	}
@@ -252,13 +252,13 @@ func (t *Tree) Clear() {
 
 // Close performs Clear and recycles t to a pool for possible later reuse. No
 // references to t should exist or such references must not be used afterwards.
-func (t *Tree) Close() {
+func (t *tree) Close() {
 	t.Clear()
 	*t = zt
 	btTPool.Put(t)
 }
 
-func (t *Tree) cat(p *x, q, r *d, pi int) {
+func (t *tree) cat(p *x, q, r *d, pi int) {
 	t.ver++
 	q.mvL(r, r.c)
 	if r.n != nil {
@@ -286,7 +286,7 @@ func (t *Tree) cat(p *x, q, r *d, pi int) {
 	t.r = q
 }
 
-func (t *Tree) catX(p, q, r *x, pi int) {
+func (t *tree) catX(p, q, r *x, pi int) {
 	t.ver++
 	q.x[q.c].k = p.x[pi].k
 	copy(q.x[q.c+1:], r.x[:r.c])
@@ -320,7 +320,7 @@ func (t *Tree) catX(p, q, r *x, pi int) {
 
 // Delete removes the k's KV pair, if it exists, in which case Delete returns
 // true.
-func (t *Tree) Delete(k uint64) (ok bool) {
+func (t *tree) Delete(k uint64) (ok bool) {
 	pi := -1
 	var p *x
 	q := t.r
@@ -370,7 +370,7 @@ func (t *Tree) Delete(k uint64) (ok bool) {
 	}
 }
 
-func (t *Tree) extract(q *d, i int) { // (r *container) {
+func (t *tree) extract(q *d, i int) { // (r *container) {
 	t.ver++
 	//r = q.d[i].v // prepared for Extract
 	q.c--
@@ -381,7 +381,7 @@ func (t *Tree) extract(q *d, i int) { // (r *container) {
 	t.c--
 }
 
-func (t *Tree) find(q interface{}, k uint64) (i int, ok bool) {
+func (t *tree) find(q interface{}, k uint64) (i int, ok bool) {
 	var mk uint64
 	l := 0
 	switch x := q.(type) {
@@ -419,7 +419,7 @@ func (t *Tree) find(q interface{}, k uint64) (i int, ok bool) {
 
 // First returns the first item of the tree in the key collating order, or
 // (zero-value, zero-value) if the tree is empty.
-func (t *Tree) First() (k uint64, v *roaring.Container) {
+func (t *tree) First() (k uint64, v *roaring.Container) {
 	if q := t.first; q != nil {
 		q := &q.d[0]
 		k, v = q.k, q.v
@@ -429,7 +429,7 @@ func (t *Tree) First() (k uint64, v *roaring.Container) {
 
 // Get returns the value associated with k and true if it exists. Otherwise Get
 // returns (zero-value, false).
-func (t *Tree) Get(k uint64) (v *roaring.Container, ok bool) {
+func (t *tree) Get(k uint64) (v *roaring.Container, ok bool) {
 	q := t.r
 	if q == nil {
 		return
@@ -455,7 +455,7 @@ func (t *Tree) Get(k uint64) (v *roaring.Container, ok bool) {
 	}
 }
 
-func (t *Tree) insert(q *d, i int, k uint64, v *roaring.Container) *d {
+func (t *tree) insert(q *d, i int, k uint64, v *roaring.Container) *d {
 	t.ver++
 	c := q.c
 	if i < c {
@@ -470,7 +470,7 @@ func (t *Tree) insert(q *d, i int, k uint64, v *roaring.Container) *d {
 
 // Last returns the last item of the tree in the key collating order, or
 // (zero-value, zero-value) if the tree is empty.
-func (t *Tree) Last() (k uint64, v *roaring.Container) {
+func (t *tree) Last() (k uint64, v *roaring.Container) {
 	if q := t.last; q != nil {
 		q := &q.d[q.c-1]
 		k, v = q.k, q.v
@@ -479,11 +479,11 @@ func (t *Tree) Last() (k uint64, v *roaring.Container) {
 }
 
 // Len returns the number of items in the tree.
-func (t *Tree) Len() int {
+func (t *tree) Len() int {
 	return t.c
 }
 
-func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
+func (t *tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
 	t.ver++
 	l, r := p.siblings(pi)
 
@@ -528,7 +528,7 @@ func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
 // Seek returns an Enumerator positioned on an item such that k >= item's key.
 // ok reports if k == item.key The Enumerator's position is possibly after the
 // last item in the tree.
-func (t *Tree) Seek(k uint64) (e *Enumerator, ok bool) {
+func (t *tree) Seek(k uint64) (e *enumerator, ok bool) {
 	q := t.r
 	if q == nil {
 		e = btEPool.get(nil, false, 0, k, nil, t, t.ver)
@@ -558,7 +558,7 @@ func (t *Tree) Seek(k uint64) (e *Enumerator, ok bool) {
 
 // SeekFirst returns an enumerator positioned on the first KV pair in the tree,
 // if any. For an empty tree, err == io.EOF is returned and e will be nil.
-func (t *Tree) SeekFirst() (e *Enumerator, err error) {
+func (t *tree) SeekFirst() (e *enumerator, err error) {
 	q := t.first
 	if q == nil {
 		return nil, io.EOF
@@ -569,7 +569,7 @@ func (t *Tree) SeekFirst() (e *Enumerator, err error) {
 
 // SeekLast returns an enumerator positioned on the last KV pair in the tree,
 // if any. For an empty tree, err == io.EOF is returned and e will be nil.
-func (t *Tree) SeekLast() (e *Enumerator, err error) {
+func (t *tree) SeekLast() (e *enumerator, err error) {
 	q := t.last
 	if q == nil {
 		return nil, io.EOF
@@ -579,7 +579,7 @@ func (t *Tree) SeekLast() (e *Enumerator, err error) {
 }
 
 // Set sets the value associated with k.
-func (t *Tree) Set(k uint64, v *roaring.Container) {
+func (t *tree) Set(k uint64, v *roaring.Container) {
 	//dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump())
 	//defer func() {
 	//	dbg("--- POST\n%s\n====\n", t.dump())
@@ -645,7 +645,7 @@ func (t *Tree) Set(k uint64, v *roaring.Container) {
 // 	tree.Put(k, func(uint64, bool){ return v, true })
 //
 // modulo the differing return values.
-func (t *Tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (newV *roaring.Container, write bool)) (oldV *roaring.Container, written bool) {
+func (t *tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (newV *roaring.Container, write bool)) (oldV *roaring.Container, written bool) {
 	pi := -1
 	var p *x
 	q := t.r
@@ -712,7 +712,7 @@ func (t *Tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (new
 	}
 }
 
-func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
+func (t *tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
 	t.ver++
 	r := btDPool.Get().(*d)
 	if q.n != nil {
@@ -747,7 +747,7 @@ func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
 	t.insert(q, i, k, v)
 }
 
-func (t *Tree) splitX(p *x, q *x, pi int, i int) (*x, int) {
+func (t *tree) splitX(p *x, q *x, pi int, i int) (*x, int) {
 	t.ver++
 	r := btXPool.Get().(*x)
 	copy(r.x[:], q.x[kx+1:])
@@ -771,7 +771,7 @@ func (t *Tree) splitX(p *x, q *x, pi int, i int) (*x, int) {
 	return q, i
 }
 
-func (t *Tree) underflow(p *x, q *d, pi int) {
+func (t *tree) underflow(p *x, q *d, pi int) {
 	t.ver++
 	l, r := p.siblings(pi)
 
@@ -796,7 +796,7 @@ func (t *Tree) underflow(p *x, q *d, pi int) {
 	t.cat(p, q, r, pi)
 }
 
-func (t *Tree) underflowX(p *x, q *x, pi int, i int) (*x, int) {
+func (t *tree) underflowX(p *x, q *x, pi int, i int) (*x, int) {
 	t.ver++
 	var l, r *x
 
@@ -850,7 +850,7 @@ func (t *Tree) underflowX(p *x, q *x, pi int, i int) (*x, int) {
 
 // Close recycles e to a pool for possible later reuse. No references to e
 // should exist or such references must not be used afterwards.
-func (e *Enumerator) Close() {
+func (e *enumerator) Close() {
 	*e = ze
 	btEPool.Put(e)
 }
@@ -858,7 +858,7 @@ func (e *Enumerator) Close() {
 // Next returns the currently enumerated item, if it exists and moves to the
 // next item in the key collation order. If there is no item to return, err ==
 // io.EOF is returned.
-func (e *Enumerator) Next() (k uint64, v *roaring.Container, err error) {
+func (e *enumerator) Next() (k uint64, v *roaring.Container, err error) {
 	if err = e.err; err != nil {
 		return
 	}
@@ -886,7 +886,7 @@ func (e *Enumerator) Next() (k uint64, v *roaring.Container, err error) {
 	return
 }
 
-func (e *Enumerator) next() error {
+func (e *enumerator) next() error {
 	if e.q == nil {
 		e.err = io.EOF
 		return io.EOF
@@ -906,7 +906,7 @@ func (e *Enumerator) next() error {
 // Prev returns the currently enumerated item, if it exists and moves to the
 // previous item in the key collation order. If there is no item to return, err
 // == io.EOF is returned.
-func (e *Enumerator) Prev() (k uint64, v *roaring.Container, err error) {
+func (e *enumerator) Prev() (k uint64, v *roaring.Container, err error) {
 	if err = e.err; err != nil {
 		return
 	}
@@ -941,7 +941,7 @@ func (e *Enumerator) Prev() (k uint64, v *roaring.Container, err error) {
 	return
 }
 
-func (e *Enumerator) prev() error {
+func (e *enumerator) prev() error {
 	if e.q == nil {
 		e.err = io.EOF
 		return io.EOF

enterprise/b/containers_btree.go

@@ -27,28 +27,28 @@ func cmp(a, b uint64) int {
 	return int(a - b)
 }
 
-type BTreeContainers struct {
-	tree *Tree
+type bTreeContainers struct {
+	tree *tree
 
 	lastKey       uint64
 	lastContainer *roaring.Container
 }
 
-func NewBTreeContainers() *BTreeContainers {
-	return &BTreeContainers{
-		tree: TreeNew(cmp),
+func newBTreeContainers() *bTreeContainers {
+	return &bTreeContainers{
+		tree: treeNew(cmp),
 	}
 }
 
 func NewBTreeBitmap(a ...uint64) *roaring.Bitmap {
 	b := &roaring.Bitmap{
-		Containers: NewBTreeContainers(),
+		Containers: newBTreeContainers(),
 	}
 	b.Add(a...)
 	return b
 }
 
-func (btc *BTreeContainers) Get(key uint64) *roaring.Container {
+func (btc *bTreeContainers) Get(key uint64) *roaring.Container {
 	// Check the last* cache for same container.
 	if key == btc.lastKey && btc.lastContainer != nil {
 		return btc.lastContainer
@@ -64,7 +64,7 @@ func (btc *BTreeContainers) Get(key uint64) *roaring.Container {
 	return c
 }
 
-func (btc *BTreeContainers) Put(key uint64, c *roaring.Container) {
+func (btc *bTreeContainers) Put(key uint64, c *roaring.Container) {
 	// If a mapped container is added to the tree, reset the
 	// lastContainer cache so that the cache is not pointing
 	// at a read-only mmap.
@@ -93,16 +93,16 @@ type updater struct {
 	mapped        bool
 }
 
-func (btc *BTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
+func (btc *bTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
 	a := updater{key, containerType, n, mapped}
 	btc.tree.Put(key, a.update)
 }
 
-func (btc *BTreeContainers) Remove(key uint64) {
+func (btc *bTreeContainers) Remove(key uint64) {
 	btc.tree.Delete(key)
 }
 
-func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container {
+func (btc *bTreeContainers) GetOrCreate(key uint64) *roaring.Container {
 	// Check the last* cache for same container.
 	if key == btc.lastKey && btc.lastContainer != nil {
 		return btc.lastContainer
@@ -121,8 +121,18 @@ func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container {
 	return btc.lastContainer
 }
 
-func (btc *BTreeContainers) Clone() roaring.Containers {
-	nbtc := NewBTreeContainers()
+func (btc *bTreeContainers) Count() (n uint64) {
+	e, _ := btc.tree.Seek(0)
+	_, c, err := e.Next()
+	for err != io.EOF {
+		n += uint64(c.N())
+		_, c, err = e.Next()
+	}
+	return
+}
+
+func (btc *bTreeContainers) Clone() roaring.Containers {
+	nbtc := newBTreeContainers()
 
 	itr, err := btc.tree.SeekFirst()
 	if err == io.EOF {
@@ -138,7 +148,7 @@ func (btc *BTreeContainers) Clone() roaring.Containers {
 	return nbtc
 }
 
-func (btc *BTreeContainers) Last() (key uint64, c *roaring.Container) {
+func (btc *bTreeContainers) Last() (key uint64, c *roaring.Container) {
 	if btc.tree.Len() == 0 {
 		return 0, nil
 	}
@@ -146,34 +156,34 @@ func (btc *BTreeContainers) Last() (key uint64, c *roaring.Container) {
 	return k, v
 }
 
-func (btc *BTreeContainers) Size() int {
+func (btc *bTreeContainers) Size() int {
 	return btc.tree.Len()
 }
 
-func (btc *BTreeContainers) Reset() {
-	btc.tree = TreeNew(cmp)
+func (btc *bTreeContainers) Reset() {
+	btc.tree = treeNew(cmp)
 	btc.lastKey = 0
 	btc.lastContainer = nil
 }
 
-func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) {
+func (btc *bTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) {
 	e, ok := btc.tree.Seek(key)
 	if ok {
 		found = true
 	}
 
-	return &BTCIterator{
+	return &btcIterator{
 		e: e,
 	}, found
 }
 
-type BTCIterator struct {
-	e   *Enumerator
+type btcIterator struct {
+	e   *enumerator
 	key uint64
 	val *roaring.Container
 }
 
-func (i *BTCIterator) Next() bool {
+func (i *btcIterator) Next() bool {
 
 	k, v, err := i.e.Next()
 	if err == io.EOF {
@@ -184,7 +194,7 @@ func (i *BTCIterator) Next() bool {
 	return true
 }
 
-func (i *BTCIterator) Value() (uint64, *roaring.Container) {
+func (i *btcIterator) Value() (uint64, *roaring.Container) {
 	if i.val == nil {
 		return 0, nil
 	}

event.go

@@ -14,8 +14,7 @@
 
 package pilosa
 
-// NodeEventType are the types of events that can be sent from the
-// ChannelEventDelegate.
+// NodeEventType are the types of node events.
 type NodeEventType int
 
 const (
@@ -29,27 +28,3 @@ type NodeEvent struct {
 	Event NodeEventType
 	Node  *Node
 }
-
-// EventHandler is the interface for the pilosa object which knows how to
-// handle broadcast messages. (Hint: this is implemented by pilosa.Server)
-type EventHandler interface {
-	ReceiveEvent(e *NodeEvent) error
-}
-
-// EventReceiver is the interface for the object which will listen for and
-// decode broadcast messages before passing them to pilosa to handle. The
-// implementation of this could be an http server which listens for messages,
-// gets the protobuf payload, and then passes it to
-// EventHandler.ReceiveMessage.
-type EventReceiver interface {
-	// Start starts listening for broadcast messages - it should return
-	// immediately, spawning a goroutine if necessary.
-	Start(EventHandler) error
-}
-
-type nopEventReceiver struct{}
-
-func (n *nopEventReceiver) Start(e EventHandler) error { return nil }
-
-// NopEventReceiver is a no-op implementation of the EventReceiver.
-var NopEventReceiver = &nopEventReceiver{}

field_internal_test.go

@@ -0,0 +1,343 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/pilosa/pilosa/pql"
+)
+
+// Ensure a bsiGroup can adjust to its baseValue.
+func TestBSIGroup_BaseValue(t *testing.T) {
+	b0 := &bsiGroup{
+		Name: "b0",
+		Type: bsiGroupTypeInt,
+		Min:  -100,
+		Max:  900,
+	}
+	b1 := &bsiGroup{
+		Name: "b1",
+		Type: bsiGroupTypeInt,
+		Min:  0,
+		Max:  1000,
+	}
+
+	b2 := &bsiGroup{
+		Name: "b2",
+		Type: bsiGroupTypeInt,
+		Min:  100,
+		Max:  1100,
+	}
+
+	t.Run("Normal Condition", func(t *testing.T) {
+
+		for _, tt := range []struct {
+			f             *bsiGroup
+			op            pql.Token
+			val           int64
+			expBaseValue  uint64
+			expOutOfRange bool
+		}{
+			// LT
+			{b0, pql.LT, 5, 105, false},
+			{b0, pql.LT, -8, 92, false},
+			{b0, pql.LT, -108, 0, true},
+			{b0, pql.LT, 1005, 1000, false},
+			{b0, pql.LT, 0, 100, false},
+
+			{b1, pql.LT, 5, 5, false},
+			{b1, pql.LT, -8, 0, true},
+			{b1, pql.LT, 1005, 1000, false},
+			{b1, pql.LT, 0, 0, false},
+
+			{b2, pql.LT, 5, 0, true},
+			{b2, pql.LT, -8, 0, true},
+			{b2, pql.LT, 105, 5, false},
+			{b2, pql.LT, 1105, 1000, false},
+
+			// GT
+			{b0, pql.GT, -105, 0, false},
+			{b0, pql.GT, 5, 105, false},
+			{b0, pql.GT, 905, 0, true},
+			{b0, pql.GT, 0, 100, false},
+
+			{b1, pql.GT, 5, 5, false},
+			{b1, pql.GT, -8, 0, false},
+			{b1, pql.GT, 1005, 0, true},
+			{b1, pql.GT, 0, 0, false},
+
+			{b2, pql.GT, 5, 0, false},
+			{b2, pql.GT, -8, 0, false},
+			{b2, pql.GT, 105, 5, false},
+			{b2, pql.GT, 1105, 0, true},
+
+			// EQ
+			{b0, pql.EQ, -105, 0, true},
+			{b0, pql.EQ, 5, 105, false},
+			{b0, pql.EQ, 905, 0, true},
+			{b0, pql.EQ, 0, 100, false},
+
+			{b1, pql.EQ, 5, 5, false},
+			{b1, pql.EQ, -8, 0, true},
+			{b1, pql.EQ, 1005, 0, true},
+			{b1, pql.EQ, 0, 0, false},
+
+			{b2, pql.EQ, 5, 0, true},
+			{b2, pql.EQ, -8, 0, true},
+			{b2, pql.EQ, 105, 5, false},
+			{b2, pql.EQ, 1105, 0, true},
+		} {
+			bv, oor := tt.f.baseValue(tt.op, tt.val)
+			if oor != tt.expOutOfRange {
+				t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor)
+			} else if !reflect.DeepEqual(bv, tt.expBaseValue) {
+				t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv)
+			}
+		}
+	})
+
+	t.Run("Betwween Condition", func(t *testing.T) {
+		for _, tt := range []struct {
+			f               *bsiGroup
+			predMin         int64
+			predMax         int64
+			expBaseValueMin uint64
+			expBaseValueMax uint64
+			expOutOfRange   bool
+		}{
+
+			{b0, -205, -105, 0, 0, true},
+			{b0, -105, 80, 0, 180, false},
+			{b0, 5, 20, 105, 120, false},
+			{b0, 20, 1005, 120, 1000, false},
+			{b0, 1005, 2000, 0, 0, true},
+
+			{b1, -105, -5, 0, 0, true},
+			{b1, -5, 20, 0, 20, false},
+			{b1, 5, 20, 5, 20, false},
+			{b1, 20, 1005, 20, 1000, false},
+			{b1, 1005, 2000, 0, 0, true},
+
+			{b2, 5, 95, 0, 0, true},
+			{b2, 95, 120, 0, 20, false},
+			{b2, 105, 120, 5, 20, false},
+			{b2, 120, 1105, 20, 1000, false},
+			{b2, 1105, 2000, 0, 0, true},
+		} {
+			min, max, oor := tt.f.baseValueBetween(tt.predMin, tt.predMax)
+			if oor != tt.expOutOfRange {
+				t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor)
+			} else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) {
+				t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max)
+			}
+		}
+	})
+}
+
+// Ensure field can open and retrieve a view.
+func TestField_DeleteView(t *testing.T) {
+	f := MustOpenField(OptFieldTypeDefault())
+	defer f.Close()
+
+	viewName := viewStandard + "_v"
+
+	// Create view.
+	view, err := f.createViewIfNotExists(viewName)
+	if err != nil {
+		t.Fatal(err)
+	} else if view == nil {
+		t.Fatal("expected view")
+	}
+
+	err = f.deleteView(viewName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if f.view(viewName) != nil {
+		t.Fatal("view still exists in field")
+	}
+
+	// Recreate view with same name, verify that the old view was not reused.
+	view2, err := f.createViewIfNotExists(viewName)
+	if err != nil {
+		t.Fatal(err)
+	} else if view == view2 {
+		t.Fatal("failed to create new view")
+	}
+}
+
+// TestField represents a test wrapper for Field.
+type TestField struct {
+	*Field
+}
+
+// NewTestField returns a new instance of TestField d/0.
+func NewTestField(opts FieldOption) *TestField {
+	path, err := ioutil.TempDir("", "pilosa-field-")
+	if err != nil {
+		panic(err)
+	}
+	field, err := NewField(path, "i", "f", opts)
+	if err != nil {
+		panic(err)
+	}
+	return &TestField{Field: field}
+}
+
+// MustOpenField returns a new, opened field at a temporary path. Panic on error.
+func MustOpenField(opts FieldOption) *TestField {
+	f := NewTestField(opts)
+	if err := f.Open(); err != nil {
+		panic(err)
+	}
+	return f
+}
+
+// Close closes the field and removes the underlying data.
+func (f *TestField) Close() error {
+	defer os.RemoveAll(f.Path())
+	return f.Field.Close()
+}
+
+// Reopen closes the index and reopens it.
+func (f *TestField) Reopen() error {
+	var err error
+	if err := f.Field.Close(); err != nil {
+		return err
+	}
+
+	path, index, name := f.Path(), f.Index(), f.Name()
+	f.Field, err = NewField(path, index, name, OptFieldTypeDefault())
+	if err != nil {
+		return err
+	}
+
+	if err := f.Open(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (f *TestField) MustSetBit(row, col uint64, ts ...time.Time) {
+	if len(ts) == 0 {
+		_, err := f.Field.SetBit(row, col, nil)
+		if err != nil {
+			panic(err)
+		}
+	}
+	for _, t := range ts {
+		_, err := f.Field.SetBit(row, col, &t)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+// Ensure field can open and retrieve a view.
+func TestField_CreateViewIfNotExists(t *testing.T) {
+	f := MustOpenField(OptFieldTypeDefault())
+	defer f.Close()
+
+	// Create view.
+	view, err := f.createViewIfNotExists("v")
+	if err != nil {
+		t.Fatal(err)
+	} else if view == nil {
+		t.Fatal("expected view")
+	}
+
+	// Retrieve existing view.
+	view2, err := f.createViewIfNotExists("v")
+	if err != nil {
+		t.Fatal(err)
+	} else if view != view2 {
+		t.Fatal("view mismatch")
+	}
+
+	if view != f.view("v") {
+		t.Fatal("view mismatch")
+	}
+}
+
+func TestField_SetTimeQuantum(t *testing.T) {
+	f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
+	defer f.Close()
+
+	// Set & retrieve time quantum.
+	if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
+		t.Fatal(err)
+	} else if q := f.TimeQuantum(); q != TimeQuantum("YMDH") {
+		t.Fatalf("unexpected quantum: %s", q)
+	}
+
+	// Reload field and verify that it is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if q := f.TimeQuantum(); q != TimeQuantum("YMDH") {
+		t.Fatalf("unexpected quantum (reopen): %s", q)
+	}
+}
+
+func TestField_RowTime(t *testing.T) {
+	f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
+	defer f.Close()
+
+	if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
+		t.Fatal(err)
+	}
+
+	f.MustSetBit(1, 1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC))
+	f.MustSetBit(1, 2, time.Date(2011, time.January, 5, 12, 0, 0, 0, time.UTC))
+	f.MustSetBit(1, 3, time.Date(2010, time.February, 5, 12, 0, 0, 0, time.UTC))
+	f.MustSetBit(1, 4, time.Date(2010, time.January, 6, 12, 0, 0, 0, time.UTC))
+	f.MustSetBit(1, 5, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC))
+
+	if r, err := f.RowTime(1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(r.Columns(), []uint64{1, 3, 4, 5}) {
+		t.Fatalf("wrong columns: %#v", r.Columns())
+	}
+
+	if r, err := f.RowTime(1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "YM"); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(r.Columns(), []uint64{3}) {
+		t.Fatalf("wrong columns: %#v", r.Columns())
+	}
+
+	if r, err := f.RowTime(1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "M"); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(r.Columns(), []uint64{3}) {
+		t.Fatalf("wrong columns: %#v", r.Columns())
+	}
+
+	if r, err := f.RowTime(1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC), "MD"); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(r.Columns(), []uint64{1, 5}) {
+		t.Fatalf("wrong columns: %#v", r.Columns())
+	}
+
+	if r, err := f.RowTime(1, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC), "MDH"); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(r.Columns(), []uint64{5}) {
+		t.Fatalf("wrong columns: %#v", r.Columns())
+	}
+
+}

field_test.go

@@ -0,0 +1,187 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/test"
+)
+
+// Ensure a field can set & read a bsiGroup value.
+func TestField_SetValue(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value on field.
+		if changed, err := f.SetValue(100, 21); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.Value(100); err != nil {
+			t.Fatal(err)
+		} else if value != 21 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected value to exist")
+		}
+
+		// Setting value should return no change.
+		if changed, err := f.SetValue(100, 21); err != nil {
+			t.Fatal(err)
+		} else if changed {
+			t.Fatal("expected no change")
+		}
+	})
+
+	t.Run("Overwrite", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value.
+		if changed, err := f.SetValue(100, 21); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Set different value.
+		if changed, err := f.SetValue(100, 23); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.Value(100); err != nil {
+			t.Fatal(err)
+		} else if value != 23 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected value to exist")
+		}
+	})
+
+	t.Run("ErrBSIGroupNotFound", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value.
+		if _, err := f.SetValue(100, 21); err != pilosa.ErrBSIGroupNotFound {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrBSIGroupValueTooLow", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value.
+		if _, err := f.SetValue(100, 15); err != pilosa.ErrBSIGroupValueTooLow {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrBSIGroupValueTooHigh", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value.
+		if _, err := f.SetValue(100, 31); err != pilosa.ErrBSIGroupValueTooHigh {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+}
+
+func TestField_NameRestriction(t *testing.T) {
+	path, err := ioutil.TempDir("", "pilosa-field-")
+	if err != nil {
+		panic(err)
+	}
+	field, err := pilosa.NewField(path, "i", ".meta", pilosa.OptFieldTypeDefault())
+	if field != nil {
+		t.Fatalf("unexpected field name %s", err)
+	}
+}
+
+// Ensure that field name validation is consistent.
+func TestField_NameValidation(t *testing.T) {
+	validFieldNames := []string{
+		"foo",
+		"hyphen-ated",
+		"under_score",
+		"abc123",
+		"trailing_",
+	}
+	invalidFieldNames := []string{
+		"",
+		"123abc",
+		"x.y",
+		"_foo",
+		"-bar",
+		"abc def",
+		"camelCase",
+		"UPPERCASE",
+		"a12345678901234567890123456789012345678901234567890123456789012345",
+	}
+
+	path, err := ioutil.TempDir("", "pilosa-field-")
+	if err != nil {
+		panic(err)
+	}
+	for _, name := range validFieldNames {
+		_, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
+		if err != nil {
+			t.Fatalf("unexpected field name: %s %s", name, err)
+		}
+	}
+	for _, name := range invalidFieldNames {
+		_, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
+		if err == nil {
+			t.Fatalf("expected error on field name: %s", name)
+		}
+	}
+}

frame_test.go

@@ -1,423 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa_test
-
-import (
-	"io/ioutil"
-	"reflect"
-	"testing"
-
-	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/pql"
-	"github.com/pilosa/pilosa/test"
-)
-
-// Ensure frame can open and retrieve a view.
-func TestFrame_CreateViewIfNotExists(t *testing.T) {
-	f := test.MustOpenFrame()
-	defer f.Close()
-
-	// Create view.
-	view, err := f.CreateViewIfNotExists("v")
-	if err != nil {
-		t.Fatal(err)
-	} else if view == nil {
-		t.Fatal("expected view")
-	}
-
-	// Retrieve existing view.
-	view2, err := f.CreateViewIfNotExists("v")
-	if err != nil {
-		t.Fatal(err)
-	} else if view != view2 {
-		t.Fatal("view mismatch")
-	}
-
-	if view != f.View("v") {
-		t.Fatal("view mismatch")
-	}
-}
-
-// Ensure frame can set its time quantum.
-func TestFrame_SetTimeQuantum(t *testing.T) {
-	f := test.MustOpenFrame()
-	defer f.Close()
-
-	// Set & retrieve time quantum.
-	if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
-		t.Fatal(err)
-	} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
-		t.Fatalf("unexpected quantum: %s", q)
-	}
-
-	// Reload frame and verify that it is persisted.
-	if err := f.Reopen(); err != nil {
-		t.Fatal(err)
-	} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
-		t.Fatalf("unexpected quantum (reopen): %s", q)
-	}
-}
-
-// Ensure a frame can set & read a field value.
-func TestFrame_SetFieldValue(t *testing.T) {
-	t.Run("OK", func(t *testing.T) {
-		idx := test.MustOpenIndex()
-		defer idx.Close()
-
-		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
-			Fields: []*pilosa.Field{
-				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
-				{Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25},
-			},
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// Set value on first field.
-		if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
-			t.Fatal(err)
-		} else if !changed {
-			t.Fatal("expected change")
-		}
-
-		// Set value on same column but different field.
-		if changed, err := f.SetFieldValue(100, "field1", 25); err != nil {
-			t.Fatal(err)
-		} else if !changed {
-			t.Fatal("expected change")
-		}
-
-		// Read value.
-		if value, exists, err := f.FieldValue(100, "field0"); err != nil {
-			t.Fatal(err)
-		} else if value != 21 {
-			t.Fatalf("unexpected value: %d", value)
-		} else if !exists {
-			t.Fatal("expected value to exist")
-		}
-
-		// Setting value should return no change.
-		if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
-			t.Fatal(err)
-		} else if changed {
-			t.Fatal("expected no change")
-		}
-	})
-
-	t.Run("Overwrite", func(t *testing.T) {
-		idx := test.MustOpenIndex()
-		defer idx.Close()
-
-		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
-			Fields: []*pilosa.Field{
-				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
-			},
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// Set value.
-		if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
-			t.Fatal(err)
-		} else if !changed {
-			t.Fatal("expected change")
-		}
-
-		// Set different value.
-		if changed, err := f.SetFieldValue(100, "field0", 23); err != nil {
-			t.Fatal(err)
-		} else if !changed {
-			t.Fatal("expected change")
-		}
-
-		// Read value.
-		if value, exists, err := f.FieldValue(100, "field0"); err != nil {
-			t.Fatal(err)
-		} else if value != 23 {
-			t.Fatalf("unexpected value: %d", value)
-		} else if !exists {
-			t.Fatal("expected value to exist")
-		}
-	})
-
-	t.Run("ErrFieldNotFound", func(t *testing.T) {
-		idx := test.MustOpenIndex()
-		defer idx.Close()
-
-		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
-			Fields: []*pilosa.Field{
-				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
-			},
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// Set value.
-		if _, err := f.SetFieldValue(100, "no_such_field", 21); err != pilosa.ErrFieldNotFound {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	})
-
-	t.Run("ErrFieldValueTooLow", func(t *testing.T) {
-		idx := test.MustOpenIndex()
-		defer idx.Close()
-
-		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
-			Fields: []*pilosa.Field{
-				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
-			},
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// Set value.
-		if _, err := f.SetFieldValue(100, "field0", 15); err != pilosa.ErrFieldValueTooLow {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	})
-
-	t.Run("ErrFieldValueTooHigh", func(t *testing.T) {
-		idx := test.MustOpenIndex()
-		defer idx.Close()
-
-		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
-			Fields: []*pilosa.Field{
-				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
-			},
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// Set value.
-		if _, err := f.SetFieldValue(100, "field0", 31); err != pilosa.ErrFieldValueTooHigh {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	})
-}
-
-func TestFrame_NameRestriction(t *testing.T) {
-	path, err := ioutil.TempDir("", "pilosa-frame-")
-	if err != nil {
-		panic(err)
-	}
-	frame, err := pilosa.NewFrame(path, "i", ".meta")
-	if frame != nil {
-		t.Fatalf("unexpected frame name %s", err)
-	}
-}
-
-// Ensure that frame name validation is consistent.
-func TestFrame_NameValidation(t *testing.T) {
-	validFrameNames := []string{
-		"foo",
-		"hyphen-ated",
-		"under_score",
-		"abc123",
-		"trailing_",
-	}
-	invalidFrameNames := []string{
-		"",
-		"123abc",
-		"x.y",
-		"_foo",
-		"-bar",
-		"abc def",
-		"camelCase",
-		"UPPERCASE",
-		"a12345678901234567890123456789012345678901234567890123456789012345",
-	}
-
-	path, err := ioutil.TempDir("", "pilosa-frame-")
-	if err != nil {
-		panic(err)
-	}
-	for _, name := range validFrameNames {
-		_, err := pilosa.NewFrame(path, "i", name)
-		if err != nil {
-			t.Fatalf("unexpected frame name: %s %s", name, err)
-		}
-	}
-	for _, name := range invalidFrameNames {
-		_, err := pilosa.NewFrame(path, "i", name)
-		if err == nil {
-			t.Fatalf("expected error on frame name: %s", name)
-		}
-	}
-}
-
-// Ensure frame can open and retrieve a view.
-func TestFrame_DeleteView(t *testing.T) {
-	f := test.MustOpenFrame()
-	defer f.Close()
-
-	viewName := pilosa.ViewStandard + "_v"
-
-	// Create view.
-	view, err := f.CreateViewIfNotExists(viewName)
-	if err != nil {
-		t.Fatal(err)
-	} else if view == nil {
-		t.Fatal("expected view")
-	}
-
-	err = f.DeleteView(viewName)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if f.View(viewName) != nil {
-		t.Fatal("view still exists in frame")
-	}
-
-	// Recreate view with same name, verify that the old view was not reused.
-	view2, err := f.CreateViewIfNotExists(viewName)
-	if err != nil {
-		t.Fatal(err)
-	} else if view == view2 {
-		t.Fatal("failed to create new view")
-	}
-}
-
-// Ensure a field can adjust to its baseValue.
-func TestField_BaseValue(t *testing.T) {
-	f0 := &pilosa.Field{
-		Name: "f0",
-		Type: pilosa.FieldTypeInt,
-		Min:  -100,
-		Max:  900,
-	}
-	f1 := &pilosa.Field{
-		Name: "f1",
-		Type: pilosa.FieldTypeInt,
-		Min:  0,
-		Max:  1000,
-	}
-
-	f2 := &pilosa.Field{
-		Name: "f2",
-		Type: pilosa.FieldTypeInt,
-		Min:  100,
-		Max:  1100,
-	}
-
-	t.Run("Normal Condition", func(t *testing.T) {
-
-		for _, tt := range []struct {
-			f             *pilosa.Field
-			op            pql.Token
-			val           int64
-			expBaseValue  uint64
-			expOutOfRange bool
-		}{
-			// LT
-			{f0, pql.LT, 5, 105, false},
-			{f0, pql.LT, -8, 92, false},
-			{f0, pql.LT, -108, 0, true},
-			{f0, pql.LT, 1005, 1000, false},
-			{f0, pql.LT, 0, 100, false},
-
-			{f1, pql.LT, 5, 5, false},
-			{f1, pql.LT, -8, 0, true},
-			{f1, pql.LT, 1005, 1000, false},
-			{f1, pql.LT, 0, 0, false},
-
-			{f2, pql.LT, 5, 0, true},
-			{f2, pql.LT, -8, 0, true},
-			{f2, pql.LT, 105, 5, false},
-			{f2, pql.LT, 1105, 1000, false},
-
-			// GT
-			{f0, pql.GT, -105, 0, false},
-			{f0, pql.GT, 5, 105, false},
-			{f0, pql.GT, 905, 0, true},
-			{f0, pql.GT, 0, 100, false},
-
-			{f1, pql.GT, 5, 5, false},
-			{f1, pql.GT, -8, 0, false},
-			{f1, pql.GT, 1005, 0, true},
-			{f1, pql.GT, 0, 0, false},
-
-			{f2, pql.GT, 5, 0, false},
-			{f2, pql.GT, -8, 0, false},
-			{f2, pql.GT, 105, 5, false},
-			{f2, pql.GT, 1105, 0, true},
-
-			// EQ
-			{f0, pql.EQ, -105, 0, true},
-			{f0, pql.EQ, 5, 105, false},
-			{f0, pql.EQ, 905, 0, true},
-			{f0, pql.EQ, 0, 100, false},
-
-			{f1, pql.EQ, 5, 5, false},
-			{f1, pql.EQ, -8, 0, true},
-			{f1, pql.EQ, 1005, 0, true},
-			{f1, pql.EQ, 0, 0, false},
-
-			{f2, pql.EQ, 5, 0, true},
-			{f2, pql.EQ, -8, 0, true},
-			{f2, pql.EQ, 105, 5, false},
-			{f2, pql.EQ, 1105, 0, true},
-		} {
-			bv, oor := tt.f.BaseValue(tt.op, tt.val)
-			if oor != tt.expOutOfRange {
-				t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor)
-			} else if !reflect.DeepEqual(bv, tt.expBaseValue) {
-				t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv)
-			}
-		}
-	})
-
-	t.Run("Betwween Condition", func(t *testing.T) {
-		for _, tt := range []struct {
-			f               *pilosa.Field
-			predMin         int64
-			predMax         int64
-			expBaseValueMin uint64
-			expBaseValueMax uint64
-			expOutOfRange   bool
-		}{
-
-			{f0, -205, -105, 0, 0, true},
-			{f0, -105, 80, 0, 180, false},
-			{f0, 5, 20, 105, 120, false},
-			{f0, 20, 1005, 120, 1000, false},
-			{f0, 1005, 2000, 0, 0, true},
-
-			{f1, -105, -5, 0, 0, true},
-			{f1, -5, 20, 0, 20, false},
-			{f1, 5, 20, 5, 20, false},
-			{f1, 20, 1005, 20, 1000, false},
-			{f1, 1005, 2000, 0, 0, true},
-
-			{f2, 5, 95, 0, 0, true},
-			{f2, 95, 120, 0, 20, false},
-			{f2, 105, 120, 5, 20, false},
-			{f2, 120, 1105, 20, 1000, false},
-			{f2, 1105, 2000, 0, 0, true},
-		} {
-			min, max, oor := tt.f.BaseValueBetween(tt.predMin, tt.predMax)
-			if oor != tt.expOutOfRange {
-				t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor)
-			} else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) {
-				t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max)
-			}
-		}
-	})
-}

gcnotify/gcnotify.go

@@ -20,25 +20,25 @@ import (
 )
 
 // Ensure ActiveGCNotifier implements interface.
-var _ pilosa.GCNotifier = &ActiveGCNotifier{}
+var _ pilosa.GCNotifier = &activeGCNotifier{}
 
-type ActiveGCNotifier struct {
+type activeGCNotifier struct {
 	gcn *gcnotifier.GCNotifier
 }
 
 // NewActiveGCNotifier creates an active GCNotifier.
-func NewActiveGCNotifier() *ActiveGCNotifier {
-	return &ActiveGCNotifier{
+func NewActiveGCNotifier() *activeGCNotifier {
+	return &activeGCNotifier{
 		gcn: gcnotifier.New(),
 	}
 }
 
 // Close implements the GCNotifier interface.
-func (n *ActiveGCNotifier) Close() {
+func (n *activeGCNotifier) Close() {
 	n.gcn.Close()
 }
 
 // AfterGC implements the GCNotifier interface.
-func (n *ActiveGCNotifier) AfterGC() <-chan struct{} {
+func (n *activeGCNotifier) AfterGC() <-chan struct{} {
 	return n.gcn.AfterGC()
 }

gopsutil/systeminfo.go

@@ -22,15 +22,15 @@ import (
 
 var _ pilosa.SystemInfo = NewSystemInfo()
 
-// SystemInfo is an implementation of pilosa.SystemInfo that uses gopsutil to collect information about the host OS.
-type SystemInfo struct {
+// systemInfo is an implementation of pilosa.systemInfo that uses gopsutil to collect information about the host OS.
+type systemInfo struct {
 	platform  string
 	family    string
 	osVersion string
 }
 
 // Uptime returns the system uptime in seconds.
-func (s *SystemInfo) Uptime() (uptime uint64, err error) {
+func (s *systemInfo) Uptime() (uptime uint64, err error) {
 	hostInfo, err := host.Info()
 	if err != nil {
 		return 0, err
@@ -39,7 +39,7 @@ func (s *SystemInfo) Uptime() (uptime uint64, err error) {
 }
 
 // collectPlatformInfo fetches and caches system platform information.
-func (s *SystemInfo) collectPlatformInfo() error {
+func (s *systemInfo) collectPlatformInfo() error {
 	var err error
 	if s.platform == "" {
 		s.platform, s.family, s.osVersion, err = host.PlatformInformation()
@@ -51,7 +51,7 @@ func (s *SystemInfo) collectPlatformInfo() error {
 }
 
 // Platform returns the system platform.
-func (s *SystemInfo) Platform() (string, error) {
+func (s *systemInfo) Platform() (string, error) {
 	err := s.collectPlatformInfo()
 	if err != nil {
 		return "", err
@@ -60,7 +60,7 @@ func (s *SystemInfo) Platform() (string, error) {
 }
 
 // Family returns the system family.
-func (s *SystemInfo) Family() (string, error) {
+func (s *systemInfo) Family() (string, error) {
 	err := s.collectPlatformInfo()
 	if err != nil {
 		return "", err
@@ -69,7 +69,7 @@ func (s *SystemInfo) Family() (string, error) {
 }
 
 // OSVersion returns the OS Version.
-func (s *SystemInfo) OSVersion() (string, error) {
+func (s *systemInfo) OSVersion() (string, error) {
 	err := s.collectPlatformInfo()
 	if err != nil {
 		return "", err
@@ -78,7 +78,7 @@ func (s *SystemInfo) OSVersion() (string, error) {
 }
 
 // MemFree returns the amount of free memory in bytes.
-func (s *SystemInfo) MemFree() (uint64, error) {
+func (s *systemInfo) MemFree() (uint64, error) {
 	memInfo, err := mem.VirtualMemory()
 	if err != nil {
 		return 0, err
@@ -87,7 +87,7 @@ func (s *SystemInfo) MemFree() (uint64, error) {
 }
 
 // MemTotal returns the amount of total memory in bytes.
-func (s *SystemInfo) MemTotal() (uint64, error) {
+func (s *systemInfo) MemTotal() (uint64, error) {
 	memInfo, err := mem.VirtualMemory()
 	if err != nil {
 		return 0, err
@@ -96,7 +96,7 @@ func (s *SystemInfo) MemTotal() (uint64, error) {
 }
 
 // MemUsed returns the amount of used memory in bytes.
-func (s *SystemInfo) MemUsed() (uint64, error) {
+func (s *systemInfo) MemUsed() (uint64, error) {
 	memInfo, err := mem.VirtualMemory()
 	if err != nil {
 		return 0, err
@@ -105,11 +105,11 @@ func (s *SystemInfo) MemUsed() (uint64, error) {
 }
 
 // KernelVersion returns the kernel version as a string.
-func (s *SystemInfo) KernelVersion() (string, error) {
+func (s *systemInfo) KernelVersion() (string, error) {
 	return host.KernelVersion()
 }
 
 // NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo.
-func NewSystemInfo() *SystemInfo {
-	return &SystemInfo{}
+func NewSystemInfo() *systemInfo {
+	return &systemInfo{}
 }

gossip/gossip.go

@@ -15,69 +15,46 @@
 package gossip
 
 import (
+	"bytes"
+	"context"
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"golang.org/x/sync/errgroup"
-
-	"github.com/gogo/protobuf/proto"
 	"github.com/hashicorp/memberlist"
 	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pilosa/pilosa/toml"
 	"github.com/pkg/errors"
 )
 
 // Ensure GossipMemberSet implements interfaces.
-var _ pilosa.BroadcastReceiver = &GossipMemberSet{}
-var _ pilosa.Gossiper = &GossipMemberSet{}
-var _ memberlist.Delegate = &GossipMemberSet{}
+var _ memberlist.Delegate = &gossipMemberSet{}
 
-// GossipMemberSet represents a gossip implementation of MemberSet using memberlist.
-type GossipMemberSet struct {
+// gossipMemberSet represents a gossip implementation of MemberSet using memberlist.
+type gossipMemberSet struct {
 	mu         sync.RWMutex
-	node       *pilosa.Node
 	memberlist *memberlist.Memberlist
-	handler    pilosa.BroadcastHandler
 
 	broadcasts *memberlist.TransmitLimitedQueue
 
-	statusHandler pilosa.StatusHandler
-	config        *gossipConfig
+	papi   *pilosa.API
+	config *gossipConfig
 
 	Logger pilosa.Logger
 
 	logger    *log.Logger
 	transport *Transport
-}
 
-// Start implements the BroadcastReceiver interface and sets the BroadcastHandler.
-func (g *GossipMemberSet) Start(h pilosa.BroadcastHandler) error {
-	g.handler = h
-	return nil
-}
-
-// GetBindAddr returns the gossip bind address based on config and auto bind port.
-// This method is currently only used in a test scenario where a second node needs
-// the auto-bind address of the first node to use as its gossip seed.
-func (g *GossipMemberSet) GetBindAddr() string {
-	return fmt.Sprintf("%s:%d", g.config.memberlistConfig.BindAddr, g.config.memberlistConfig.BindPort)
+	gossipEventReceiver *gossipEventReceiver
 }
 
 // Open implements the MemberSet interface to start network activity.
-func (g *GossipMemberSet) Open(n *pilosa.Node) error {
-	if g.handler == nil {
-		return fmt.Errorf("must call Start(pilosa.BroadcastHandler) before calling Open()")
-	}
-
-	g.node = n
-
-	err := error(nil)
+func (g *gossipMemberSet) Open() (err error) {
 	g.mu.Lock()
 	g.memberlist, err = memberlist.Create(g.config.memberlistConfig)
 	g.mu.Unlock()
@@ -117,7 +94,7 @@ func (g *GossipMemberSet) Open(n *pilosa.Node) error {
 }
 
 // joinWithRetry wraps the standard memberlist Join function in a retry.
-func (g *GossipMemberSet) joinWithRetry(hosts []string) error {
+func (g *gossipMemberSet) joinWithRetry(hosts []string) error {
 	err := retry(60, 2*time.Second, func() error {
 		_, err := g.memberlist.Join(hosts)
 		return err
@@ -148,28 +125,30 @@ type gossipConfig struct {
 	memberlistConfig *memberlist.Config
 }
 
-// GossipMemberSetOption describes a functional option for GossipMemberSet.
-type GossipMemberSetOption func(*GossipMemberSet) error
+// gossipMemberSetOption describes a functional option for GossipMemberSet.
+type gossipMemberSetOption func(*gossipMemberSet) error
 
 // WithTransport is a functional option for providing a transport to NewGossipMemberSet.
-func WithTransport(transport *Transport) GossipMemberSetOption {
-	return func(g *GossipMemberSet) error {
+func WithTransport(transport *Transport) gossipMemberSetOption {
+	return func(g *gossipMemberSet) error {
 		g.transport = transport
 		return nil
 	}
 }
 
 // WithLogger is a functional option for providing a logger to NewGossipMemberSet.
-func WithLogger(logger *log.Logger) GossipMemberSetOption {
-	return func(g *GossipMemberSet) error {
+func WithLogger(logger *log.Logger) gossipMemberSetOption {
+	return func(g *gossipMemberSet) error {
 		g.logger = logger
 		return nil
 	}
 }
 
 // NewGossipMemberSet returns a new instance of GossipMemberSet based on options.
-func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventReceiver, sh pilosa.StatusHandler, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
-	g := &GossipMemberSet{
+func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...gossipMemberSetOption) (*gossipMemberSet, error) {
+	host := api.Node().URI.Host
+	g := &gossipMemberSet{
+		papi:   api,
 		Logger: pilosa.NopLogger,
 	}
 
@@ -179,6 +158,8 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe
 			return nil, errors.Wrap(err, "executing option")
 		}
 	}
+	ger := newGossipEventReceiver(g.logger, api)
+	g.gossipEventReceiver = ger
 
 	if g.transport == nil {
 		port, err := strconv.Atoi(cfg.Port)
@@ -195,7 +176,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe
 		g.transport = transport
 	}
 
-	port := g.transport.Net.GetAutoBindPort()
+	port := g.transport.net.GetAutoBindPort()
 
 	var gossipKey []byte
 	var err error
@@ -208,12 +189,12 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe
 
 	// memberlist config
 	conf := memberlist.DefaultWANConfig()
-	conf.Transport = g.transport.Net
-	conf.Name = name
-	conf.BindAddr = host
+	conf.Transport = g.transport.net
+	conf.Name = api.Node().ID
+	conf.BindAddr = api.Node().URI.Host
 	conf.BindPort = port
 	conf.AdvertisePort = port
-	conf.AdvertiseAddr = pilosa.HostToIP(host)
+	conf.AdvertiseAddr = hostToIP(api.Node().URI.Host)
 	//
 	conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
 	conf.SuspicionMult = cfg.SuspicionMult
@@ -234,57 +215,12 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe
 		gossipSeeds:      cfg.Seeds,
 	}
 
-	g.statusHandler = sh
-
 	return g, nil
 }
 
-// SendSync implementation of the Broadcaster interface.
-func (g *GossipMemberSet) SendSync(pb proto.Message) error {
-	msg, err := pilosa.MarshalMessage(pb)
-	if err != nil {
-		return fmt.Errorf("marshal message: %s", err)
-	}
-
-	mlist := g.memberlist
-
-	// Direct sends the message directly to every node.
-	// An error from any node raises an error on the entire operation.
-	//
-	// Gossip uses the gossip protocol to eventually deliver the message
-	// to every node.
-	var eg errgroup.Group
-	for _, n := range mlist.Members() {
-		// Don't send the message to the local node.
-		if n == mlist.LocalNode() {
-			continue
-		}
-		node := n
-		eg.Go(func() error {
-			return mlist.SendToTCP(node, msg)
-		})
-	}
-	return eg.Wait()
-}
-
-// SendAsync implementation of the Gossiper interface.
-func (g *GossipMemberSet) SendAsync(pb proto.Message) error {
-	msg, err := pilosa.MarshalMessage(pb)
-	if err != nil {
-		return fmt.Errorf("marshal message: %s", err)
-	}
-
-	b := &broadcast{
-		msg:    msg,
-		notify: nil,
-	}
-	g.broadcasts.QueueBroadcast(b)
-	return nil
-}
-
 // NodeMeta implementation of the memberlist.Delegate interface.
-func (g *GossipMemberSet) NodeMeta(limit int) []byte {
-	buf, err := proto.Marshal(pilosa.EncodeNode(g.node))
+func (g *gossipMemberSet) NodeMeta(limit int) []byte {
+	buf, err := g.papi.Serializer.Marshal(g.papi.Node())
 	if err != nil {
 		g.Logger.Printf("marshal message error: %s", err)
 		return []byte{}
@@ -294,35 +230,30 @@ func (g *GossipMemberSet) NodeMeta(limit int) []byte {
 
 // NotifyMsg implementation of the memberlist.Delegate interface
 // called when a user-data message is received.
-func (g *GossipMemberSet) NotifyMsg(b []byte) {
-	m, err := pilosa.UnmarshalMessage(b)
+func (g *gossipMemberSet) NotifyMsg(b []byte) {
+	err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(b))
 	if err != nil {
-		g.Logger.Printf("unmarshal message error: %s", err)
-		return
-	}
-	if err := g.handler.ReceiveMessage(m); err != nil {
-		g.Logger.Printf("receive message error: %s", err)
-		return
+		g.Logger.Printf("cluster message error: %s", err)
 	}
 }
 
 // GetBroadcasts implementation of the memberlist.Delegate interface
 // called when user data messages can be broadcast.
-func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte {
+func (g *gossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte {
 	return g.broadcasts.GetBroadcasts(overhead, limit)
 }
 
 // LocalState implementation of the memberlist.Delegate interface
 // sends this Node's state data.
-func (g *GossipMemberSet) LocalState(join bool) []byte {
-	pb, err := g.statusHandler.LocalStatus()
-	if err != nil {
-		g.Logger.Printf("error getting local state, err=%s", err)
-		return []byte{}
+func (g *gossipMemberSet) LocalState(join bool) []byte {
+	m := &pilosa.NodeStatus{
+		Node:      g.papi.Node(),
+		MaxShards: g.papi.MaxShards(context.Background()),
+		Schema:    &pilosa.Schema{Indexes: g.papi.Schema(context.Background())},
 	}
 
 	// Marshal nodestate data to bytes.
-	buf, err := proto.Marshal(pb)
+	buf, err := pilosa.MarshalInternalMessage(m, g.papi.Serializer)
 	if err != nil {
 		g.Logger.Printf("error marshalling nodestate data, err=%s", err)
 		return []byte{}
@@ -332,59 +263,49 @@ func (g *GossipMemberSet) LocalState(join bool) []byte {
 
 // MergeRemoteState implementation of the memberlist.Delegate interface
 // receive and process the remote side's LocalState.
-func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) {
-	// Unmarshal nodestate data.
-	var pb internal.NodeStatus
-	if err := proto.Unmarshal(buf, &pb); err != nil {
-		g.Logger.Printf("error unmarshalling nodestate data, err=%s", err)
-		return
-	}
-	err := g.statusHandler.HandleRemoteStatus(&pb)
+func (g *gossipMemberSet) MergeRemoteState(buf []byte, join bool) {
+	err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf))
 	if err != nil {
 		g.Logger.Printf("merge state error: %s", err)
 	}
 }
 
-// GossipEventReceiver is used to enable an application to receive
+// gossipEventReceiver is used to enable an application to receive
 // events about joins and leaves over a channel.
 //
 // Care must be taken that events are processed in a timely manner from
 // the channel, since this delegate will block until an event can be sent.
-type GossipEventReceiver struct {
-	ch           chan memberlist.NodeEvent
-	eventHandler pilosa.EventHandler
+type gossipEventReceiver struct {
+	ch   chan memberlist.NodeEvent
+	papi *pilosa.API
 
-	Logger pilosa.Logger
+	logger *log.Logger
 }
 
-// NewGossipEventReceiver returns a new instance of GossipEventReceiver.
-func NewGossipEventReceiver(logger pilosa.Logger) *GossipEventReceiver {
-	return &GossipEventReceiver{
+// newGossipEventReceiver returns a new instance of GossipEventReceiver.
+func newGossipEventReceiver(logger *log.Logger, papi *pilosa.API) *gossipEventReceiver {
+	ger := &gossipEventReceiver{
 		ch:     make(chan memberlist.NodeEvent, 1),
-		Logger: logger,
+		logger: logger,
+		papi:   papi,
 	}
+	go ger.listen()
+	return ger
 }
 
-func (g *GossipEventReceiver) NotifyJoin(n *memberlist.Node) {
+func (g *gossipEventReceiver) NotifyJoin(n *memberlist.Node) {
 	g.ch <- memberlist.NodeEvent{memberlist.NodeJoin, n}
 }
 
-func (g *GossipEventReceiver) NotifyLeave(n *memberlist.Node) {
+func (g *gossipEventReceiver) NotifyLeave(n *memberlist.Node) {
 	g.ch <- memberlist.NodeEvent{memberlist.NodeLeave, n}
 }
 
-func (g *GossipEventReceiver) NotifyUpdate(n *memberlist.Node) {
+func (g *gossipEventReceiver) NotifyUpdate(n *memberlist.Node) {
 	g.ch <- memberlist.NodeEvent{memberlist.NodeUpdate, n}
 }
 
-// Start implements the pilosa.EventReceiver interface and sets the EventHandler.
-func (g *GossipEventReceiver) Start(h pilosa.EventHandler) error {
-	g.eventHandler = h
-	go g.listen()
-	return nil
-}
-
-func (g *GossipEventReceiver) listen() {
+func (g *gossipEventReceiver) listen() {
 	var nodeEventType pilosa.NodeEventType
 	for {
 		e := <-g.ch
@@ -400,46 +321,29 @@ func (g *GossipEventReceiver) listen() {
 		}
 
 		// Get the node from the event.Node meta data.
-		var n internal.Node
-		if err := proto.Unmarshal(e.Node.Meta, &n); err != nil {
-			panic("failed to unmarshal event node meta data")
+		var n pilosa.Node
+		if err := g.papi.Serializer.Unmarshal(e.Node.Meta, &n); err != nil {
+			panic("failed to unmarshal event node meta into node")
 		}
-		node := pilosa.DecodeNode(&n)
 
 		ne := &pilosa.NodeEvent{
 			Event: nodeEventType,
-			Node:  node,
+			Node:  &n,
 		}
-		if err := g.eventHandler.ReceiveEvent(ne); err != nil {
-			g.Logger.Printf("receive event error: %s", err)
+		buf, err := pilosa.MarshalInternalMessage(ne, g.papi.Serializer)
+		if err != nil {
+			panic(err)
+		}
+		if err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf)); err != nil {
+			g.logger.Printf("receive event error: %s", err)
 		}
-	}
-}
-
-// broadcast represents an implementation of memberlist.Broadcast
-type broadcast struct {
-	msg    []byte
-	notify chan<- struct{}
-}
-
-func (b *broadcast) Invalidates(other memberlist.Broadcast) bool {
-	return false
-}
-
-func (b *broadcast) Message() []byte {
-	return b.msg
-}
-
-func (b *broadcast) Finished() {
-	if b.notify != nil {
-		close(b.notify)
 	}
 }
 
 // Transport is a gossip transport for binding to a port.
 type Transport struct {
 	//memberlist.Transport
-	Net *memberlist.NetTransport
+	net *memberlist.NetTransport
 	URI *pilosa.URI
 }
 
@@ -466,7 +370,7 @@ func NewTransport(host string, port int, logger *log.Logger) (*Transport, error)
 	}
 
 	return &Transport{
-		Net: net,
+		net: net,
 		URI: uri,
 	}, nil
 }
@@ -580,3 +484,21 @@ type Config struct {
 	Nodes         int           `toml:"nodes"`
 	ToTheDeadTime toml.Duration `toml:"to-the-dead-time"`
 }
+
+// hostToIP converts host to an IP4 address based on net.LookupIP().
+func hostToIP(host string) string {
+	// if host is not an IP addr, check net.LookupIP()
+	if net.ParseIP(host) == nil {
+		hosts, err := net.LookupIP(host)
+		if err != nil {
+			return host
+		}
+		for _, h := range hosts {
+			// this restricts pilosa to IP4
+			if h.To4() != nil {
+				return h.String()
+			}
+		}
+	}
+	return host
+}

handler_internal_test.go

@@ -1,89 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa
-
-import (
-	"encoding/json"
-	"reflect"
-	"testing"
-)
-
-// Test custom UnmarshalJSON for postIndexRequest object
-func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
-	tests := []struct {
-		json     string
-		expected postIndexRequest
-		err      string
-	}{
-		{json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}},
-		{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
-		{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
-		{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
-	}
-	for _, test := range tests {
-		actual := &postIndexRequest{}
-		err := json.Unmarshal([]byte(test.json), actual)
-
-		if err != nil {
-			if test.err == "" || test.err != err.Error() {
-				t.Errorf("expected error: %v, but got result: %v", test.err, err)
-			}
-		} else {
-			if test.err != "" {
-				t.Errorf("expected error: %v, but got no error", test.err)
-			}
-		}
-
-		if test.err == "" {
-			if !reflect.DeepEqual(*actual, test.expected) {
-				t.Errorf("expected: %v, but got: %v", test.expected, *actual)
-			}
-		}
-
-	}
-}
-
-// Test custom UnmarshalJSON for postFrameRequest object
-func TestPostFrameRequestUnmarshalJSON(t *testing.T) {
-	tests := []struct {
-		json     string
-		expected postFrameRequest
-		err      string
-	}{
-		{json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}},
-		{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
-		{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
-		{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
-		{json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"},
-		{json: `{"options": {"cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{CacheType: "type"}}},
-		{json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"},
-	}
-	for _, test := range tests {
-		actual := &postFrameRequest{}
-		err := json.Unmarshal([]byte(test.json), actual)
-		if err != nil {
-			if test.err == "" || test.err != err.Error() {
-				t.Errorf("expected error: %v, but got result: %v", test.err, err)
-			}
-		}
-
-		if test.err == "" {
-			if !reflect.DeepEqual(*actual, test.expected) {
-				t.Errorf("expected: %v, but got: %v", test.expected, *actual)
-			}
-		}
-
-	}
-}

holder.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -28,17 +27,16 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pkg/errors"
 	uuid "github.com/satori/go.uuid"
 )
 
 const (
-	// DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval.
-	DefaultCacheFlushInterval = 1 * time.Minute
+	// defaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval.
+	defaultCacheFlushInterval = 1 * time.Minute
 
-	// FileLimit is the maximum open file limit (ulimit -n) to automatically set.
-	FileLimit = 262144 // (512^2)
+	// fileLimit is the maximum open file limit (ulimit -n) to automatically set.
+	fileLimit = 262144 // (512^2)
 )
 
 // Holder represents a container for indexes.
@@ -51,7 +49,7 @@ type Holder struct {
 	// opened channel is closed once Open() completes.
 	opened chan struct{}
 
-	Broadcaster Broadcaster
+	broadcaster broadcaster
 
 	NewAttrStore func(string) AttrStore
 
@@ -66,7 +64,7 @@ type Holder struct {
 	Path string
 
 	// The interval at which the cached row ids are persisted to disk.
-	CacheFlushInterval time.Duration
+	cacheFlushInterval time.Duration
 
 	Logger Logger
 }
@@ -79,12 +77,12 @@ func NewHolder() *Holder {
 
 		opened: make(chan struct{}),
 
-		Broadcaster: NopBroadcaster,
+		broadcaster: NopBroadcaster,
 		Stats:       NopStatsClient,
 
-		NewAttrStore: NewNopAttrStore,
+		NewAttrStore: newNopAttrStore,
 
-		CacheFlushInterval: DefaultCacheFlushInterval,
+		cacheFlushInterval: defaultCacheFlushInterval,
 
 		Logger: NopLogger,
 	}
@@ -112,7 +110,8 @@ func (h *Holder) Open() error {
 	}
 
 	for _, fi := range fis {
-		if !fi.IsDir() {
+		// Skip files or hidden directories.
+		if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
 			continue
 		}
 
@@ -200,37 +199,53 @@ func (h *Holder) HasData() (bool, error) {
 	return false, nil
 }
 
-// MaxSlices returns MaxSlice map for all indexes.
-func (h *Holder) MaxSlices() map[string]uint64 {
+// maxShards returns MaxShard map for all indexes.
+func (h *Holder) maxShards() map[string]uint64 {
 	a := make(map[string]uint64)
 	for _, index := range h.Indexes() {
-		a[index.Name()] = index.MaxSlice()
+		a[index.Name()] = index.maxShard()
 	}
 	return a
 }
 
-// Schema returns schema information for all indexes, frames, and views.
+// Schema returns schema information for all indexes, fields, and views.
 func (h *Holder) Schema() []*IndexInfo {
 	var a []*IndexInfo
 	for _, index := range h.Indexes() {
 		di := &IndexInfo{Name: index.Name()}
-		for _, frame := range index.Frames() {
-			fi := &FrameInfo{Name: frame.Name(), Options: frame.Options()}
-			for _, view := range frame.Views() {
-				fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()})
+		for _, field := range index.Fields() {
+			fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
+			for _, view := range field.views() {
+				fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
 			}
 			sort.Sort(viewInfoSlice(fi.Views))
-			di.Frames = append(di.Frames, fi)
+			di.Fields = append(di.Fields, fi)
 		}
-		sort.Sort(frameInfoSlice(di.Frames))
+		sort.Sort(fieldInfoSlice(di.Fields))
 		a = append(a, di)
 	}
 	sort.Sort(indexInfoSlice(a))
 	return a
 }
 
-// ApplySchema applies an internal Schema to Holder.
-func (h *Holder) ApplySchema(schema *internal.Schema) error {
+// limitedSchema returns schema information for all indexes and fields.
+func (h *Holder) limitedSchema() []*IndexInfo {
+	var a []*IndexInfo
+	for _, index := range h.Indexes() {
+		di := &IndexInfo{Name: index.Name()}
+		for _, field := range index.Fields() {
+			fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
+			di.Fields = append(di.Fields, fi)
+		}
+		sort.Sort(fieldInfoSlice(di.Fields))
+		a = append(a, di)
+	}
+	sort.Sort(indexInfoSlice(a))
+	return a
+}
+
+// applySchema applies an internal Schema to Holder.
+func (h *Holder) applySchema(schema *Schema) error {
 	// Create indexes that don't exist.
 	for _, index := range schema.Indexes {
 		opt := IndexOptions{}
@@ -238,16 +253,15 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error {
 		if err != nil {
 			return errors.Wrap(err, "creating index")
 		}
-		// Create frames that don't exist.
-		for _, f := range index.Frames {
-			opt := decodeFrameOptions(f.Meta)
-			frame, err := idx.CreateFrameIfNotExists(f.Name, *opt)
+		// Create fields that don't exist.
+		for _, f := range index.Fields {
+			field, err := idx.createFieldIfNotExists(f.Name, f.Options)
 			if err != nil {
-				return errors.Wrap(err, "creating frame")
+				return errors.Wrap(err, "creating field")
 			}
 			// Create views that don't exist.
 			for _, v := range f.Views {
-				_, err := frame.CreateViewIfNotExists(v)
+				_, err := field.createViewIfNotExists(v.Name)
 				if err != nil {
 					return errors.Wrap(err, "creating view")
 				}
@@ -257,20 +271,6 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error {
 	return nil
 }
 
-// EncodeMaxSlices creates and internal representation of max slices.
-func (h *Holder) EncodeMaxSlices() *internal.MaxSlices {
-	return &internal.MaxSlices{
-		Standard: h.MaxSlices(),
-	}
-}
-
-// EncodeSchema creates an internal representation of schema.
-func (h *Holder) EncodeSchema() *internal.Schema {
-	return &internal.Schema{
-		Indexes: EncodeIndexes(h.Indexes()),
-	}
-}
-
 // IndexPath returns the path where a given index is stored.
 func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }
 
@@ -304,7 +304,7 @@ func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) {
 
 	// Ensure index doesn't already exist.
 	if h.indexes[name] != nil {
-		return nil, ErrIndexExists
+		return nil, newConflictError(ErrIndexExists)
 	}
 	return h.createIndex(name, opt)
 }
@@ -339,12 +339,15 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
 		return nil, errors.Wrap(err, "creating")
 	}
 
+	index.keys = opt.Keys
+
 	if err := index.Open(); err != nil {
 		return nil, errors.Wrap(err, "opening")
+	} else if err := index.saveMeta(); err != nil {
+		return nil, errors.Wrap(err, "meta")
 	}
 
 	// Update options.
-
 	h.indexes[index.Name()] = index
 
 	return index, nil
@@ -355,11 +358,11 @@ func (h *Holder) newIndex(path, name string) (*Index, error) {
 	if err != nil {
 		return nil, err
 	}
-	index.Logger = h.Logger
+	index.logger = h.Logger
 	index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name()))
-	index.broadcaster = h.Broadcaster
-	index.NewAttrStore = h.NewAttrStore
-	index.columnAttrStore = h.NewAttrStore(filepath.Join(index.path, ".data"))
+	index.broadcaster = h.broadcaster
+	index.newAttrStore = h.NewAttrStore
+	index.columnAttrs = h.NewAttrStore(filepath.Join(index.path, ".data"))
 	return index, nil
 }
 
@@ -368,10 +371,10 @@ func (h *Holder) DeleteIndex(name string) error {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 
-	// Ignore if index doesn't exist.
+	// Confirm index exists.
 	index := h.index(name)
 	if index == nil {
-		return nil
+		return newNotFoundError(ErrIndexNotFound)
 	}
 
 	// Close index.
@@ -390,37 +393,37 @@ func (h *Holder) DeleteIndex(name string) error {
 	return nil
 }
 
-// Frame returns the frame for an index and name.
-func (h *Holder) Frame(index, name string) *Frame {
+// Field returns the field for an index and name.
+func (h *Holder) Field(index, name string) *Field {
 	idx := h.Index(index)
 	if idx == nil {
 		return nil
 	}
-	return idx.Frame(name)
+	return idx.Field(name)
 }
 
-// View returns the view for an index, frame, and name.
-func (h *Holder) View(index, frame, name string) *View {
-	f := h.Frame(index, frame)
+// view returns the view for an index, field, and name.
+func (h *Holder) view(index, field, name string) *view {
+	f := h.Field(index, field)
 	if f == nil {
 		return nil
 	}
-	return f.View(name)
+	return f.view(name)
 }
 
-// Fragment returns the fragment for an index, frame & slice.
-func (h *Holder) Fragment(index, frame, view string, slice uint64) *Fragment {
-	v := h.View(index, frame, view)
+// fragment returns the fragment for an index, field & shard.
+func (h *Holder) fragment(index, field, view string, shard uint64) *fragment {
+	v := h.view(index, field, view)
 	if v == nil {
 		return nil
 	}
-	return v.Fragment(slice)
+	return v.Fragment(shard)
 }
 
 // monitorCacheFlush periodically flushes all fragment caches sequentially.
 // This is run in a goroutine.
 func (h *Holder) monitorCacheFlush() {
-	ticker := time.NewTicker(h.CacheFlushInterval)
+	ticker := time.NewTicker(h.cacheFlushInterval)
 	defer ticker.Stop()
 
 	for {
@@ -435,9 +438,9 @@ func (h *Holder) monitorCacheFlush() {
 
 func (h *Holder) flushCaches() {
 	for _, index := range h.Indexes() {
-		for _, frame := range index.Frames() {
-			for _, view := range frame.Views() {
-				for _, fragment := range view.Fragments() {
+		for _, field := range index.Fields() {
+			for _, view := range field.views() {
+				for _, fragment := range view.allFragments() {
 					select {
 					case <-h.closing:
 						return
@@ -445,7 +448,7 @@ func (h *Holder) flushCaches() {
 					}
 
 					if err := fragment.FlushCache(); err != nil {
-						h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath())
+						h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.cachePath())
 					}
 				}
 			}
@@ -453,13 +456,13 @@ func (h *Holder) flushCaches() {
 	}
 }
 
-// RecalculateCaches recalculates caches on every index in the holder. This is
+// recalculateCaches recalculates caches on every index in the holder. This is
 // probably not practical to call in real-world workloads, but makes writing
 // integration tests much eaiser, since one doesn't have to wait 10 seconds
 // after setting bits to get expected response.
-func (h *Holder) RecalculateCaches() {
+func (h *Holder) recalculateCaches() {
 	for _, index := range h.Indexes() {
-		index.RecalculateCaches()
+		index.recalculateCaches()
 	}
 }
 
@@ -473,11 +476,11 @@ func (h *Holder) setFileLimit() {
 		return
 	}
 	// If the soft limit is lower than the FileLimit constant, we will try to change it.
-	if oldLimit.Cur < FileLimit {
-		newLimit.Cur = FileLimit
+	if oldLimit.Cur < fileLimit {
+		newLimit.Cur = fileLimit
 		// If the hard limit is not high enough, we will try to change it too.
-		if oldLimit.Max < FileLimit {
-			newLimit.Max = FileLimit
+		if oldLimit.Max < fileLimit {
+			newLimit.Max = fileLimit
 		} else {
 			newLimit.Max = oldLimit.Max
 		}
@@ -505,8 +508,8 @@ func (h *Holder) setFileLimit() {
 		if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil {
 			h.Logger.Printf("ERROR checking open file limit: %s", err)
 		} else {
-			if oldLimit.Cur < FileLimit {
-				h.Logger.Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit)
+			if oldLimit.Cur < fileLimit {
+				h.Logger.Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", fileLimit, oldLimit.Cur, fileLimit)
 			}
 		}
 	}
@@ -558,14 +561,15 @@ func (h *Holder) logStartup() error {
 	return nil
 }
 
-// HolderSyncer is an active anti-entropy tool that compares the local holder
+// holderSyncer is an active anti-entropy tool that compares the local holder
 // with a remote holder based on block checksums and resolves differences.
-type HolderSyncer struct {
+type holderSyncer struct {
+	mu sync.Mutex
+
 	Holder *Holder
 
-	Node         *Node
-	Cluster      *Cluster
-	RemoteClient *http.Client
+	Node    *Node
+	Cluster *cluster
 
 	// Stats
 	Stats StatsClient
@@ -575,7 +579,7 @@ type HolderSyncer struct {
 }
 
 // IsClosing returns true if the syncer has been marked to close.
-func (s *HolderSyncer) IsClosing() bool {
+func (s *holderSyncer) IsClosing() bool {
 	select {
 	case <-s.Closing:
 		return true
@@ -585,7 +589,9 @@ func (s *HolderSyncer) IsClosing() bool {
 }
 
 // SyncHolder compares the holder on host with the local holder and resolves differences.
-func (s *HolderSyncer) SyncHolder() error {
+func (s *holderSyncer) SyncHolder() error {
+	s.mu.Lock() // only allow one instance of SyncHolder to be running at a time
+	defer s.mu.Unlock()
 	ti := time.Now()
 	// Iterate over schema in sorted order.
 	for _, di := range s.Holder.Schema() {
@@ -600,15 +606,15 @@ func (s *HolderSyncer) SyncHolder() error {
 		}
 
 		tf := time.Now()
-		for _, fi := range di.Frames {
+		for _, fi := range di.Fields {
 			// Verify syncer has not closed.
 			if s.IsClosing() {
 				return nil
 			}
 
-			// Sync frame row attributes.
-			if err := s.syncFrame(di.Name, fi.Name); err != nil {
-				return fmt.Errorf("frame sync error: index=%s, frame=%s, err=%s", di.Name, fi.Name, err)
+			// Sync field row attributes.
+			if err := s.syncField(di.Name, fi.Name); err != nil {
+				return fmt.Errorf("field sync error: index=%s, field=%s, err=%s", di.Name, fi.Name, err)
 			}
 
 			for _, vi := range fi.Views {
@@ -617,9 +623,9 @@ func (s *HolderSyncer) SyncHolder() error {
 					return nil
 				}
 
-				for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ {
-					// Ignore slices that this host doesn't own.
-					if !s.Cluster.OwnsSlice(s.Node.ID, di.Name, slice) {
+				for shard := uint64(0); shard <= s.Holder.Index(di.Name).maxShard(); shard++ {
+					// Ignore shards that this host doesn't own.
+					if !s.Cluster.ownsShard(s.Node.ID, di.Name, shard) {
 						continue
 					}
 
@@ -629,12 +635,12 @@ func (s *HolderSyncer) SyncHolder() error {
 					}
 
 					// Sync fragment if own it.
-					if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil {
-						return fmt.Errorf("fragment sync error: index=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err)
+					if err := s.syncFragment(di.Name, fi.Name, vi.Name, shard); err != nil {
+						return fmt.Errorf("fragment sync error: index=%s, field=%s, shard=%d, err=%s", di.Name, fi.Name, shard, err)
 					}
 				}
 			}
-			s.Stats.Histogram("syncFrame", float64(time.Since(tf)), 1.0)
+			s.Stats.Histogram("syncField", float64(time.Since(tf)), 1.0)
 			tf = time.Now() // reset tf
 		}
 		s.Stats.Histogram("syncIndex", float64(time.Since(ti)), 1.0)
@@ -645,7 +651,7 @@ func (s *HolderSyncer) SyncHolder() error {
 }
 
 // syncIndex synchronizes index attributes with the rest of the cluster.
-func (s *HolderSyncer) syncIndex(index string) error {
+func (s *holderSyncer) syncIndex(index string) error {
 	// Retrieve index reference.
 	idx := s.Holder.Index(index)
 	if idx == nil {
@@ -662,11 +668,9 @@ func (s *HolderSyncer) syncIndex(index string) error {
 
 	// Sync with every other host.
 	for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) {
-		client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
-
 		// Retrieve attributes from differing blocks.
 		// Skip update and recomputation if no attributes have changed.
-		m, err := client.ColumnAttrDiff(context.Background(), index, blks)
+		m, err := s.Cluster.InternalClient.ColumnAttrDiff(context.Background(), &node.URI, index, blks)
 		if err != nil {
 			return errors.Wrap(err, "getting differing blocks")
 		} else if len(m) == 0 {
@@ -689,38 +693,36 @@ func (s *HolderSyncer) syncIndex(index string) error {
 	return nil
 }
 
-// syncFrame synchronizes frame attributes with the rest of the cluster.
-func (s *HolderSyncer) syncFrame(index, name string) error {
-	// Retrieve frame reference.
-	f := s.Holder.Frame(index, name)
+// syncField synchronizes field attributes with the rest of the cluster.
+func (s *holderSyncer) syncField(index, name string) error {
+	// Retrieve field reference.
+	f := s.Holder.Field(index, name)
 	if f == nil {
 		return nil
 	}
 	indexTag := fmt.Sprintf("index:%s", index)
-	frameTag := fmt.Sprintf("frame:%s", name)
+	fieldTag := fmt.Sprintf("field:%s", name)
 
 	// Read block checksums.
 	blks, err := f.RowAttrStore().Blocks()
 	if err != nil {
 		return errors.Wrap(err, "getting blocks")
 	}
-	s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag})
+	s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, fieldTag})
 
 	// Sync with every other host.
 	for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) {
-		client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
-
 		// Retrieve attributes from differing blocks.
 		// Skip update and recomputation if no attributes have changed.
-		m, err := client.RowAttrDiff(context.Background(), index, name, blks)
-		if err == ErrFrameNotFound {
-			continue // frame not created remotely yet, skip
+		m, err := s.Cluster.InternalClient.RowAttrDiff(context.Background(), &node.URI, index, name, blks)
+		if err == ErrFieldNotFound {
+			continue // field not created remotely yet, skip
 		} else if err != nil {
 			return errors.Wrap(err, "getting differing blocks")
 		} else if len(m) == 0 {
 			continue
 		}
-		s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.ID})
+		s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, fieldTag, node.ID})
 
 		// Update local copy.
 		if err := f.RowAttrStore().SetBulkAttrs(m); err != nil {
@@ -738,53 +740,52 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
 }
 
 // syncFragment synchronizes a fragment with the rest of the cluster.
-func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) error {
-	// Retrieve local frame.
-	f := s.Holder.Frame(index, frame)
+func (s *holderSyncer) syncFragment(index, field, view string, shard uint64) error {
+	// Retrieve local field.
+	f := s.Holder.Field(index, field)
 	if f == nil {
-		return ErrFrameNotFound
+		return ErrFieldNotFound
 	}
 
 	// Ensure view exists locally.
-	v, err := f.CreateViewIfNotExists(view)
+	v, err := f.createViewIfNotExists(view)
 	if err != nil {
 		return errors.Wrap(err, "creating view")
 	}
 
 	// Ensure fragment exists locally.
-	frag, err := v.CreateFragmentIfNotExists(slice)
+	frag, err := v.CreateFragmentIfNotExists(shard)
 	if err != nil {
 		return errors.Wrap(err, "creating fragment")
 	}
 
 	// Sync fragments together.
-	fs := FragmentSyncer{
-		Fragment:     frag,
-		Node:         s.Node,
-		Cluster:      s.Cluster,
-		Closing:      s.Closing,
-		RemoteClient: s.RemoteClient,
+	fs := fragmentSyncer{
+		Fragment: frag,
+		Node:     s.Node,
+		Cluster:  s.Cluster,
+		Closing:  s.Closing,
 	}
-	if err := fs.SyncFragment(); err != nil {
+	if err := fs.syncFragment(); err != nil {
 		return errors.Wrap(err, "syncing fragment")
 	}
 
 	return nil
 }
 
-// HolderCleaner removes fragments and data files that are no longer used.
-type HolderCleaner struct {
+// holderCleaner removes fragments and data files that are no longer used.
+type holderCleaner struct {
 	Node *Node
 
 	Holder  *Holder
-	Cluster *Cluster
+	Cluster *cluster
 
 	// Signals that the sync should stop.
 	Closing <-chan struct{}
 }
 
 // IsClosing returns true if the cleaner has been marked to close.
-func (c *HolderCleaner) IsClosing() bool {
+func (c *holderCleaner) IsClosing() bool {
 	select {
 	case <-c.Closing:
 		return true
@@ -795,7 +796,7 @@ func (c *HolderCleaner) IsClosing() bool {
 
 // CleanHolder compares the holder with the cluster state and removes
 // any unnecessary fragments and files.
-func (c *HolderCleaner) CleanHolder() error {
+func (c *holderCleaner) CleanHolder() error {
 	for _, index := range c.Holder.Indexes() {
 		// Verify cleaner has not closed.
 		if c.IsClosing() {
@@ -803,19 +804,19 @@ func (c *HolderCleaner) CleanHolder() error {
 		}
 
 		// Get the fragments that node is responsible for (based on hash(index, node)).
-		containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node)
+		containedShards := c.Cluster.containsShards(index.Name(), index.maxShard(), c.Node)
 
 		// Get the fragments registered in memory.
-		for _, frame := range index.Frames() {
-			for _, view := range frame.Views() {
-				for _, fragment := range view.Fragments() {
-					fragSlice := fragment.Slice()
+		for _, field := range index.Fields() {
+			for _, view := range field.views() {
+				for _, fragment := range view.allFragments() {
+					fragShard := fragment.shard
 					// Ignore fragments that should be present.
-					if uint64InSlice(fragSlice, containedSlices) {
+					if uint64InSlice(fragShard, containedShards) {
 						continue
 					}
 					// Delete fragment.
-					if err := view.DeleteFragment(fragSlice); err != nil {
+					if err := view.deleteFragment(fragShard); err != nil {
 						return errors.Wrap(err, "deleting fragment")
 					}
 				}

holder_internal_test.go

@@ -0,0 +1,291 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type tHolder struct {
+	*Holder
+}
+
+// Close closes the holder and removes all underlying data.
+func (h *tHolder) Close() error {
+	defer os.RemoveAll(h.Path)
+	return h.Holder.Close()
+}
+
+// Reopen instantiates and opens a new holder.
+// Note that the holder must be Closed first.
+func (h *tHolder) Reopen() error {
+	path, logger := h.Path, h.Holder.Logger
+	h.Holder = NewHolder()
+	h.Holder.Path = path
+	h.Holder.Logger = logger
+	if err := h.Holder.Open(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func newHolder() *tHolder {
+	path, err := ioutil.TempDir("", "pilosa-")
+	if err != nil {
+		panic(err)
+	}
+
+	h := &tHolder{Holder: NewHolder()}
+	h.Path = path
+	return h
+}
+
+// MustCreateFieldIfNotExists returns a given field. Panic on error.
+func (h *tHolder) MustCreateFieldIfNotExists(index, field string) *Field {
+	f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, OptFieldTypeDefault())
+	if err != nil {
+		panic(err)
+	}
+	return f
+}
+
+// MustCreateIndexIfNotExists returns a given index. Panic on error.
+func (h *tHolder) MustCreateIndexIfNotExists(index string, opt IndexOptions) *Index {
+	idx, err := h.Holder.CreateIndexIfNotExists(index, opt)
+	if err != nil {
+		panic(err)
+	}
+	return idx
+}
+
+// SetBit clears a bit on the given field.
+func (h *tHolder) SetBit(index, field string, rowID, columnID uint64) {
+	f := h.MustCreateFieldIfNotExists(index, field)
+	_, err := f.SetBit(rowID, columnID, nil)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// Row returns a Row for a given field.
+func (h *tHolder) Row(index, field string, rowID uint64) *Row {
+	f := h.MustCreateFieldIfNotExists(index, field)
+	row, err := f.Row(rowID)
+	if err != nil {
+		panic(err)
+	}
+	return row
+}
+
+func TestHolder_Optn(t *testing.T) {
+	t.Run("ErrViewPermission", func(t *testing.T) {
+		if os.Geteuid() == 0 {
+			t.Skip("Skipping permissions test since user is root.")
+		}
+		h := newHolder()
+		defer h.Close()
+
+		if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
+			t.Fatal(err)
+		} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
+			t.Fatal(err)
+		} else if _, err := field.createViewIfNotExists(viewStandard); err != nil {
+			t.Fatal(err)
+		} else if err := h.Holder.Close(); err != nil {
+			t.Fatal(err)
+		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil {
+			t.Fatal(err)
+		}
+		defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0777)
+
+		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+	t.Run("ErrViewFragmentsMkdir", func(t *testing.T) {
+		if os.Geteuid() == 0 {
+			t.Skip("Skipping permissions test since user is root.")
+		}
+		h := newHolder()
+		defer h.Close()
+
+		if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
+			t.Fatal(err)
+		} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
+			t.Fatal(err)
+		} else if _, err := field.createViewIfNotExists(viewStandard); err != nil {
+			t.Fatal(err)
+		} else if err := h.Holder.Close(); err != nil {
+			t.Fatal(err)
+		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil {
+			t.Fatal(err)
+		}
+		defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0777)
+
+		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrFragmentCachePermission", func(t *testing.T) {
+		if os.Geteuid() == 0 {
+			t.Skip("Skipping permissions test since user is root.")
+		}
+		h := newHolder()
+		defer h.Close()
+
+		if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
+			t.Fatal(err)
+		} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
+			t.Fatal(err)
+		} else if view, err := field.createViewIfNotExists(viewStandard); err != nil {
+			t.Fatal(err)
+		} else if _, err := field.SetBit(0, 0, nil); err != nil {
+			t.Fatal(err)
+		} else if err := view.Fragment(0).FlushCache(); err != nil {
+			t.Fatal(err)
+		} else if err := h.Holder.Close(); err != nil {
+			t.Fatal(err)
+		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil {
+			t.Fatal(err)
+		}
+		defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0666)
+
+		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+}
+
+// Ensure holder can clean up orphaned fragments.
+func TestHolderCleaner_CleanHolder(t *testing.T) {
+	cluster := NewTestCluster(2)
+
+	// Create a local holder.
+	hldr0 := newHolder()
+	defer hldr0.Close()
+
+	// Mock 2-node, fully replicated cluster.
+	cluster.ReplicaN = 2
+
+	cluster.Nodes[0].URI = NewTestURIFromHostPort("localhost", 0)
+
+	// Create fields on nodes.
+	for _, hldr := range []*tHolder{hldr0} {
+		hldr.MustCreateFieldIfNotExists("i", "f")
+		hldr.MustCreateFieldIfNotExists("i", "f0")
+		hldr.MustCreateFieldIfNotExists("y", "z")
+	}
+
+	// Set data on the local holder.
+	hldr0.SetBit("i", "f", 0, 10)
+	hldr0.SetBit("i", "f", 0, 4000)
+	hldr0.SetBit("i", "f", 2, 20)
+	hldr0.SetBit("i", "f", 3, 10)
+	hldr0.SetBit("i", "f", 120, 10)
+	hldr0.SetBit("i", "f", 200, 4)
+
+	hldr0.SetBit("i", "f0", 9, ShardWidth+5)
+
+	hldr0.SetBit("y", "z", 10, (2*ShardWidth)+4)
+	hldr0.SetBit("y", "z", 10, (2*ShardWidth)+5)
+	hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7)
+
+	// Set highest shard.
+	hldr0.Index("i").setRemoteMaxShard(1)
+	hldr0.Index("y").setRemoteMaxShard(2)
+
+	// Keep replication the same and ensure we get the expected results.
+	cluster.ReplicaN = 2
+
+	// Set up cleaner for replication 2.
+	cleaner2 := holderCleaner{
+		Node:    cluster.Nodes[0],
+		Holder:  hldr0.Holder,
+		Cluster: cluster,
+	}
+
+	if err := cleaner2.CleanHolder(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify data is the same on both nodes.
+	for i, hldr := range []*tHolder{hldr0} {
+		if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
+			t.Fatalf("unexpected columns(%d/0): %+v", i, a)
+		} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
+			t.Fatalf("unexpected columns(%d/2): %+v", i, a)
+		} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
+			t.Fatalf("unexpected columns(%d/3): %+v", i, a)
+		} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
+			t.Fatalf("unexpected columns(%d/120): %+v", i, a)
+		} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
+			t.Fatalf("unexpected columns(%d/200): %+v", i, a)
+		}
+
+		if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) {
+			t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
+		}
+
+		if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
+			t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
+		}
+	}
+
+	// Change replication factor to ensure we have fragments to remove.
+	cluster.ReplicaN = 1
+
+	// Set up cleaner for replication 1.
+	cleaner1 := holderCleaner{
+		Node:    cluster.Nodes[0],
+		Holder:  hldr0.Holder,
+		Cluster: cluster,
+	}
+
+	if err := cleaner1.CleanHolder(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify data is the same on both nodes.
+	for i, hldr := range []*tHolder{hldr0} {
+		if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
+			t.Fatalf("unexpected columns(%d/0): %+v", i, a)
+		} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
+			t.Fatalf("unexpected columns(%d/2): %+v", i, a)
+		} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
+			t.Fatalf("unexpected columns(%d/3): %+v", i, a)
+		} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
+			t.Fatalf("unexpected columns(%d/120): %+v", i, a)
+		} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
+			t.Fatalf("unexpected columns(%d/200): %+v", i, a)
+		}
+
+		f := hldr.fragment("i", "f0", viewStandard, 1)
+		if f != nil {
+			t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f)
+		}
+
+		if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
+			t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
+		}
+	}
+}

holder_test.go

@@ -24,8 +24,6 @@ import (
 	"testing"
 
 	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/pql"
-	"github.com/pilosa/pilosa/server"
 	"github.com/pilosa/pilosa/test"
 )
 
@@ -91,7 +89,7 @@ func TestHolder_Open(t *testing.T) {
 		}
 	})
 
-	t.Run("ErrFramePermission", func(t *testing.T) {
+	t.Run("ErrFieldPermission", func(t *testing.T) {
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
 		}
@@ -100,7 +98,7 @@ func TestHolder_Open(t *testing.T) {
 
 		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
+		} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
@@ -113,13 +111,13 @@ func TestHolder_Open(t *testing.T) {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
-	t.Run("ErrFrameMetaCorrupt", func(t *testing.T) {
+	t.Run("ErrFieldOptionsCorrupt", func(t *testing.T) {
 		h := test.MustOpenHolder()
 		defer h.Close()
 
 		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
+		} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
@@ -127,17 +125,17 @@ func TestHolder_Open(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening frames: open frame: name=bar, err=loading meta: unmarshaling: unexpected EOF") {
+		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=loading meta: unmarshaling: unexpected EOF") {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
-	t.Run("ErrFrameAttrStoreCorrupt", func(t *testing.T) {
+	t.Run("ErrFieldAttrStoreCorrupt", func(t *testing.T) {
 		h := test.MustOpenHolder()
 		defer h.Close()
 
 		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
+		} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
@@ -145,56 +143,7 @@ func TestHolder_Open(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening frames: open frame: name=bar, err=opening attrstore: opening storage: invalid database") {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	})
-
-	t.Run("ErrViewPermission", func(t *testing.T) {
-		if os.Geteuid() == 0 {
-			t.Skip("Skipping permissions test since user is root.")
-		}
-		h := test.MustOpenHolder()
-		defer h.Close()
-
-		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
-			t.Fatal(err)
-		} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
-			t.Fatal(err)
-		} else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
-			t.Fatal(err)
-		} else if err := h.Holder.Close(); err != nil {
-			t.Fatal(err)
-		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil {
-			t.Fatal(err)
-		}
-		defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0777)
-
-		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	})
-	t.Run("ErrViewFragmentsMkdir", func(t *testing.T) {
-		if os.Geteuid() == 0 {
-			t.Skip("Skipping permissions test since user is root.")
-		}
-		h := test.MustOpenHolder()
-		defer h.Close()
-
-		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
-			t.Fatal(err)
-		} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
-			t.Fatal(err)
-		} else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
-			t.Fatal(err)
-		} else if err := h.Holder.Close(); err != nil {
-			t.Fatal(err)
-		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil {
-			t.Fatal(err)
-		}
-		defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0777)
-
-		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
+		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=opening attrstore: opening storage: invalid database") {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
@@ -208,11 +157,9 @@ func TestHolder_Open(t *testing.T) {
 
 		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
+		} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
-		} else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
-			t.Fatal(err)
-		} else if _, err := view.SetBit(0, 0); err != nil {
+		} else if _, err := field.SetBit(0, 0, nil); err != nil {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
@@ -231,11 +178,9 @@ func TestHolder_Open(t *testing.T) {
 
 		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
+		} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
-		} else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
-			t.Fatal(err)
-		} else if _, err := view.SetBit(0, 0); err != nil {
+		} else if _, err := field.SetBit(0, 0, nil); err != nil {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
@@ -243,39 +188,11 @@ func TestHolder_Open(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: slice=0, err=opening storage: unmarshal storage") {
+		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: shard=0, err=opening storage: unmarshal storage") {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
 
-	t.Run("ErrFragmentCachePermission", func(t *testing.T) {
-		if os.Geteuid() == 0 {
-			t.Skip("Skipping permissions test since user is root.")
-		}
-		h := test.MustOpenHolder()
-		defer h.Close()
-
-		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
-			t.Fatal(err)
-		} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
-			t.Fatal(err)
-		} else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
-			t.Fatal(err)
-		} else if _, err := view.SetBit(0, 0); err != nil {
-			t.Fatal(err)
-		} else if err := view.Fragment(0).FlushCache(); err != nil {
-			t.Fatal(err)
-		} else if err := h.Holder.Close(); err != nil {
-			t.Fatal(err)
-		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil {
-			t.Fatal(err)
-		}
-		defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0666)
-
-		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	})
 }
 
 func TestHolder_HasData(t *testing.T) {
@@ -331,14 +248,8 @@ func TestHolder_DeleteIndex(t *testing.T) {
 	defer hldr.Close()
 
 	// Write bits to separate indexes.
-	f0 := hldr.MustCreateFragmentIfNotExists("i0", "f", pilosa.ViewStandard, 0)
-	if _, err := f0.SetBit(100, 200); err != nil {
-		t.Fatal(err)
-	}
-	f1 := hldr.MustCreateFragmentIfNotExists("i1", "f", pilosa.ViewStandard, 0)
-	if _, err := f1.SetBit(100, 200); err != nil {
-		t.Fatal(err)
-	}
+	hldr.SetBit("i0", "f", 100, 200)
+	hldr.SetBit("i1", "f", 100, 200)
 
 	// Ensure i0 exists.
 	if _, err := os.Stat(hldr.IndexPath("i0")); err != nil {
@@ -360,263 +271,94 @@ func TestHolder_DeleteIndex(t *testing.T) {
 
 // Ensure holder can sync with a remote holder.
 func TestHolderSyncer_SyncHolder(t *testing.T) {
-	cluster := test.NewCluster(2)
-	client := server.GetHTTPClient(nil)
-	// Create a local holder.
-	hldr0 := test.MustOpenHolder()
-	defer hldr0.Close()
-
-	// Create a remote holder wrapped by an HTTP
-	hldr1 := test.MustOpenHolder()
-	defer hldr1.Close()
-	s := test.NewServer()
-	defer s.Close()
-	s.Handler.API.Holder = hldr1.Holder
-	s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
-		e := pilosa.NewExecutor(client)
-		e.Holder = hldr1.Holder
-		e.Node = cluster.Nodes[1]
-		e.Cluster = cluster
-		return e.Execute(ctx, index, query, slices, opt)
-	}
-
-	// Mock 2-node, fully replicated cluster.
-	cluster.ReplicaN = 2
-
-	uri, err := pilosa.NewURIFromAddress(s.URL)
+	c := test.MustNewCluster(t, 2)
+	c[0].Config.Cluster.ReplicaN = 2
+	c[0].Config.AntiEntropy.Interval = 0
+	c[1].Config.Cluster.ReplicaN = 2
+	c[1].Config.AntiEntropy.Interval = 0
+	err := c.Start()
 	if err != nil {
-		t.Fatal(err)
+		t.Fatalf("starting cluster: %v", err)
+	}
+	defer c.Close()
+
+	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index i: %v", err)
+	}
+	_, err = c[0].API.CreateIndex(context.Background(), "y", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index y: %v", err)
+	}
+	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	if err != nil {
+		t.Fatalf("creating field f: %v", err)
+	}
+	_, err = c[0].API.CreateField(context.Background(), "i", "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	if err != nil {
+		t.Fatalf("creating field f0: %v", err)
+	}
+	_, err = c[0].API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	if err != nil {
+		t.Fatalf("creating field z in y: %v", err)
 	}
 
-	cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
-	cluster.Nodes[1].URI = *uri
-
-	// Create frames on nodes.
-	for _, hldr := range []*test.Holder{hldr0, hldr1} {
-		hldr.MustCreateFrameIfNotExists("i", "f")
-		hldr.MustCreateFrameIfNotExists("i", "f0")
-		hldr.MustCreateFrameIfNotExists("y", "z")
-	}
+	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
+	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
 
 	// Set data on the local holder.
-	f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
-	if _, err := f.SetBit(0, 10); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(2, 20); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(120, 10); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(200, 4); err != nil {
-		t.Fatal(err)
-	}
+	hldr0.SetBit("i", "f", 0, 10)
+	hldr0.SetBit("i", "f", 2, 20)
+	hldr0.SetBit("i", "f", 120, 10)
+	hldr0.SetBit("i", "f", 200, 4)
 
-	f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1)
-	if _, err := f.SetBit(9, SliceWidth+5); err != nil {
-		t.Fatal(err)
-	}
+	hldr0.SetBit("i", "f0", 9, ShardWidth+5)
 
-	hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0)
+	// Set a bit to create the fragment.
+	hldr0.SetBit("y", "z", 0, 0)
 
 	// Set data on the remote holder.
-	f = hldr1.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
-	if _, err := f.SetBit(0, 4000); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(3, 10); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(120, 10); err != nil {
-		t.Fatal(err)
+	hldr1.SetBit("i", "f", 0, 4000)
+	hldr1.SetBit("i", "f", 3, 10)
+	hldr1.SetBit("i", "f", 120, 10)
+
+	hldr1.SetBit("y", "z", 10, (3*ShardWidth)+4)
+	hldr1.SetBit("y", "z", 10, (3*ShardWidth)+5)
+	hldr1.SetBit("y", "z", 10, (3*ShardWidth)+7)
+
+	err = c[0].Server.SyncData()
+	if err != nil {
+		t.Fatalf("syncing node 0: %v", err)
 	}
-
-	f = hldr1.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 3)
-	if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(10, (3*SliceWidth)+7); err != nil {
-		t.Fatal(err)
-	}
-
-	// Set highest slice.
-	hldr0.Index("i").SetRemoteMaxSlice(1)
-	hldr0.Index("y").SetRemoteMaxSlice(3)
-
-	// Set up syncer.
-	syncer := pilosa.HolderSyncer{
-		Holder:       hldr0.Holder,
-		Node:         cluster.Nodes[0],
-		Cluster:      cluster,
-		RemoteClient: server.GetHTTPClient(nil),
-		Stats:        pilosa.NopStatsClient,
-	}
-
-	if err := syncer.SyncHolder(); err != nil {
-		t.Fatal(err)
+	err = c[1].Server.SyncData()
+	if err != nil {
+		t.Fatalf("syncing node 1: %v", err)
 	}
 
 	// Verify data is the same on both nodes.
 	for i, hldr := range []*test.Holder{hldr0, hldr1} {
-		f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
-		if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
-			t.Fatalf("unexpected columns(%d/0): %+v", i, a)
-		} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
-			t.Fatalf("unexpected columns(%d/2): %+v", i, a)
-		} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/3): %+v", i, a)
-		} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/120): %+v", i, a)
-		} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
-			t.Fatalf("unexpected columns(%d/200): %+v", i, a)
+		if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
+			t.Errorf("unexpected columns(%d/0): %+v", i, a)
+		}
+		if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
+			t.Errorf("unexpected columns(%d/2): %+v", i, a)
+		}
+		if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
+			t.Errorf("unexpected columns(%d/3): %+v", i, a)
+		}
+		if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
+			t.Errorf("unexpected columns(%d/120): %+v", i, a)
+		}
+		if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
+			t.Errorf("unexpected columns(%d/200): %+v", i, a)
 		}
 
-		f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
-		a := f.Row(9).Columns()
-		if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
-			t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a)
+		if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) {
+			t.Errorf("unexpected columns(%d/d/f0): %+v", i, a)
 		}
-		if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
-			t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
-		}
-		f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3)
-		if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
-			t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
-		}
-	}
-}
-
-// Ensure holder can clean up orphaned fragments.
-func TestHolderCleaner_CleanHolder(t *testing.T) {
-	cluster := test.NewCluster(2)
-
-	// Create a local holder.
-	hldr0 := test.MustOpenHolder()
-	defer hldr0.Close()
-
-	// Mock 2-node, fully replicated cluster.
-	cluster.ReplicaN = 2
-
-	cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
-
-	// Create frames on nodes.
-	for _, hldr := range []*test.Holder{hldr0} {
-		hldr.MustCreateFrameIfNotExists("i", "f")
-		hldr.MustCreateFrameIfNotExists("i", "f0")
-		hldr.MustCreateFrameIfNotExists("y", "z")
-	}
-
-	// Set data on the local holder.
-	f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
-	if _, err := f.SetBit(0, 10); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(0, 4000); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(2, 20); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(3, 10); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(120, 10); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(200, 4); err != nil {
-		t.Fatal(err)
-	}
-
-	f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1)
-	if _, err := f.SetBit(9, SliceWidth+5); err != nil {
-		t.Fatal(err)
-	}
-
-	f = hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 2)
-	if _, err := f.SetBit(10, (2*SliceWidth)+4); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(10, (2*SliceWidth)+5); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.SetBit(10, (2*SliceWidth)+7); err != nil {
-		t.Fatal(err)
-	}
-
-	// Set highest slice.
-	hldr0.Index("i").SetRemoteMaxSlice(1)
-	hldr0.Index("y").SetRemoteMaxSlice(2)
-
-	// Keep replication the same and ensure we get the expected results.
-	cluster.ReplicaN = 2
-
-	// Set up cleaner for replication 2.
-	cleaner2 := pilosa.HolderCleaner{
-		Node:    cluster.Nodes[0],
-		Holder:  hldr0.Holder,
-		Cluster: cluster,
-	}
-
-	if err := cleaner2.CleanHolder(); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify data is the same on both nodes.
-	for i, hldr := range []*test.Holder{hldr0} {
-		f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
-		if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
-			t.Fatalf("unexpected columns(%d/0): %+v", i, a)
-		} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
-			t.Fatalf("unexpected columns(%d/2): %+v", i, a)
-		} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/3): %+v", i, a)
-		} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/120): %+v", i, a)
-		} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
-			t.Fatalf("unexpected columns(%d/200): %+v", i, a)
-		}
-
-		f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
-		a := f.Row(9).Columns()
-		if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
-			t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a)
-		}
-		if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
-			t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
-		}
-		f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
-		if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
-			t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
-		}
-	}
-
-	// Change replication factor to ensure we have fragments to remove.
-	cluster.ReplicaN = 1
-
-	// Set up cleaner for replication 1.
-	cleaner1 := pilosa.HolderCleaner{
-		Node:    cluster.Nodes[0],
-		Holder:  hldr0.Holder,
-		Cluster: cluster,
-	}
-
-	if err := cleaner1.CleanHolder(); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify data is the same on both nodes.
-	for i, hldr := range []*test.Holder{hldr0} {
-		f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
-		if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
-			t.Fatalf("unexpected columns(%d/0): %+v", i, a)
-		} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
-			t.Fatalf("unexpected columns(%d/2): %+v", i, a)
-		} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/3): %+v", i, a)
-		} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/120): %+v", i, a)
-		} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
-			t.Fatalf("unexpected columns(%d/200): %+v", i, a)
-		}
-
-		f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
-		if f != nil {
-			t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f)
-		}
-
-		f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
-		if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
-			t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
+
+		if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 5, (3 * ShardWidth) + 7}) {
+			t.Errorf("unexpected columns(%d/y/z): %+v", i, a)
 		}
 	}
 }

http/client.go

@@ -0,0 +1,998 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"sort"
+	"strconv"
+
+	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/encoding/proto"
+	"github.com/pkg/errors"
+)
+
+// InternalClient represents a client to the Pilosa cluster.
+type InternalClient struct {
+	defaultURI *pilosa.URI
+	serializer pilosa.Serializer
+
+	// The client to use for HTTP communication.
+	httpClient *http.Client
+}
+
+// NewInternalClient returns a new instance of InternalClient to connect to host.
+func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, error) {
+	if host == "" {
+		return nil, pilosa.ErrHostRequired
+	}
+
+	uri, err := pilosa.NewURIFromAddress(host)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting URI")
+	}
+
+	client := NewInternalClientFromURI(uri, remoteClient)
+	return client, nil
+}
+
+func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient {
+	return &InternalClient{
+		defaultURI: defaultURI,
+		serializer: proto.Serializer{},
+		httpClient: remoteClient,
+	}
+}
+
+// MaxShardByIndex returns the number of shards on a server by index.
+func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) {
+	return c.maxShardByIndex(ctx)
+}
+
+// maxShardByIndex returns the number of shards on a server by index.
+func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64, error) {
+	// Execute request against the host.
+	u := uriPathToURL(c.defaultURI, "/internal/shards/max")
+
+	// Build request.
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req.Header.Set("Accept", "application/json")
+
+	// Execute request.
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	var rsp getShardsMaxResponse
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("http: status=%d", resp.StatusCode)
+	} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
+		return nil, fmt.Errorf("json decode: %s", err)
+	}
+
+	return rsp.Standard, nil
+}
+
+// Schema returns all index and field schema information.
+func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) {
+	// Execute request against the host.
+	u := c.defaultURI.Path("/schema")
+
+	// Build request.
+	req, err := http.NewRequest("GET", u, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req.Header.Set("Accept", "application/json")
+
+	// Execute request.
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	var rsp getSchemaResponse
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("http: status=%d", resp.StatusCode)
+	} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
+		return nil, fmt.Errorf("json decode: %s", err)
+	}
+	return rsp.Indexes, nil
+}
+
+// CreateIndex creates a new index on the server.
+func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilosa.IndexOptions) error {
+	// Encode query request.
+	buf, err := json.Marshal(&postIndexRequest{
+		Options: opt,
+	})
+	if err != nil {
+		return errors.Wrap(err, "encoding request")
+	}
+
+	// Create URL & HTTP request.
+	u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index))
+	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
+	if err != nil {
+		return errors.Wrap(err, "creating request")
+	}
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	// Read body.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return errors.Wrap(err, "reading")
+	}
+
+	// Handle response based on status code.
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return nil // ok
+	case http.StatusConflict:
+		return pilosa.ErrIndexExists
+	default:
+		return errors.New(string(body))
+	}
+}
+
+// FragmentNodes returns a list of nodes that own a shard.
+func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*pilosa.Node, error) {
+	// Execute request against the host.
+	u := uriPathToURL(c.defaultURI, "/internal/fragment/nodes")
+	u.RawQuery = (url.Values{"index": {index}, "shard": {strconv.FormatUint(shard, 10)}}).Encode()
+
+	// Build request.
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req.Header.Set("Accept", "application/json")
+
+	// Execute request.
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	var a []*pilosa.Node
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("http: status=%d", resp.StatusCode)
+	} else if err := json.NewDecoder(resp.Body).Decode(&a); err != nil {
+		return nil, fmt.Errorf("json decode: %s", err)
+	}
+
+	return a, nil
+}
+
+// Query executes query against the index.
+func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
+	return c.QueryNode(ctx, c.defaultURI, index, queryRequest)
+}
+
+// QueryNode executes query against the index, sending the request to the node specified.
+func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
+	if index == "" {
+		return nil, pilosa.ErrIndexRequired
+	} else if queryRequest.Query == "" {
+		return nil, pilosa.ErrQueryRequired
+	}
+
+	buf, err := c.serializer.Marshal(queryRequest)
+	if err != nil {
+		return nil, errors.Wrap(err, "marshaling queryRequest")
+	}
+
+	// Create HTTP request.
+	u := uri.Path(fmt.Sprintf("/index/%s/query", index))
+	req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	// Read body and unmarshal response.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading")
+	} else if resp.StatusCode != http.StatusOK {
+		return nil, errors.New(string(body))
+	}
+
+	qresp := &pilosa.QueryResponse{}
+	if err := c.serializer.Unmarshal(body, qresp); err != nil {
+		return nil, fmt.Errorf("unmarshal response: %s", err)
+	} else if qresp.Err != nil {
+		return nil, qresp.Err
+	}
+
+	return qresp, nil
+}
+
+// Import bulk imports bits for a single shard to a host.
+func (c *InternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []pilosa.Bit) error {
+	if index == "" {
+		return pilosa.ErrIndexRequired
+	} else if field == "" {
+		return pilosa.ErrFieldRequired
+	}
+
+	buf, err := c.marshalImportPayload(index, field, shard, bits)
+	if err != nil {
+		return fmt.Errorf("Error Creating Payload: %s", err)
+	}
+
+	// Retrieve a list of nodes that own the shard.
+	nodes, err := c.FragmentNodes(ctx, index, shard)
+	if err != nil {
+		return fmt.Errorf("shard nodes: %s", err)
+	}
+
+	// Import to each node.
+	for _, node := range nodes {
+		if err := c.importNode(ctx, node, index, field, buf); err != nil {
+			return fmt.Errorf("import node: host=%s, err=%s", node.URI, err)
+		}
+	}
+
+	return nil
+}
+
+// ImportK bulk imports bits specified by string keys to a host.
+func (c *InternalClient) ImportK(ctx context.Context, index, field string, columns []pilosa.Bit) error {
+	if index == "" {
+		return pilosa.ErrIndexRequired
+	} else if field == "" {
+		return pilosa.ErrFieldRequired
+	}
+
+	buf, err := c.marshalImportPayloadK(index, field, columns)
+	if err != nil {
+		return fmt.Errorf("Error Creating Payload: %s", err)
+	}
+
+	node := &pilosa.Node{
+		URI: *c.defaultURI,
+	}
+
+	// Import to node.
+	if err := c.importNode(ctx, node, index, field, buf); err != nil {
+		return fmt.Errorf("import node: host=%s, err=%s", node.URI, err)
+	}
+
+	return nil
+}
+
+func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options pilosa.IndexOptions) error {
+	err := c.CreateIndex(ctx, name, options)
+	if err == nil || errors.Cause(err) == pilosa.ErrIndexExists {
+		return nil
+	}
+	return err
+}
+
+func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error {
+	err := c.CreateField(ctx, indexName, fieldName)
+	if err == nil || errors.Cause(err) == pilosa.ErrFieldExists {
+		return nil
+	}
+	return err
+}
+
+// marshalImportPayload marshalls the import parameters into a protobuf byte slice.
+func (c *InternalClient) marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) {
+	// Separate row and column IDs to reduce allocations.
+	rowIDs := Bits(bits).RowIDs()
+	columnIDs := Bits(bits).ColumnIDs()
+	timestamps := Bits(bits).Timestamps()
+
+	// Marshal data to protobuf.
+	buf, err := c.serializer.Marshal(&pilosa.ImportRequest{
+		Index:      index,
+		Field:      field,
+		Shard:      shard,
+		RowIDs:     rowIDs,
+		ColumnIDs:  columnIDs,
+		Timestamps: timestamps,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("marshal import request: %s", err)
+	}
+	return buf, nil
+}
+
+// marshalImportPayloadK marshalls the import parameters into a protobuf byte slice.
+func (c *InternalClient) marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) {
+	// Separate row and column IDs to reduce allocations.
+	rowKeys := Bits(bits).RowKeys()
+	columnKeys := Bits(bits).ColumnKeys()
+	timestamps := Bits(bits).Timestamps()
+
+	// Marshal data to protobuf.
+	buf, err := c.serializer.Marshal(&pilosa.ImportRequest{
+		Index:      index,
+		Field:      field,
+		RowKeys:    rowKeys,
+		ColumnKeys: columnKeys,
+		Timestamps: timestamps,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("marshal import request: %s", err)
+	}
+	return buf, nil
+}
+
+// importNode sends a pre-marshaled import request to a node.
+func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, index, field string, buf []byte) error {
+	// Create URL & HTTP request.
+	path := fmt.Sprintf("/index/%s/field/%s/import", index, field)
+	u := nodePathToURL(node, path)
+	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
+	if err != nil {
+		return errors.Wrap(err, "creating request")
+	}
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	// Read body and unmarshal response.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return errors.Wrap(err, "reading")
+	} else if resp.StatusCode != http.StatusOK {
+		return errors.New(string(body))
+	}
+
+	var isresp pilosa.ImportResponse
+	if err := c.serializer.Unmarshal(body, &isresp); err != nil {
+		return fmt.Errorf("unmarshal import response: %s", err)
+	} else if s := isresp.Err; s != "" {
+		return errors.New(s)
+	}
+
+	return nil
+}
+
+// ImportValue bulk imports field values for a single shard to a host.
+func (c *InternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []pilosa.FieldValue) error {
+	if index == "" {
+		return pilosa.ErrIndexRequired
+	} else if field == "" {
+		return pilosa.ErrFieldRequired
+	}
+
+	buf, err := c.marshalImportValuePayload(index, field, shard, vals)
+	if err != nil {
+		return fmt.Errorf("Error Creating Payload: %s", err)
+	}
+
+	// Retrieve a list of nodes that own the shard.
+	nodes, err := c.FragmentNodes(ctx, index, shard)
+	if err != nil {
+		return fmt.Errorf("shard nodes: %s", err)
+	}
+
+	// Import to each node.
+	for _, node := range nodes {
+		if err := c.importNode(ctx, node, index, field, buf); err != nil {
+			return fmt.Errorf("import node: host=%s, err=%s", node.URI, err)
+		}
+	}
+
+	return nil
+}
+
+// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice.
+func (c *InternalClient) marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) {
+	// Separate row and column IDs to reduce allocations.
+	columnIDs := FieldValues(vals).ColumnIDs()
+	values := FieldValues(vals).Values()
+
+	// Marshal data to protobuf.
+	buf, err := c.serializer.Marshal(&pilosa.ImportValueRequest{
+		Index:     index,
+		Field:     field,
+		Shard:     shard,
+		ColumnIDs: columnIDs,
+		Values:    values,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("marshal import request: %s", err)
+	}
+	return buf, nil
+}
+
+// ExportCSV bulk exports data for a single shard from a host to CSV format.
+func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
+	if index == "" {
+		return pilosa.ErrIndexRequired
+	} else if field == "" {
+		return pilosa.ErrFieldRequired
+	}
+
+	// Retrieve a list of nodes that own the shard.
+	nodes, err := c.FragmentNodes(ctx, index, shard)
+	if err != nil {
+		return fmt.Errorf("shard nodes: %s", err)
+	}
+
+	// Attempt nodes in random order.
+	var e error
+	for _, i := range rand.Perm(len(nodes)) {
+		node := nodes[i]
+
+		if err := c.exportNodeCSV(ctx, node, index, field, shard, w); err != nil {
+			e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err)
+			continue
+		} else {
+			return nil
+		}
+	}
+
+	return e
+}
+
+// exportNode copies a CSV export from a node to w.
+func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, w io.Writer) error {
+	// Create URL.
+	u := nodePathToURL(node, "/export")
+	u.RawQuery = url.Values{
+		"index": {index},
+		"field": {field},
+		"shard": {strconv.FormatUint(shard, 10)},
+	}.Encode()
+
+	// Generate HTTP request.
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return errors.Wrap(err, "creating request")
+	}
+	req.Header.Set("Accept", "text/csv")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	// Validate status code.
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("invalid status: %d", resp.StatusCode)
+	}
+
+	// Copy body to writer.
+	if _, err := io.Copy(w, resp.Body); err != nil {
+		return errors.Wrap(err, "copying")
+	}
+
+	return nil
+}
+
+func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) {
+	node := &pilosa.Node{
+		URI: uri,
+	}
+	return c.backupShardNode(ctx, index, field, shard, node)
+}
+
+func (c *InternalClient) backupShardNode(ctx context.Context, index, field string, shard uint64, node *pilosa.Node) (io.ReadCloser, error) {
+	u := nodePathToURL(node, "/fragment/data")
+	u.RawQuery = url.Values{
+		"index": {index},
+		"field": {field},
+		"shard": {strconv.FormatUint(shard, 10)},
+	}.Encode()
+
+	// Build request.
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request.
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+
+	// Return error if status is not OK.
+	if resp.StatusCode == http.StatusNotFound {
+		resp.Body.Close()
+		return nil, pilosa.ErrFragmentNotFound
+	} else if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode)
+	}
+
+	return resp.Body, nil
+}
+
+// CreateField creates a new field on the server.
+func (c *InternalClient) CreateField(ctx context.Context, index, field string) error {
+	if index == "" {
+		return pilosa.ErrIndexRequired
+	}
+
+	// TODO: remove buf completely? (depends on whether importer needs to create specific field types)
+	// Encode query request.
+	buf, err := json.Marshal(&postFieldRequest{
+		//Options: opt,
+	})
+	if err != nil {
+		return errors.Wrap(err, "marshaling")
+	}
+
+	// Create URL & HTTP request.
+	u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/field/%s", index, field))
+	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
+	if err != nil {
+		return errors.Wrap(err, "creating request")
+	}
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	// Read body.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return errors.Wrap(err, "reading")
+	}
+
+	// Handle response based on status code.
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return nil // ok
+	case http.StatusConflict:
+		return pilosa.ErrFieldExists
+	default:
+		return errors.New(string(body))
+	}
+}
+
+// FragmentBlocks returns a list of block checksums for a fragment on a host.
+// Only returns blocks which contain data.
+func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64) ([]pilosa.FragmentBlock, error) {
+	if uri == nil {
+		uri = c.defaultURI
+	}
+	u := uriPathToURL(uri, "/internal/fragment/blocks")
+	u.RawQuery = url.Values{
+		"index": {index},
+		"field": {field},
+		"shard": {strconv.FormatUint(shard, 10)},
+	}.Encode()
+
+	// Build request.
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req.Header.Set("Accept", "application/json")
+
+	// Execute request.
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	// Return error if status is not OK.
+	switch resp.StatusCode {
+	case http.StatusOK: // ok
+	case http.StatusNotFound:
+		return nil, pilosa.ErrFragmentNotFound
+	default:
+		return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode)
+	}
+
+	// Decode response object.
+	var rsp getFragmentBlocksResponse
+	if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
+		return nil, errors.Wrap(err, "decoding")
+	}
+	return rsp.Blocks, nil
+}
+
+// BlockData returns row/column id pairs for a block.
+func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) {
+	if uri == nil {
+		panic("need to pass a URI to BlockData")
+	}
+	buf, err := c.serializer.Marshal(&pilosa.BlockDataRequest{
+		Index: index,
+		Field: field,
+		Shard: shard,
+		Block: uint64(block),
+	})
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "marshaling")
+	}
+
+	u := uriPathToURL(uri, "/internal/fragment/block/data")
+	req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf))
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "creating request")
+	}
+	req.Header.Set("Content-Type", "application/protobuf")
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Accept", "application/protobuf")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	// Return error if status is not OK.
+	switch resp.StatusCode {
+	case http.StatusOK: // fallthrough
+	case http.StatusNotFound:
+		return nil, nil, nil
+	default:
+		return nil, nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode)
+	}
+
+	// Decode response object.
+	var rsp pilosa.BlockDataResponse
+	if body, err := ioutil.ReadAll(resp.Body); err != nil {
+		return nil, nil, errors.Wrap(err, "reading")
+	} else if err := c.serializer.Unmarshal(body, &rsp); err != nil {
+		return nil, nil, errors.Wrap(err, "unmarshalling")
+	}
+	return rsp.RowIDs, rsp.ColumnIDs, nil
+}
+
+// ColumnAttrDiff returns data from differing blocks on a remote host.
+func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) {
+	if uri == nil {
+		uri = c.defaultURI
+	}
+	u := uriPathToURL(uri, fmt.Sprintf("/internal/index/%s/attr/diff", index))
+
+	// Encode request.
+	buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks})
+	if err != nil {
+		return nil, errors.Wrap(err, "marshaling")
+	}
+
+	// Build request.
+	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req.Header.Set("Accept", "application/json")
+
+	// Execute request.
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	// Return error if status is not OK.
+	switch resp.StatusCode {
+	case http.StatusOK: // ok
+	default:
+		return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode)
+	}
+
+	// Decode response object.
+	var rsp postIndexAttrDiffResponse
+	if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
+		return nil, errors.Wrap(err, "decoding")
+	}
+	return rsp.Attrs, nil
+}
+
+// RowAttrDiff returns data from differing blocks on a remote host.
+func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) {
+	if uri == nil {
+		uri = c.defaultURI
+	}
+	u := uriPathToURL(uri, fmt.Sprintf("/internal/index/%s/field/%s/attr/diff", index, field))
+
+	// Encode request.
+	buf, err := json.Marshal(postFieldAttrDiffRequest{Blocks: blks})
+	if err != nil {
+		return nil, errors.Wrap(err, "marshaling")
+	}
+
+	// Build request.
+	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req.Header.Set("Accept", "application/json")
+
+	// Execute request.
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	// Return error if status is not OK.
+	switch resp.StatusCode {
+	case http.StatusOK: // ok
+	case http.StatusNotFound:
+		return nil, pilosa.ErrFieldNotFound
+	default:
+		return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode)
+	}
+
+	// Decode response object.
+	var rsp postFieldAttrDiffResponse
+	if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
+		return nil, errors.Wrap(err, "decoding")
+	}
+	return rsp.Attrs, nil
+}
+
+// SendMessage posts a message synchronously.
+func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg []byte) error {
+	u := uriPathToURL(uri, "/internal/cluster/message")
+	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg))
+	if err != nil {
+		return errors.Wrap(err, "making new request")
+	}
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req.Header.Set("Accept", "application/json")
+
+	// Execute request.
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return fmt.Errorf("executing http request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// Read body.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("reading response body: %v", err)
+	}
+
+	// Return error if status is not OK.
+	switch resp.StatusCode {
+	case http.StatusOK: // ok
+	default:
+		return fmt.Errorf("unexpected response status code: %d: %s", resp.StatusCode, body)
+	}
+
+	return nil
+}
+
+// Bits is a slice of Bit.
+type Bits []pilosa.Bit
+
+func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+func (p Bits) Len() int      { return len(p) }
+
+func (p Bits) Less(i, j int) bool {
+	if p[i].RowID == p[j].RowID {
+		if p[i].ColumnID < p[j].ColumnID {
+			return p[i].Timestamp < p[j].Timestamp
+		}
+		return p[i].ColumnID < p[j].ColumnID
+	}
+	return p[i].RowID < p[j].RowID
+}
+
+// RowIDs returns a slice of all the row IDs.
+func (p Bits) RowIDs() []uint64 {
+	other := make([]uint64, len(p))
+	for i := range p {
+		other[i] = p[i].RowID
+	}
+	return other
+}
+
+// ColumnIDs returns a slice of all the column IDs.
+func (p Bits) ColumnIDs() []uint64 {
+	other := make([]uint64, len(p))
+	for i := range p {
+		other[i] = p[i].ColumnID
+	}
+	return other
+}
+
+// RowKeys returns a slice of all the row keys.
+func (p Bits) RowKeys() []string {
+	other := make([]string, len(p))
+	for i := range p {
+		other[i] = p[i].RowKey
+	}
+	return other
+}
+
+// ColumnKeys returns a slice of all the column keys.
+func (p Bits) ColumnKeys() []string {
+	other := make([]string, len(p))
+	for i := range p {
+		other[i] = p[i].ColumnKey
+	}
+	return other
+}
+
+// Timestamps returns a slice of all the timestamps.
+func (p Bits) Timestamps() []int64 {
+	other := make([]int64, len(p))
+	for i := range p {
+		other[i] = p[i].Timestamp
+	}
+	return other
+}
+
+// GroupByShard returns a map of bits by shard.
+func (p Bits) GroupByShard() map[uint64][]pilosa.Bit {
+	m := make(map[uint64][]pilosa.Bit)
+	for _, bit := range p {
+		shard := bit.ColumnID / pilosa.ShardWidth
+		m[shard] = append(m[shard], bit)
+	}
+
+	for shard, bits := range m {
+		sort.Sort(Bits(bits))
+		m[shard] = bits
+	}
+
+	return m
+}
+
+// FieldValues represents a slice of field values.
+type FieldValues []pilosa.FieldValue
+
+func (p FieldValues) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+func (p FieldValues) Len() int      { return len(p) }
+
+func (p FieldValues) Less(i, j int) bool {
+	return p[i].ColumnID < p[j].ColumnID
+}
+
+// ColumnIDs returns a slice of all the column IDs.
+func (p FieldValues) ColumnIDs() []uint64 {
+	other := make([]uint64, len(p))
+	for i := range p {
+		other[i] = p[i].ColumnID
+	}
+	return other
+}
+
+// Values returns a slice of all the values.
+func (p FieldValues) Values() []int64 {
+	other := make([]int64, len(p))
+	for i := range p {
+		other[i] = p[i].Value
+	}
+	return other
+}
+
+// GroupByShard returns a map of field values by shard.
+func (p FieldValues) GroupByShard() map[uint64][]pilosa.FieldValue {
+	m := make(map[uint64][]pilosa.FieldValue)
+	for _, val := range p {
+		shard := val.ColumnID / pilosa.ShardWidth
+		m[shard] = append(m[shard], val)
+	}
+
+	for shard, vals := range m {
+		sort.Sort(FieldValues(vals))
+		m[shard] = vals
+	}
+
+	return m
+}
+
+// BitsByPos is a slice of bits sorted row then column.
+type BitsByPos []pilosa.Bit
+
+func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+func (p BitsByPos) Len() int      { return len(p) }
+func (p BitsByPos) Less(i, j int) bool {
+	p0, p1 := pos(p[i].RowID, p[i].ColumnID), pos(p[j].RowID, p[j].ColumnID)
+	if p0 == p1 {
+		return p[i].Timestamp < p[j].Timestamp
+	}
+	return p0 < p1
+}
+
+// pos returns the row position of a row/column pair.
+func pos(rowID, columnID uint64) uint64 {
+	return (rowID * pilosa.ShardWidth) + (columnID % pilosa.ShardWidth)
+}
+
+func uriPathToURL(uri *pilosa.URI, path string) url.URL {
+	return url.URL{
+		Scheme: uri.Scheme,
+		Host:   uri.HostPort(),
+		Path:   path,
+	}
+}
+
+func nodePathToURL(node *pilosa.Node, path string) url.URL {
+	return url.URL{
+		Scheme: node.URI.Scheme,
+		Host:   node.URI.HostPort(),
+		Path:   path,
+	}
+}

http/client_test.go

@@ -0,0 +1,322 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http_test
+
+import (
+	"context"
+	"fmt"
+	gohttp "net/http"
+	"reflect"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/http"
+	"github.com/pilosa/pilosa/pql"
+	"github.com/pilosa/pilosa/server"
+	"github.com/pilosa/pilosa/test"
+)
+
+var defaultClient *gohttp.Client
+
+func init() {
+	defaultClient = http.GetHTTPClient(nil)
+
+}
+
+// Test distributed TopN Row count across 3 nodes.
+func TestClient_MultiNode(t *testing.T) {
+	c := test.MustRunCluster(t, 3,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&test.ModHasher{}))},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&test.ModHasher{}))},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(pilosa.OptServerNodeID("node2"), pilosa.OptServerClusterHasher(&test.ModHasher{}))},
+	)
+	defer c.Close()
+
+	hldr := []test.Holder{}
+	for _, command := range c {
+		hldr = append(hldr, test.Holder{Holder: command.Server.Holder()})
+	}
+
+	// Create a dispersed set of bitmaps across 3 nodes such that each individual node and shard width increment would reveal a different TopN.
+	shardNums := []uint64{1, 2, 6}
+
+	// This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsShards("i", 20, s[i].HostURI())`
+	owns := [][]uint64{
+		{1, 3, 4, 8, 10, 13, 17, 19},
+		{2, 5, 7, 11, 12, 14, 18},
+		{0, 6, 9, 15, 16, 20},
+	}
+
+	for i, num := range shardNums {
+		ownsNum := false
+		for _, ownNum := range owns[i] {
+			if ownNum == num {
+				ownsNum = true
+				break
+			}
+		}
+		if !ownsNum {
+			t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, c[i].URL(), owns)
+		}
+	}
+
+	baseBit0 := pilosa.ShardWidth * shardNums[0]
+	baseBit1 := pilosa.ShardWidth * shardNums[1]
+	baseBit2 := pilosa.ShardWidth * shardNums[2]
+
+	maxShard := uint64(0)
+	for _, x := range shardNums {
+		if x > maxShard {
+			maxShard = x
+		}
+	}
+	_, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+	if err != nil {
+		t.Fatalf("creating field: %v", err)
+	}
+
+	hldr[0].MustSetBits("i", "f", 100, baseBit0+10)
+	hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12)
+	hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15)
+	hldr[0].MustSetBits("i", "f", 2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4)
+	hldr[0].MustSetBits("i", "f", 3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5)
+	hldr[0].MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2)
+
+	hldr[1].MustSetBits("i", "f", 99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4)
+	hldr[1].MustSetBits("i", "f", 100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10)
+	hldr[1].MustSetBits("i", "f", 98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6)
+	hldr[1].MustSetBits("i", "f", 1, baseBit1+4)
+	hldr[1].MustSetBits("i", "f", 22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5)
+
+	hldr[2].MustSetBits("i", "f", 24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14)
+	hldr[2].MustSetBits("i", "f", 20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13)
+	hldr[2].MustSetBits("i", "f", 21, baseBit2+10)
+	hldr[2].MustSetBits("i", "f", 100, baseBit2+10)
+	hldr[2].MustSetBits("i", "f", 99, baseBit2+10, baseBit2+11, baseBit2+12)
+	hldr[2].MustSetBits("i", "f", 98, baseBit2+10, baseBit2+11)
+	hldr[2].MustSetBits("i", "f", 22, baseBit2+10, baseBit2+11, baseBit2+12)
+
+	// Rebuild the RankCache.
+	// We have to do this to avoid the 10-second cache invalidation delay
+	// built into cache.Invalidate()
+	c[0].RecalculateCaches()
+	c[1].RecalculateCaches()
+	c[2].RecalculateCaches()
+
+	// Connect to each node to compare results.
+	client := make([]*Client, 3)
+	client[0] = MustNewClient(c[0].URL(), defaultClient)
+	client[1] = MustNewClient(c[1].URL(), defaultClient)
+	client[2] = MustNewClient(c[2].URL(), defaultClient)
+
+	topN := 4
+	queryRequest := &pilosa.QueryRequest{
+		Query:  fmt.Sprintf(`TopN(f, n=%d)`, topN),
+		Remote: false,
+	}
+
+	result, err := client[0].Query(context.Background(), "i", queryRequest)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	result, err = client[0].Query(context.Background(), "i", queryRequest)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Test must return exactly N results.
+	if len(result.Results[0].([]pilosa.Pair)) != topN {
+		t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result))
+	}
+	p := []pilosa.Pair{
+		{ID: 100, Count: 12},
+		{ID: 22, Count: 10},
+		{ID: 98, Count: 8},
+		{ID: 99, Count: 7}}
+
+	// Valdidate the Top 4 result counts.
+	if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) {
+		t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
+	}
+
+	result1, err := client[1].Query(context.Background(), "i", queryRequest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	result2, err := client[2].Query(context.Background(), "i", queryRequest)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Compare TopN results across all nodes in the cluster.
+	if !reflect.DeepEqual(result, result1) {
+		t.Fatalf("TopN result should be the same on node0 and node1: %s", spew.Sdump(result1))
+	}
+
+	if !reflect.DeepEqual(result, result2) {
+		t.Fatalf("TopN result should be the same on node0 and node2: %s", spew.Sdump(result2))
+	}
+}
+
+// Ensure client can bulk import data.
+func TestClient_Import(t *testing.T) {
+	cmd := test.MustRunCluster(t, 1)[0]
+	host := cmd.URL()
+	holder := cmd.Server.Holder()
+	hldr := test.Holder{Holder: holder}
+
+	// Load bitmap into cache to ensure cache gets updated.
+	hldr.SetBit("i", "f", 1, 0) // set a bit so the view gets created.
+	hldr.Row("i", "f", 0)
+
+	// Send import request.
+	c := MustNewClient(host, defaultClient)
+	if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
+		{RowID: 0, ColumnID: 1},
+		{RowID: 0, ColumnID: 5},
+		{RowID: 200, ColumnID: 6},
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify data.
+	if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) {
+		t.Fatalf("unexpected columns: %+v", a)
+	}
+	if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{6}) {
+		t.Fatalf("unexpected columns: %+v", a)
+	}
+}
+
+// Ensure client can bulk import value data.
+func TestClient_ImportValue(t *testing.T) {
+	cmd := test.MustRunCluster(t, 1)[0]
+	host := cmd.URL()
+	holder := cmd.Server.Holder()
+	hldr := test.Holder{Holder: holder}
+
+	fldName := "f"
+
+	// Load bitmap into cache to ensure cache gets updated.
+	index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
+	field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Send import request.
+	c := MustNewClient(host, defaultClient)
+	if err := c.ImportValue(context.Background(), "i", "f", 0, []pilosa.FieldValue{
+		{ColumnID: 1, Value: -10},
+		{ColumnID: 2, Value: 20},
+		{ColumnID: 3, Value: 40},
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify Sum.
+	sum, cnt, err := field.Sum(nil, fldName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if sum != 50 || cnt != 3 {
+		t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt)
+	}
+
+	// Verify Min.
+	min, cnt, err := field.Min(nil, fldName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if min != -10 || cnt != 1 {
+		t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-10, cnt=1", min, cnt)
+	}
+
+	// Verify Min with Filter.
+	filter, err := field.Range(fldName, pql.GT, 40)
+	if err != nil {
+		t.Fatal(err)
+	}
+	min, cnt, err = field.Min(filter, fldName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if min != -100 || cnt != 0 {
+		t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt)
+	}
+
+	// Verify Max.
+	max, cnt, err := field.Max(nil, fldName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if max != 40 || cnt != 1 {
+		t.Fatalf("unexpected values: got max=%v, count=%v; expected max=40, cnt=1", max, cnt)
+	}
+}
+
+// Ensure client can retrieve a list of all checksums for blocks in a fragment.
+func TestClient_FragmentBlocks(t *testing.T) {
+	cmd := test.MustRunCluster(t, 1)[0]
+	holder := cmd.Server.Holder()
+	hldr := test.Holder{Holder: holder}
+
+	hldr.SetBit("i", "f", 0, 1)
+	hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100)
+
+	// Set a bit on a different shard.
+	hldr.SetBit("i", "f", 0, 1)
+	c := MustNewClient(cmd.URL(), defaultClient)
+	blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0)
+	if err != nil {
+		t.Fatal(err)
+	} else if len(blocks) != 2 {
+		t.Fatalf("unexpected blocks: %s", spew.Sdump(blocks))
+	} else if blocks[0].ID != 0 {
+		t.Fatalf("unexpected block id(0): %d", blocks[0].ID)
+	} else if blocks[1].ID != 3 {
+		t.Fatalf("unexpected block id(1): %d", blocks[1].ID)
+	}
+
+	// Verify data matches local blocks.
+	if a, err := cmd.API.FragmentBlocks(context.Background(), "i", "f", 0); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(a, blocks) {
+		t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks))
+	}
+}
+
+// Client represents a test wrapper for pilosa.Client.
+type Client struct {
+	*http.InternalClient
+}
+
+// MustNewClient returns a new instance of Client. Panic on error.
+func MustNewClient(host string, h *gohttp.Client) *Client {
+	c, err := http.NewInternalClient(host, h)
+	if err != nil {
+		panic(err)
+	}
+	return &Client{InternalClient: c}
+}

http/error.go

@@ -12,4 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package test
+package http
+
+// Error defines a standard application error.
+type Error struct {
+	// Machine-readable error code.
+	code string `json:"code,omitempty"`
+
+	// Human-readable message.
+	Message string `json:"message"`
+}
+
+// Error returns the string representation of the error message.
+func (e *Error) Error() string {
+	return e.Message
+}

http/handler_internal_test.go

@@ -0,0 +1,180 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http
+
+import (
+	"bytes"
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"github.com/pilosa/pilosa"
+)
+
+// Test custom UnmarshalJSON for postIndexRequest object
+func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
+	tests := []struct {
+		json     string
+		expected postIndexRequest
+		err      string
+	}{
+		{json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{}}},
+		{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
+		{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
+		{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
+	}
+	for _, test := range tests {
+		actual := &postIndexRequest{}
+		err := json.Unmarshal([]byte(test.json), actual)
+
+		if err != nil {
+			if test.err == "" || test.err != err.Error() {
+				t.Errorf("expected error: %v, but got result: %v", test.err, err)
+			}
+		} else {
+			if test.err != "" {
+				t.Errorf("expected error: %v, but got no error", test.err)
+			}
+		}
+
+		if test.err == "" {
+			if !reflect.DeepEqual(*actual, test.expected) {
+				t.Errorf("expected: %v, but got: %v", test.expected, *actual)
+			}
+		}
+
+	}
+}
+
+// Test custom UnmarshalJSON for postFieldRequest object
+func TestPostFieldRequestUnmarshalJSON(t *testing.T) {
+	foo := "foo"
+	tests := []struct {
+		json     string
+		expected postFieldRequest
+		err      string
+	}{
+		{json: `{"options": {}}`, expected: postFieldRequest{}},
+		{json: `{"options": 4}`, err: "json: cannot unmarshal number into Go struct field postFieldRequest.options of type http.fieldOptions"},
+		{json: `{"option": {}}`, err: `json: unknown field "option"`},
+		{json: `{"options": {"badKey": "test"}}`, err: `json: unknown field "badKey"`},
+		{json: `{"options": {"inverseEnabled": true}}`, err: `json: unknown field "inverseEnabled"`},
+		{json: `{"options": {"cacheType": "foo"}}`, expected: postFieldRequest{Options: fieldOptions{CacheType: &foo}}},
+		{json: `{"options": {"inverse": true, "cacheType": "foo"}}`, err: `json: unknown field "inverse"`},
+	}
+	for i, test := range tests {
+		actual := &postFieldRequest{}
+		dec := json.NewDecoder(bytes.NewReader([]byte(test.json)))
+		dec.DisallowUnknownFields()
+		err := dec.Decode(actual)
+		if err != nil {
+			if test.err == "" || test.err != err.Error() {
+				t.Errorf("test %d: expected error: %v, but got result: %v", i, test.err, err)
+			}
+		}
+
+		if test.err == "" {
+			if !reflect.DeepEqual(*actual, test.expected) {
+				t.Errorf("test %d: expected: %v, but got: %v", i, test.expected, *actual)
+			}
+		}
+
+	}
+}
+
+func stringPtr(s string) *string {
+	return &s
+}
+
+func int64Ptr(i int64) *int64 {
+	return &i
+}
+
+// Test fieldOption validation.
+func TestFieldOptionValidation(t *testing.T) {
+	timeQuantum := pilosa.TimeQuantum("YMD")
+	defaultCacheSize := uint32(pilosa.DefaultCacheSize)
+	tests := []struct {
+		json     string
+		expected postFieldRequest
+		err      string
+	}{
+		// FieldType: Set
+		{json: `{"options": {}}`, expected: postFieldRequest{Options: fieldOptions{
+			Type:      pilosa.FieldTypeSet,
+			CacheType: stringPtr(pilosa.DefaultCacheType),
+			CacheSize: &defaultCacheSize,
+		}}},
+		{json: `{"options": {"type": "set"}}`, expected: postFieldRequest{Options: fieldOptions{
+			Type:      pilosa.FieldTypeSet,
+			CacheType: stringPtr(pilosa.DefaultCacheType),
+			CacheSize: &defaultCacheSize,
+		}}},
+		{json: `{"options": {"type": "set", "cacheType": "lru"}}`, expected: postFieldRequest{Options: fieldOptions{
+			Type:      pilosa.FieldTypeSet,
+			CacheType: stringPtr("lru"),
+			CacheSize: &defaultCacheSize,
+		}}},
+		{json: `{"options": {"type": "set", "min": 0}}`, err: "min does not apply to field type set"},
+		{json: `{"options": {"type": "set", "max": 100}}`, err: "max does not apply to field type set"},
+		{json: `{"options": {"type": "set", "timeQuantum": "YMD"}}`, err: "timeQuantum does not apply to field type set"},
+
+		// FieldType: Int
+		{json: `{"options": {"type": "int"}}`, err: "min is required for field type int"},
+		{json: `{"options": {"type": "int", "min": 0}}`, err: "max is required for field type int"},
+		{json: `{"options": {"type": "int", "min": 0, "max": 1000}}`, expected: postFieldRequest{Options: fieldOptions{
+			Type: pilosa.FieldTypeInt,
+			Min:  int64Ptr(0),
+			Max:  int64Ptr(1000),
+		}}},
+		{json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheType": "ranked"}}`, err: "cacheType does not apply to field type int"},
+		{json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheSize": 1000}}`, err: "cacheSize does not apply to field type int"},
+		{json: `{"options": {"type": "int", "min": 0, "max": 1000, "timeQuantum": "YMD"}}`, err: "timeQuantum does not apply to field type int"},
+
+		// FieldType: Time
+		{json: `{"options": {"type": "time"}}`, err: "timeQuantum is required for field type time"},
+		{json: `{"options": {"type": "time", "timeQuantum": "YMD"}}`, expected: postFieldRequest{Options: fieldOptions{
+			Type:        pilosa.FieldTypeTime,
+			TimeQuantum: &timeQuantum,
+		}}},
+		{json: `{"options": {"type": "time", "timeQuantum": "YMD", "min": 0}}`, err: "min does not apply to field type time"},
+		{json: `{"options": {"type": "time", "timeQuantum": "YMD", "max": 1000}}`, err: "max does not apply to field type time"},
+		{json: `{"options": {"type": "time", "timeQuantum": "YMD", "cacheType": "ranked"}}`, err: "cacheType does not apply to field type time"},
+		{json: `{"options": {"type": "time", "timeQuantum": "YMD", "cacheSize": 1000}}`, err: "cacheSize does not apply to field type time"},
+	}
+	for i, test := range tests {
+		actual := &postFieldRequest{}
+		dec := json.NewDecoder(bytes.NewReader([]byte(test.json)))
+		dec.DisallowUnknownFields()
+		err := dec.Decode(actual)
+		if err != nil {
+			t.Errorf("test %d: %v", i, err)
+		}
+
+		// Validate field options.
+		if err := actual.Options.validate(); err != nil {
+			if test.err == "" || test.err != err.Error() {
+				t.Errorf("test %d: expected error: %v, but got result: %v", i, test.err, err)
+			}
+		}
+
+		if test.err == "" {
+			if !reflect.DeepEqual(*actual, test.expected) {
+				t.Errorf("test %d: expected: %v, but got: %v", i, test.expected, *actual)
+			}
+		}
+
+	}
+}

http/handler_test.go

@@ -11,27 +11,32 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-//go:generate statik -src=../webui -dest=..
-//
-// Package statik contains static assets for the Web UI. `go generate` or
-// `make generate-statik` will produce statik.go, which is ignored by git.
-package statik
+
+package http_test
 
 import (
-	"net/http"
+	"net"
+	"testing"
 
 	"github.com/pilosa/pilosa"
-	"github.com/rakyll/statik/fs"
+	"github.com/pilosa/pilosa/http"
 )
 
-// Ensure nopFileSystem implements interface.
-var _ pilosa.FileSystem = &FileSystem{}
-
-// FileSystem represents a static FileSystem.
-type FileSystem struct{}
-
-// New is a statik implementation of FileSystem New method.
-func (s *FileSystem) New() (http.FileSystem, error) {
-	return fs.New()
+func TestHandlerOptions(t *testing.T) {
+	_, err := http.NewHandler()
+	if err == nil {
+		t.Fatalf("expected error making handler without options, got nil")
+	}
+	_, err = http.NewHandler(http.OptHandlerAPI(&pilosa.API{}))
+	if err == nil {
+		t.Fatalf("expected error making handler without options, got nil")
+	}
+	ln, err := net.Listen("tcp", ":0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = http.NewHandler(http.OptHandlerListener(ln))
+	if err == nil {
+		t.Fatalf("expected error making handler without options, got nil")
+	}
 }

http/translator.go

@@ -0,0 +1,87 @@
+package http
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+
+	"github.com/pilosa/pilosa"
+)
+
+// Ensure implementation implements inteface.
+var _ pilosa.TranslateStore = (*translateStore)(nil)
+
+// translateStore represents an implementation of translateStore that
+// communicates over HTTP. This is used with the TranslateHandler.
+type translateStore struct {
+	URL string
+}
+
+// NewTranslateStore returns a new instance of TranslateStore.
+func NewTranslateStore(rawurl string) *translateStore {
+	return &translateStore{URL: rawurl}
+}
+
+// TranslateColumnsToUint64 is not currently implemented.
+func (s *translateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) {
+	return nil, pilosa.ErrNotImplemented
+}
+
+// TranslateColumnToString is not currently implemented.
+func (s *translateStore) TranslateColumnToString(index string, values uint64) (string, error) {
+	return "", pilosa.ErrNotImplemented
+}
+
+// TranslateRowsToUint64 is not currently implemented.
+func (s *translateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
+	return nil, pilosa.ErrNotImplemented
+}
+
+// TranslateRowToString is not currently implemented.
+func (s *translateStore) TranslateRowToString(index, frame string, values uint64) (string, error) {
+	return "", pilosa.ErrNotImplemented
+}
+
+// Reader returns a reader that can stream data from a remote store.
+func (s *translateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) {
+	// Generate remote URL.
+	u, err := url.Parse(s.URL)
+	if err != nil {
+		return nil, err
+	}
+	u.Path = "/internal/translate/data"
+	u.RawQuery = (url.Values{
+		"offset": {strconv.FormatInt(off, 10)},
+	}).Encode()
+
+	// Connect a stream to the remote server.
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(ctx)
+
+	// Connect a stream to the remote server.
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("http: cannot connect to translate store endpoint: %s", err)
+	}
+
+	// Handle error codes or return body as stream.
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return resp.Body, nil
+	case http.StatusNotImplemented:
+		resp.Body.Close()
+		return nil, pilosa.ErrNotImplemented
+	default:
+		body, _ := ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+		return nil, fmt.Errorf("http: invalid translate store endpoint status: code=%d url=%s body=%q", resp.StatusCode, u.String(), bytes.TrimSpace(body))
+	}
+}

http/translator_test.go

@@ -0,0 +1,163 @@
+package http_test
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	gohttp "net/http"
+	"testing"
+	"time"
+
+	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/http"
+	"github.com/pilosa/pilosa/mock"
+	"github.com/pilosa/pilosa/server"
+	"github.com/pilosa/pilosa/test"
+)
+
+func newMockReadCloser() *mock.ReadCloser {
+	return &mock.ReadCloser{
+		ReadFunc: func(p []byte) (int, error) {
+			return 0, io.EOF
+		},
+		CloseFunc: func() error {
+			return nil
+		},
+	}
+}
+
+func TestTranslateStore_Reader(t *testing.T) {
+	// Ensure client can connect and stream the translate store data.
+	t.Run("OK", func(t *testing.T) {
+		t.Run("ServerDisconnect", func(t *testing.T) {
+			var mrc mock.ReadCloser
+			var readN int
+			mrc.ReadFunc = func(p []byte) (int, error) {
+				readN++
+				switch readN {
+				case 1:
+					copy(p, []byte("foo"))
+					return 3, nil
+				case 2:
+					copy(p, []byte("barbaz"))
+					return 6, nil
+				case 3:
+					return 0, io.EOF
+				default:
+					t.Fatal("unexpected read")
+					return 0, nil
+				}
+			}
+			closeInvoked := make(chan struct{})
+			mrc.CloseFunc = func() error {
+				close(closeInvoked)
+				return nil
+			}
+
+			// Setup handler on test server.
+			var translateStore mock.TranslateStore
+
+			translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
+				// Check context to make sure this is the call we are looking for.
+				// (Something else calls ReaderFunc on server startup)
+				if ctx.Value(gohttp.ServerContextKey) != nil {
+					if off != 100 {
+						t.Fatalf("unexpected off: %d", off)
+					}
+					return &mrc, nil
+				}
+				return newMockReadCloser(), nil
+			}
+
+			opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
+			main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
+
+			defer main.Close()
+
+			// Connect to server and stream all available data.
+			store := http.NewTranslateStore(main.URL())
+
+			rc, err := store.Reader(context.Background(), 100)
+			if err != nil {
+				t.Fatal(err)
+			} else if data, err := ioutil.ReadAll(rc); err != nil {
+				t.Fatal(err)
+			} else if string(data) != `foobarbaz` {
+				t.Fatalf("unexpected data: %q", data)
+			} else if err := rc.Close(); err != nil {
+				t.Fatal(err)
+			}
+
+			select {
+			case <-time.NewTimer(time.Millisecond * 100).C:
+				t.Fatal("expected server close")
+			case <-closeInvoked:
+				return
+			}
+		})
+
+		// Ensure server closes store reader if client disconnects.
+		t.Run("ClientDisconnect", func(t *testing.T) {
+			// Setup mock so that Read() hangs.
+			done := make(chan struct{})
+
+			var mrc mock.ReadCloser
+			mrc.ReadFunc = func(p []byte) (int, error) {
+				<-done
+				return 0, io.EOF
+			}
+
+			closeInvoked := make(chan struct{})
+
+			mrc.CloseFunc = func() error {
+				close(closeInvoked)
+				return nil
+			}
+
+			var translateStore mock.TranslateStore
+
+			translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
+				return &mrc, nil
+			}
+
+			opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
+			main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
+
+			defer main.Close()
+			defer close(done)
+
+			// Connect to server and begin streaming.
+			ctx, cancel := context.WithCancel(context.Background())
+			store := http.NewTranslateStore(main.URL())
+			if _, err := store.Reader(ctx, 0); err != nil {
+				t.Fatal(err)
+			}
+
+			// Cancel the context and check if server is closed.
+			cancel()
+			select {
+			case <-time.NewTimer(time.Millisecond * 100).C:
+				t.Fatal("expected server close")
+			case <-closeInvoked:
+				return
+			}
+		})
+	})
+
+	// Ensure client is notified if the server doesn't support streaming replication.
+	t.Run("ErrNotImplemented", func(t *testing.T) {
+		var translateStore mock.TranslateStore
+		translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
+			return nil, pilosa.ErrNotImplemented
+		}
+
+		opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
+		main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
+		defer main.Close()
+
+		_, err := http.NewTranslateStore(main.URL()).Reader(context.Background(), 0)
+		if err != pilosa.ErrNotImplemented {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+}

index.go

@@ -28,32 +28,33 @@ import (
 	"github.com/pkg/errors"
 )
 
-// Index represents a container for frames.
+// Index represents a container for fields.
 type Index struct {
 	mu   sync.RWMutex
 	path string
 	name string
+	keys bool // use string keys
 
-	// Frames by name.
-	frames map[string]*Frame
+	// Fields by name.
+	fields map[string]*Field
 
-	// Max Slice on any node in the cluster, according to this node.
-	remoteMaxSlice uint64
+	// Max shard on any node in the cluster, according to this node.
+	remoteMaxShard uint64
 
-	NewAttrStore func(string) AttrStore
+	newAttrStore func(string) AttrStore
 
 	// Column attribute storage and cache.
-	columnAttrStore AttrStore
+	columnAttrs AttrStore
 
-	broadcaster Broadcaster
+	broadcaster broadcaster
 	Stats       StatsClient
 
-	Logger Logger
+	logger Logger
 }
 
 // NewIndex returns a new instance of Index.
 func NewIndex(path, name string) (*Index, error) {
-	err := ValidateName(name)
+	err := validateName(name)
 	if err != nil {
 		return nil, errors.Wrap(err, "validating name")
 	}
@@ -61,16 +62,16 @@ func NewIndex(path, name string) (*Index, error) {
 	return &Index{
 		path:   path,
 		name:   name,
-		frames: make(map[string]*Frame),
+		fields: make(map[string]*Field),
 
-		remoteMaxSlice: 0,
+		remoteMaxShard: 0,
 
-		NewAttrStore:    NewNopAttrStore,
-		columnAttrStore: NopAttrStore,
+		newAttrStore: newNopAttrStore,
+		columnAttrs:  nopStore,
 
 		broadcaster: NopBroadcaster,
 		Stats:       NopStatsClient,
-		Logger:      NopLogger,
+		logger:      NopLogger,
 	}, nil
 }
 
@@ -80,8 +81,11 @@ func (i *Index) Name() string { return i.name }
 // Path returns the path the index was initialized with.
 func (i *Index) Path() string { return i.path }
 
+// Keys returns true if the index uses string keys.
+func (i *Index) Keys() bool { return i.keys }
+
 // ColumnAttrStore returns the storage for column attributes.
-func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore }
+func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrs }
 
 // Options returns all options for this index.
 func (i *Index) Options() IndexOptions {
@@ -106,19 +110,19 @@ func (i *Index) Open() error {
 		return errors.Wrap(err, "loading meta file")
 	}
 
-	if err := i.openFrames(); err != nil {
-		return errors.Wrap(err, "opening frames")
+	if err := i.openFields(); err != nil {
+		return errors.Wrap(err, "opening fields")
 	}
 
-	if err := i.columnAttrStore.Open(); err != nil {
+	if err := i.columnAttrs.Open(); err != nil {
 		return errors.Wrap(err, "opening attrstore")
 	}
 
 	return nil
 }
 
-// openFrames opens and initializes the frames inside the index.
-func (i *Index) openFrames() error {
+// openFields opens and initializes the fields inside the index.
+func (i *Index) openFields() error {
 	f, err := os.Open(i.path)
 	if err != nil {
 		return errors.Wrap(err, "opening directory")
@@ -135,14 +139,14 @@ func (i *Index) openFrames() error {
 			continue
 		}
 
-		fr, err := i.newFrame(i.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
+		fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
 		if err != nil {
 			return ErrName
 		}
-		if err := fr.Open(); err != nil {
-			return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err)
+		if err := fld.Open(); err != nil {
+			return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
 		}
-		i.frames[fr.Name()] = fr
+		i.fields[fld.Name()] = fld
 	}
 	return nil
 }
@@ -164,18 +168,17 @@ func (i *Index) loadMeta() error {
 	}
 
 	// Copy metadata fields.
+	i.keys = pb.Keys
 
 	return nil
 }
 
-// NOTE: Until we introduce new attributes to store in the index .meta file,
-// we don't need to actually write the file. The code related to index.options
-// and the index meta file are left in place for future use.
-/*
 // saveMeta writes meta data for the index.
 func (i *Index) saveMeta() error {
 	// Marshal metadata.
-	buf, err := proto.Marshal(&internal.IndexMeta{})
+	buf, err := proto.Marshal(&internal.IndexMeta{
+		Keys: i.keys,
+	})
 	if err != nil {
 		return errors.Wrap(err, "marshalling")
 	}
@@ -187,201 +190,209 @@ func (i *Index) saveMeta() error {
 
 	return nil
 }
-*/
 
-// Close closes the index and its frames.
+// Close closes the index and its fields.
 func (i *Index) Close() error {
 	i.mu.Lock()
 	defer i.mu.Unlock()
 
 	// Close the attribute store.
-	i.columnAttrStore.Close()
+	i.columnAttrs.Close()
 
-	// Close all frames.
-	for _, f := range i.frames {
+	// Close all fields.
+	for _, f := range i.fields {
 		if err := f.Close(); err != nil {
-			return errors.Wrap(err, "closing frame")
+			return errors.Wrap(err, "closing field")
 		}
 	}
-	i.frames = make(map[string]*Frame)
+	i.fields = make(map[string]*Field)
 
 	return nil
 }
 
-// MaxSlice returns the max slice in the index according to this node.
-func (i *Index) MaxSlice() uint64 {
+// maxShard returns the max shard in the index according to this node.
+func (i *Index) maxShard() uint64 {
 	if i == nil {
 		return 0
 	}
 	i.mu.RLock()
 	defer i.mu.RUnlock()
 
-	max := i.remoteMaxSlice
-	for _, f := range i.frames {
-		if slice := f.MaxSlice(); slice > max {
-			max = slice
+	max := i.remoteMaxShard
+	for _, f := range i.fields {
+		if shard := f.maxShard(); shard > max {
+			max = shard
 		}
 	}
 
-	i.Stats.Gauge("maxSlice", float64(max), 1.0)
+	i.Stats.Gauge("maxShard", float64(max), 1.0)
 	return max
 }
 
-// SetRemoteMaxSlice sets the remote max slice value received from another node.
-func (i *Index) SetRemoteMaxSlice(newmax uint64) {
+// setRemoteMaxShard sets the remote max shard value received from another node.
+func (i *Index) setRemoteMaxShard(newmax uint64) {
 	i.mu.Lock()
 	defer i.mu.Unlock()
-	i.remoteMaxSlice = newmax
+	i.remoteMaxShard = newmax
 }
 
-// FramePath returns the path to a frame in the index.
-func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) }
+// fieldPath returns the path to a field in the index.
+func (i *Index) fieldPath(name string) string { return filepath.Join(i.path, name) }
 
-// Frame returns a frame in the index by name.
-func (i *Index) Frame(name string) *Frame {
+// Field returns a field in the index by name.
+func (i *Index) Field(name string) *Field {
 	i.mu.RLock()
 	defer i.mu.RUnlock()
-	return i.frame(name)
+	return i.field(name)
 }
 
-func (i *Index) frame(name string) *Frame { return i.frames[name] }
+func (i *Index) field(name string) *Field { return i.fields[name] }
 
-// Frames returns a list of all frames in the index.
-func (i *Index) Frames() []*Frame {
+// Fields returns a list of all fields in the index.
+func (i *Index) Fields() []*Field {
 	i.mu.RLock()
 	defer i.mu.RUnlock()
 
-	a := make([]*Frame, 0, len(i.frames))
-	for _, f := range i.frames {
+	a := make([]*Field, 0, len(i.fields))
+	for _, f := range i.fields {
 		a = append(a, f)
 	}
-	sort.Sort(frameSlice(a))
+	sort.Sort(fieldSlice(a))
 
 	return a
 }
 
-// RecalculateCaches recalculates caches on every frame in the index.
-func (i *Index) RecalculateCaches() {
-	for _, frame := range i.Frames() {
-		frame.RecalculateCaches()
+// recalculateCaches recalculates caches on every field in the index.
+func (i *Index) recalculateCaches() {
+	for _, field := range i.Fields() {
+		field.recalculateCaches()
 	}
 }
 
-// CreateFrame creates a frame.
-func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) {
+// CreateField creates a field.
+func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) {
 	i.mu.Lock()
 	defer i.mu.Unlock()
 
-	// Ensure frame doesn't already exist.
-	if i.frames[name] != nil {
-		return nil, ErrFrameExists
-	}
-	return i.createFrame(name, opt)
-}
-
-// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
-func (i *Index) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) {
-	i.mu.Lock()
-	defer i.mu.Unlock()
-
-	// Find frame in cache first.
-	if f := i.frames[name]; f != nil {
-		return f, nil
+	// Ensure field doesn't already exist.
+	if i.fields[name] != nil {
+		return nil, newConflictError(ErrFieldExists)
 	}
 
-	return i.createFrame(name, opt)
-}
-
-func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
-	if name == "" {
-		return nil, errors.New("frame name required")
-	} else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) {
-		return nil, ErrInvalidCacheType
-	}
-
-	// Validate fields.
-	for _, field := range opt.Fields {
-		if err := ValidateField(field); err != nil {
-			return nil, err
+	// Apply functional options.
+	fo := FieldOptions{}
+	for _, opt := range opts {
+		err := opt(&fo)
+		if err != nil {
+			return nil, errors.Wrap(err, "applying option")
 		}
 	}
 
-	// Initialize frame.
-	f, err := i.newFrame(i.FramePath(name), name)
+	return i.createField(name, fo)
+}
+
+// CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
+func (i *Index) CreateFieldIfNotExists(name string, opts FieldOption) (*Field, error) {
+	i.mu.Lock()
+	defer i.mu.Unlock()
+
+	// Find field in cache first.
+	if f := i.fields[name]; f != nil {
+		return f, nil
+	}
+
+	// Apply functional option.
+	fo := FieldOptions{}
+	err := opts(&fo)
+	if err != nil {
+		return nil, errors.Wrap(err, "applying option")
+	}
+
+	return i.createField(name, fo)
+}
+
+func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, error) {
+	i.mu.Lock()
+	defer i.mu.Unlock()
+
+	// Find field in cache first.
+	if f := i.fields[name]; f != nil {
+		return f, nil
+	}
+
+	return i.createField(name, opt)
+}
+
+func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
+	if name == "" {
+		return nil, errors.New("field name required")
+	} else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) {
+		return nil, ErrInvalidCacheType
+	}
+
+	// Initialize field.
+	f, err := i.newField(i.fieldPath(name), name)
 	if err != nil {
 		return nil, errors.Wrap(err, "initializing")
 	}
 
-	// Open frame.
+	// Open field.
 	if err := f.Open(); err != nil {
 		return nil, errors.Wrap(err, "opening")
 	}
 
-	// Set the time quantum.
-	if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil {
+	// Apply field options.
+	if err := f.applyOptions(opt); err != nil {
 		f.Close()
-		return nil, errors.Wrap(err, "setting time quantum")
+		return nil, errors.Wrap(err, "applying options")
 	}
 
-	// Set cache type.
-	if opt.CacheType == "" {
-		opt.CacheType = DefaultCacheType
-	}
-	f.cacheType = opt.CacheType
-
-	if opt.CacheSize != 0 {
-		f.cacheSize = opt.CacheSize
-	}
-
-	// Set fields.
-	f.fields = opt.Fields
-
 	if err := f.saveMeta(); err != nil {
 		f.Close()
 		return nil, errors.Wrap(err, "saving meta")
 	}
 
-	// Add to index's frame lookup.
-	i.frames[name] = f
+	// Add to index's field lookup.
+	i.fields[name] = f
 
 	return f, nil
 }
 
-func (i *Index) newFrame(path, name string) (*Frame, error) {
-	f, err := NewFrame(path, i.name, name)
+func (i *Index) newField(path, name string) (*Field, error) {
+	f, err := NewField(path, i.name, name, OptFieldTypeDefault())
 	if err != nil {
 		return nil, err
 	}
-	f.Logger = i.Logger
-	f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name))
+	f.logger = i.logger
+	f.Stats = i.Stats.WithTags(fmt.Sprintf("field:%s", name))
 	f.broadcaster = i.broadcaster
-	f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data"))
+	f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data"))
 	return f, nil
 }
 
-// DeleteFrame removes a frame from the index.
-func (i *Index) DeleteFrame(name string) error {
+// DeleteField removes a field from the index.
+func (i *Index) DeleteField(name string) error {
 	i.mu.Lock()
 	defer i.mu.Unlock()
 
-	// Ignore if frame doesn't exist.
-	f := i.frame(name)
+	// Confirm field exists.
+	f := i.field(name)
 	if f == nil {
-		return nil
+		return newNotFoundError(ErrFieldNotFound)
 	}
 
-	// Close frame.
+	// Close field.
 	if err := f.Close(); err != nil {
 		return errors.Wrap(err, "closing")
 	}
 
-	// Delete frame directory.
-	if err := os.RemoveAll(i.FramePath(name)); err != nil {
+	// Delete field directory.
+	if err := os.RemoveAll(i.fieldPath(name)); err != nil {
 		return errors.Wrap(err, "removing directory")
 	}
 
 	// Remove reference.
-	delete(i.frames, name)
+	delete(i.fields, name)
 
 	return nil
 }
@@ -394,8 +405,9 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
 
 // IndexInfo represents schema information for an index.
 type IndexInfo struct {
-	Name   string       `json:"name"`
-	Frames []*FrameInfo `json:"frames"`
+	Name    string       `json:"name"`
+	options IndexOptions `json:"options"`
+	Fields  []*FieldInfo `json:"fields"`
 }
 
 type indexInfoSlice []*IndexInfo
@@ -404,29 +416,9 @@ func (p indexInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p indexInfoSlice) Len() int           { return len(p) }
 func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
 
-// EncodeIndexes converts a into its internal representation.
-func EncodeIndexes(a []*Index) []*internal.Index {
-	other := make([]*internal.Index, len(a))
-	for i := range a {
-		other[i] = encodeIndex(a[i])
-	}
-	return other
-}
-
-// encodeIndex converts d into its internal representation.
-func encodeIndex(d *Index) *internal.Index {
-	return &internal.Index{
-		Name:   d.name,
-		Frames: encodeFrames(d.Frames()),
-	}
-}
-
 // IndexOptions represents options to set when initializing an index.
-type IndexOptions struct{}
-
-// Encode converts i into its internal representation.
-func (i *IndexOptions) Encode() *internal.IndexMeta {
-	return &internal.IndexMeta{}
+type IndexOptions struct {
+	Keys bool `json:"keys"`
 }
 
 // hasTime returns true if a contains a non-nil time.
@@ -441,7 +433,7 @@ func hasTime(a []*time.Time) bool {
 
 type importKey struct {
 	View  string
-	Slice uint64
+	Shard uint64
 }
 
 type importData struct {

index_internal_test.go

@@ -15,28 +15,32 @@
 package pilosa
 
 import (
-	"fmt"
-	"net/http"
+	"io/ioutil"
 )
 
-// Ensure nopFileSystem implements interface.
-var _ FileSystem = &nopFileSystem{}
-
-// FileSystem represents an interface for a WebUI file system.
-type FileSystem interface {
-	New() (http.FileSystem, error)
+// mustOpenIndex returns a new, opened index at a temporary path. Panic on error.
+func mustOpenIndex() *Index {
+	path, err := ioutil.TempDir("", "pilosa-index-")
+	if err != nil {
+		panic(err)
+	}
+	index, err := NewIndex(path, "i")
+	if err != nil {
+		panic(err)
+	}
+	if err := index.Open(); err != nil {
+		panic(err)
+	}
+	return index
 }
 
-func init() {
-	NopFileSystem = &nopFileSystem{}
-}
-
-// NopFileSystem represents a FileSystem that returns an error if called.
-var NopFileSystem FileSystem
-
-type nopFileSystem struct{}
-
-// New is a no-op implementation of FileSystem New method.
-func (n *nopFileSystem) New() (http.FileSystem, error) {
-	return nil, fmt.Errorf("file system not implemented")
+// reopen closes the index and reopens it.
+func (i *Index) reopen() error {
+	if err := i.Close(); err != nil {
+		return err
+	}
+	if err := i.Open(); err != nil {
+		return err
+	}
+	return nil
 }

index_test.go

@@ -21,183 +21,181 @@ import (
 
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/test"
+	"github.com/pkg/errors"
 )
 
-// Ensure index can open and retrieve a frame.
-func TestIndex_CreateFrameIfNotExists(t *testing.T) {
+// ShardWidth is a helper reference to use when testing.
+const ShardWidth = pilosa.ShardWidth
+
+// Ensure index can open and retrieve a field.
+func TestIndex_CreateFieldIfNotExists(t *testing.T) {
 	index := test.MustOpenIndex()
 	defer index.Close()
 
-	// Create frame.
-	f, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
+	// Create field.
+	f, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
 	} else if f == nil {
-		t.Fatal("expected frame")
+		t.Fatal("expected field")
 	}
 
-	// Retrieve existing frame.
-	other, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
+	// Retrieve existing field.
+	other, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
-	} else if f.Frame != other.Frame {
-		t.Fatal("frame mismatch")
+	} else if f.Field != other.Field {
+		t.Fatal("field mismatch")
 	}
 
-	if f.Frame != index.Frame("f") {
-		t.Fatal("frame mismatch")
+	if f.Field != index.Field("f") {
+		t.Fatal("field mismatch")
 	}
 }
 
-func TestIndex_CreateFrame(t *testing.T) {
-	// Ensure time quantum can be set appropriately on a new frame.
+func TestIndex_CreateField(t *testing.T) {
+	// Ensure time quantum can be set appropriately on a new field.
 	t.Run("TimeQuantum", func(t *testing.T) {
 		t.Run("Explicit", func(t *testing.T) {
 			index := test.MustOpenIndex()
 			defer index.Close()
 
-			// Create frame with explicit quantum.
-			f, err := index.CreateFrame("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")})
+			// Create field with explicit quantum.
+			f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
 			if err != nil {
 				t.Fatal(err)
 			} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
-				t.Fatalf("unexpected frame time quantum: %s", q)
+				t.Fatalf("unexpected field time quantum: %s", q)
 			}
 		})
 	})
 
-	// Ensure frame can include range columns.
+	// Ensure field can include range columns.
 	t.Run("BSIFields", func(t *testing.T) {
 		t.Run("OK", func(t *testing.T) {
 			index := test.MustOpenIndex()
 			defer index.Close()
 
-			// Create frame with schema and verify it exists.
-			if f, err := index.CreateFrame("f", pilosa.FrameOptions{
-				Fields: []*pilosa.Field{
-					{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
-					{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
-				},
-			}); err != nil {
+			// Create field with schema and verify it exists.
+			if f, err := index.CreateField("f", pilosa.OptFieldTypeInt(10, 20)); err != nil {
 				t.Fatal(err)
-			} else if !reflect.DeepEqual(f.Fields(), []*pilosa.Field{
-				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
-				{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
-			}) {
-				t.Fatalf("unexpected fields: %#v", f.Fields())
+			} else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) {
+				t.Fatalf("unexpected type: %#v", f.Type())
 			}
 
 			// Reopen the index & verify the fields are loaded.
 			if err := index.Reopen(); err != nil {
 				t.Fatal(err)
-			} else if f := index.Frame("f"); !reflect.DeepEqual(f.Fields(), []*pilosa.Field{
-				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
-				{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
-			}) {
-				t.Fatalf("unexpected fields after reopen: %#v", f.Fields())
+			} else if f := index.Field("f"); !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) {
+				t.Fatalf("unexpected type after reopen: %#v", f.Type())
 			}
 		})
 
-		t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
-			index := test.MustOpenIndex()
-			defer index.Close()
+		// TODO: These errors don't apply  here. Instead, we need these tests
+		// on field creation FieldOptions validation.
+		/*
+			t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
+				index := test.MustOpenIndex()
+				defer index.Close()
 
-			if _, err := index.CreateFrame("f", pilosa.FrameOptions{
-				CacheType: pilosa.CacheTypeRanked,
-			}); err != nil {
-				t.Fatal(err)
-			}
-		})
+				if _, err := index.CreateField("f", pilosa.FieldOptions{
+					CacheType: pilosa.CacheTypeRanked,
+				}); err != nil {
+					t.Fatal(err)
+				}
+			})
 
-		t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) {
-			index := test.MustOpenIndex()
-			defer index.Close()
-			if _, err := index.CreateFrame("f", pilosa.FrameOptions{
-				CacheType: pilosa.CacheTypeNone,
-				CacheSize: uint32(5),
-			}); err != nil {
-				t.Fatal(err)
-			}
-		})
+			t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) {
+				index := test.MustOpenIndex()
+				defer index.Close()
+				if _, err := index.CreateField("f", pilosa.FieldOptions{
+					CacheType: pilosa.CacheTypeNone,
+					CacheSize: uint32(5),
+				}); err != nil {
+					t.Fatal(err)
+				}
+			})
 
-		t.Run("ErrFrameFieldsAllowed", func(t *testing.T) {
-			index := test.MustOpenIndex()
-			defer index.Close()
+			t.Run("ErrFieldFieldsAllowed", func(t *testing.T) {
+				index := test.MustOpenIndex()
+				defer index.Close()
 
-			if _, err := index.CreateFrame("f", pilosa.FrameOptions{
-				Fields: []*pilosa.Field{
-					{Name: "field0", Type: pilosa.FieldTypeInt},
-				},
-			}); err != nil {
-				t.Fatal(err)
-			}
-		})
+				if _, err := index.CreateField("f", pilosa.FieldOptions{
+					Fields: []*pilosa.Field{
+						{Name: "field0", Type: pilosa.FieldTypeInt},
+					},
+				}); err != nil {
+					t.Fatal(err)
+				}
+			})
 
-		t.Run("ErrFieldNameRequired", func(t *testing.T) {
-			index := test.MustOpenIndex()
-			defer index.Close()
+			t.Run("ErrFieldNameRequired", func(t *testing.T) {
+				index := test.MustOpenIndex()
+				defer index.Close()
 
-			if _, err := index.CreateFrame("f", pilosa.FrameOptions{
-				Fields: []*pilosa.Field{
-					{Name: "", Type: pilosa.FieldTypeInt},
-				},
-			}); err != pilosa.ErrFieldNameRequired {
-				t.Fatal(err)
-			}
-		})
+				if _, err := index.CreateField("f", pilosa.FieldOptions{
+					Fields: []*pilosa.Field{
+						{Name: "", Type: pilosa.FieldTypeInt},
+					},
+				}); err != pilosa.ErrFieldNameRequired {
+					t.Fatal(err)
+				}
+			})
 
-		t.Run("ErrInvalidFieldType", func(t *testing.T) {
-			index := test.MustOpenIndex()
-			defer index.Close()
+			t.Run("ErrInvalidFieldType", func(t *testing.T) {
+				index := test.MustOpenIndex()
+				defer index.Close()
 
-			if _, err := index.CreateFrame("f", pilosa.FrameOptions{
-				Fields: []*pilosa.Field{
-					{Name: "field0", Type: "bad_type"},
-				},
-			}); err != pilosa.ErrInvalidFieldType {
-				t.Fatal(err)
-			}
-		})
+				if _, err := index.CreateField("f", pilosa.FieldOptions{
+					Fields: []*pilosa.Field{
+						{Name: "field0", Type: "bad_type"},
+					},
+				}); err != pilosa.ErrInvalidFieldType {
+					t.Fatal(err)
+				}
+			})
 
-		t.Run("ErrInvalidFieldRange", func(t *testing.T) {
-			index := test.MustOpenIndex()
-			defer index.Close()
+			t.Run("ErrInvalidBSIGroupRange", func(t *testing.T) {
+				index := test.MustOpenIndex()
+				defer index.Close()
 
-			if _, err := index.CreateFrame("f", pilosa.FrameOptions{
-				Fields: []*pilosa.Field{
-					{Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50},
-				},
-			}); err != pilosa.ErrInvalidFieldRange {
-				t.Fatal(err)
-			}
-		})
+				if _, err := index.CreateField("f", pilosa.FieldOptions{
+					Fields: []*pilosa.Field{
+						{Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50},
+					},
+				}); err != pilosa.ErrInvalidBSIGroupRange {
+					t.Fatal(err)
+				}
+			})
+		*/
 	})
 }
 
-// Ensure index can delete a frame.
-func TestIndex_DeleteFrame(t *testing.T) {
+// Ensure index can delete a field.
+func TestIndex_DeleteField(t *testing.T) {
 	index := test.MustOpenIndex()
 	defer index.Close()
 
-	// Create frame.
-	if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
+	// Create field.
+	if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()); err != nil {
 		t.Fatal(err)
 	}
 
-	// Delete frame & verify it's gone.
-	if err := index.DeleteFrame("f"); err != nil {
+	// Delete field & verify it's gone.
+	if err := index.DeleteField("f"); err != nil {
 		t.Fatal(err)
-	} else if index.Frame("f") != nil {
-		t.Fatal("expected nil frame")
+	} else if index.Field("f") != nil {
+		t.Fatal("expected nil field")
 	}
 
-	// Delete again to make sure it doesn't error.
-	if err := index.DeleteFrame("f"); err != nil {
-		t.Fatal(err)
+	// Delete again to make sure it errors.
+	err := index.DeleteField("f")
+	if !isNotFoundError(err) {
+		t.Fatalf("expected 'field not found' error, got: %#v", err)
 	}
 }
 
-// Ensure index can delete a frame.
+// Ensure index can validate its name.
 func TestIndex_InvalidName(t *testing.T) {
 	path, err := ioutil.TempDir("", "pilosa-index-")
 	if err != nil {
@@ -211,3 +209,9 @@ func TestIndex_InvalidName(t *testing.T) {
 		t.Fatalf("unexpected index name %v", index)
 	}
 }
+
+func isNotFoundError(err error) bool {
+	root := errors.Cause(err)
+	_, ok := root.(pilosa.NotFoundError)
+	return ok
+}

inmem/translator.go

@@ -0,0 +1,215 @@
+package inmem
+
+import (
+	"context"
+	"io"
+	"sync"
+
+	"github.com/pilosa/pilosa"
+)
+
+// Ensure type implements interface.
+var _ pilosa.TranslateStore = &translateStore{}
+
+// translateStore is an in-memory storage engine for translating string-to-uint64 values.
+type translateStore struct {
+	mu sync.RWMutex
+
+	cols map[string]*translateIndex
+	rows map[frameKey]*translateIndex
+}
+
+// NewTranslateStore returns a new instance of TranslateStore.
+func NewTranslateStore() *translateStore {
+	return &translateStore{
+		cols: make(map[string]*translateIndex),
+		rows: make(map[frameKey]*translateIndex),
+	}
+}
+
+// Reader returns an error because it is not supported by the inmem store.
+func (s *translateStore) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) {
+	return nil, pilosa.ErrReplicationNotSupported
+}
+
+// TranslateColumnsToUint64 converts value to a uint64 id.
+// If value does not have an associated id then one is created.
+func (s *translateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) {
+	ret := make([]uint64, len(values))
+
+	// Read value under read lock.
+	s.mu.RLock()
+	if idx := s.cols[index]; idx != nil {
+		var writeRequired bool
+		for i := range values {
+			v, ok := idx.lookup[values[i]]
+			if !ok {
+				writeRequired = true
+			}
+			ret[i] = v
+		}
+		if !writeRequired {
+			s.mu.RUnlock()
+			return ret, nil
+		}
+	}
+	s.mu.RUnlock()
+
+	// If any values not found then recheck and then add under a write lock.
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Recheck if value was created between the read lock and write lock.
+	idx := s.cols[index]
+	if idx != nil {
+		var writeRequired bool
+		for i := range values {
+			if ret[i] != 0 {
+				continue
+			}
+			v, ok := idx.lookup[values[i]]
+			if !ok {
+				writeRequired = true
+				continue
+			}
+			ret[i] = v
+		}
+		if !writeRequired {
+			return ret, nil
+		}
+	}
+
+	// Create index map if it doesn't exists.
+	if idx == nil {
+		idx = newTranslateIndex()
+		s.cols[index] = idx
+	}
+
+	// Add new identifiers.
+	for i := range values {
+		if ret[i] != 0 {
+			continue
+		}
+
+		idx.seq++
+		v := idx.seq
+		ret[i] = v
+		idx.lookup[values[i]] = v
+		idx.reverse[v] = values[i]
+	}
+
+	return ret, nil
+}
+
+// TranslateColumnToString converts a uint64 id to its associated string value.
+// If the id is not associated with a string value then a blank string is returned.
+func (s *translateStore) TranslateColumnToString(index string, value uint64) (string, error) {
+	s.mu.RLock()
+	if idx := s.cols[index]; idx != nil {
+		if ret, ok := idx.reverse[value]; ok {
+			s.mu.RUnlock()
+			return ret, nil
+		}
+	}
+	s.mu.RUnlock()
+	return "", nil
+}
+
+func (s *translateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
+	key := frameKey{index, frame}
+
+	ret := make([]uint64, len(values))
+
+	// Read value under read lock.
+	s.mu.RLock()
+	if idx := s.rows[key]; idx != nil {
+		var writeRequired bool
+		for i := range values {
+			v, ok := idx.lookup[values[i]]
+			if !ok {
+				writeRequired = true
+			}
+			ret[i] = v
+		}
+		if !writeRequired {
+			s.mu.RUnlock()
+			return ret, nil
+		}
+	}
+	s.mu.RUnlock()
+
+	// If any values not found then recheck and then add under a write lock.
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Recheck if value was created between the read lock and write lock.
+	idx := s.rows[key]
+	if idx != nil {
+		var writeRequired bool
+		for i := range values {
+			if ret[i] != 0 {
+				continue
+			}
+			v, ok := idx.lookup[values[i]]
+			if !ok {
+				writeRequired = true
+				continue
+			}
+			ret[i] = v
+		}
+		if !writeRequired {
+			return ret, nil
+		}
+	}
+
+	// Create map if it doesn't exists.
+	if idx == nil {
+		idx = newTranslateIndex()
+		s.rows[key] = idx
+	}
+
+	// Add new identifiers.
+	for i := range values {
+		if ret[i] != 0 {
+			continue
+		}
+
+		idx.seq++
+		v := idx.seq
+		ret[i] = v
+		idx.lookup[values[i]] = v
+		idx.reverse[v] = values[i]
+	}
+
+	return ret, nil
+}
+
+func (s *translateStore) TranslateRowToString(index, frame string, value uint64) (string, error) {
+	s.mu.RLock()
+	if idx := s.rows[frameKey{index, frame}]; idx != nil {
+		if ret, ok := idx.reverse[value]; ok {
+			s.mu.RUnlock()
+			return ret, nil
+		}
+	}
+	s.mu.RUnlock()
+	return "", nil
+}
+
+type frameKey struct {
+	index string
+	frame string
+}
+
+type translateIndex struct {
+	seq     uint64
+	lookup  map[string]uint64
+	reverse map[uint64]string
+}
+
+func newTranslateIndex() *translateIndex {
+	return &translateIndex{
+		lookup:  make(map[string]uint64),
+		reverse: make(map[uint64]string),
+	}
+}

inmem/translator_test.go

@@ -0,0 +1,132 @@
+package inmem_test
+
+import (
+	"fmt"
+	"math/rand"
+	"reflect"
+	"testing"
+
+	"github.com/pilosa/pilosa/inmem"
+)
+
+func TestTranslateStore_TranslateColumn(t *testing.T) {
+	s := inmem.NewTranslateStore()
+
+	// First translation should start id at zero.
+	if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(ids, []uint64{1}) {
+		t.Fatalf("unexpected id: %#v", ids)
+	}
+
+	// Next translation on the same index should move to one.
+	if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(ids, []uint64{2}) {
+		t.Fatalf("unexpected id: %#v", ids)
+	}
+
+	// Translation on a different index restarts at 0.
+	if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(ids, []uint64{1}) {
+		t.Fatalf("unexpected id: %#v", ids)
+	}
+
+	// Ensure that string values can be looked up by ID.
+	if value, err := s.TranslateColumnToString("IDX0", 2); err != nil {
+		t.Fatal(err)
+	} else if value != "bar" {
+		t.Fatalf("unexpected value: %s", value)
+	}
+}
+
+func TestTranslateStore_TranslateRow(t *testing.T) {
+	s := inmem.NewTranslateStore()
+
+	// First translation should start id at zero.
+	if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(ids, []uint64{1}) {
+		t.Fatalf("unexpected id: %#v", ids)
+	}
+
+	// Next translation on the same index should move to one.
+	if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(ids, []uint64{2}) {
+		t.Fatalf("unexpected id: %#v", ids)
+	}
+
+	// Translation on a different index restarts at 0.
+	if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(ids, []uint64{1}) {
+		t.Fatalf("unexpected id: %#v", ids)
+	}
+
+	// Translation on a different frame restarts at 0.
+	if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(ids, []uint64{1}) {
+		t.Fatalf("unexpected id: %#v", ids)
+	}
+
+	// Ensure that string values can be looked up by ID.
+	if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil {
+		t.Fatal(err)
+	} else if value != "bar" {
+		t.Fatalf("unexpected value: %s", value)
+	}
+}
+
+func BenchmarkTranslateStore_TranslateColumnsToUint64(b *testing.B) {
+	const batchSize = 1000
+
+	s := inmem.NewTranslateStore()
+
+	// Generate keys before benchmark begins
+	keySets := make([][]string, b.N/1000)
+	for i := range keySets {
+		keySets[i] = make([]string, batchSize)
+		for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) {
+			keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i)
+		}
+	}
+
+	b.ResetTimer()
+
+	for _, keySet := range keySets {
+		if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkTranslateStore_TranslateColumnToString(b *testing.B) {
+	const batchSize = 1000
+
+	s := inmem.NewTranslateStore()
+
+	// Generate keys before benchmark begins
+	for i := 0; i < b.N; i += batchSize {
+		keySet := make([]string, batchSize)
+		for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) {
+			keySet[j] = fmt.Sprintf("%08d%08d", jv, i)
+		}
+		if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	// Generate random key access.
+	perm := rand.New(rand.NewSource(0)).Perm(b.N)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

internal/private.proto

@@ -3,13 +3,17 @@ syntax = "proto3";
 package internal;
 
 message IndexMeta {
+	bool Keys = 3;
 }
 
-message FrameMeta {
+message FieldOptions {
+    string Type = 8;
 	string CacheType = 3;
 	uint32 CacheSize = 4;
+    int64 Min = 9;
+    int64 Max = 10;
 	string TimeQuantum = 5;
-	repeated Field Fields = 7;
+    bool Keys = 11;
 }
 
 message ImportResponse {
@@ -18,9 +22,9 @@ message ImportResponse {
 
 message BlockDataRequest {
 	string Index = 1;
-	string Frame = 2;
+	string Field = 2;
 	string View = 5;
-	uint64 Slice = 4;
+	uint64 Shard = 4;
 	uint64 Block = 3;
 }
 
@@ -33,13 +37,13 @@ message Cache {
 	repeated uint64 IDs = 1;
 }
 
-message MaxSlices {
+message MaxShards {
     map<string, uint64> Standard = 1;
 }
 
-message CreateSliceMessage {
+message CreateShardMessage {
     string Index = 1;
-    uint64 Slice = 2;
+    uint64 Shard = 2;
 }
 
 message DeleteIndexMessage {
@@ -51,32 +55,20 @@ message CreateIndexMessage {
     IndexMeta Meta = 2;
 }
 
-message CreateFrameMessage {
-    string Index = 1;
-    string Frame = 2;
-    FrameMeta Meta = 3;
-}
-
-message DeleteFrameMessage {
-    string Index = 1;
-    string Frame = 2;
-}
-
 message CreateFieldMessage {
     string Index = 1;
-    string Frame = 2;
-    Field Field = 3;
+    string Field = 2;
+    FieldOptions Meta = 3;
 }
 
 message DeleteFieldMessage {
     string Index = 1;
-    string Frame = 2;
-    string Field = 3;
+    string Field = 2;
 }
 
-message Frame {
+message Field {
     string Name = 1;
-    FrameMeta Meta = 2;
+    FieldOptions Meta = 2;
     repeated string Views = 3;
 }
 
@@ -86,7 +78,7 @@ message Schema {
 
 message Index {
     string Name = 1;
-    repeated Frame Frames = 4;
+    repeated Field Fields = 4;
 }
 
 message URI {
@@ -113,7 +105,7 @@ message NodeEventMessage {
 
 message NodeStatus {
     Node Node = 1;
-    MaxSlices MaxSlices = 2;
+    MaxShards MaxShards = 2;
     Schema Schema = 3;
 }
 
@@ -123,7 +115,7 @@ message ClusterStatus {
     repeated Node Nodes = 3;
 }
 
-message Field {
+message BSIGroup {
     string Name = 1;
     string Type = 2;
     int64 Min = 3;
@@ -132,13 +124,13 @@ message Field {
 
 message CreateViewMessage {
     string Index = 1;
-    string Frame = 2;
+    string Field = 2;
     string View = 3;
 }
 
 message DeleteViewMessage {
     string Index = 1;
-    string Frame = 2;
+    string Field = 2;
     string View = 3;
 }
 
@@ -154,9 +146,9 @@ message ResizeInstruction {
 message ResizeSource {
     Node Node = 1;
     string Index = 2;
-    string Frame = 3;
+    string Field = 3;
     string View = 4;
-    uint64 Slice = 5;
+    uint64 Shard = 5;
 }
 
 message ResizeInstructionComplete {

internal/public.pb.go

@@ -1,5 +1,6 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// Code generated by protoc-gen-gogo.
 // source: public.proto
+// DO NOT EDIT!
 
 /*
 	Package internal is a generated protocol buffer package.
@@ -27,8 +28,6 @@ import proto "github.com/golang/protobuf/proto"
 import fmt "fmt"
 import math "math"
 
-import binary "encoding/binary"
-
 import io "io"
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -268,7 +267,7 @@ func (m *AttrMap) GetAttrs() []*Attr {
 
 type QueryRequest struct {
 	Query           string   `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"`
-	Slices          []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"`
+	Shards          []uint64 `protobuf:"varint,2,rep,packed,name=Shards" json:"Shards,omitempty"`
 	ColumnAttrs     bool     `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"`
 	Remote          bool     `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"`
 	ExcludeRowAttrs bool     `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"`
@@ -287,9 +286,9 @@ func (m *QueryRequest) GetQuery() string {
 	return ""
 }
 
-func (m *QueryRequest) GetSlices() []uint64 {
+func (m *QueryRequest) GetShards() []uint64 {
 	if m != nil {
-		return m.Slices
+		return m.Shards
 	}
 	return nil
 }
@@ -412,8 +411,8 @@ func (m *QueryResult) GetChanged() bool {
 
 type ImportRequest struct {
 	Index      string   `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
-	Frame      string   `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
-	Slice      uint64   `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
+	Field      string   `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
+	Shard      uint64   `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"`
 	RowIDs     []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"`
 	ColumnIDs  []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"`
 	RowKeys    []string `protobuf:"bytes,7,rep,name=RowKeys" json:"RowKeys,omitempty"`
@@ -433,16 +432,16 @@ func (m *ImportRequest) GetIndex() string {
 	return ""
 }
 
-func (m *ImportRequest) GetFrame() string {
+func (m *ImportRequest) GetField() string {
 	if m != nil {
-		return m.Frame
+		return m.Field
 	}
 	return ""
 }
 
-func (m *ImportRequest) GetSlice() uint64 {
+func (m *ImportRequest) GetShard() uint64 {
 	if m != nil {
-		return m.Slice
+		return m.Shard
 	}
 	return 0
 }
@@ -484,9 +483,8 @@ func (m *ImportRequest) GetTimestamps() []int64 {
 
 type ImportValueRequest struct {
 	Index      string   `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
-	Frame      string   `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
-	Slice      uint64   `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
-	Field      string   `protobuf:"bytes,4,opt,name=Field,proto3" json:"Field,omitempty"`
+	Field      string   `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
+	Shard      uint64   `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"`
 	ColumnIDs  []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"`
 	ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"`
 	Values     []int64  `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"`
@@ -504,20 +502,6 @@ func (m *ImportValueRequest) GetIndex() string {
 	return ""
 }
 
-func (m *ImportValueRequest) GetFrame() string {
-	if m != nil {
-		return m.Frame
-	}
-	return ""
-}
-
-func (m *ImportValueRequest) GetSlice() uint64 {
-	if m != nil {
-		return m.Slice
-	}
-	return 0
-}
-
 func (m *ImportValueRequest) GetField() string {
 	if m != nil {
 		return m.Field
@@ -525,6 +509,13 @@ func (m *ImportValueRequest) GetField() string {
 	return ""
 }
 
+func (m *ImportValueRequest) GetShard() uint64 {
+	if m != nil {
+		return m.Shard
+	}
+	return 0
+}
+
 func (m *ImportValueRequest) GetColumnIDs() []uint64 {
 	if m != nil {
 		return m.ColumnIDs
@@ -808,8 +799,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) {
 	if m.FloatValue != 0 {
 		dAtA[i] = 0x31
 		i++
-		binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue))))
-		i += 8
+		i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue))))
 	}
 	return i, nil
 }
@@ -865,10 +855,10 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) {
 		i = encodeVarintPublic(dAtA, i, uint64(len(m.Query)))
 		i += copy(dAtA[i:], m.Query)
 	}
-	if len(m.Slices) > 0 {
-		dAtA4 := make([]byte, len(m.Slices)*10)
+	if len(m.Shards) > 0 {
+		dAtA4 := make([]byte, len(m.Shards)*10)
 		var j3 int
-		for _, num := range m.Slices {
+		for _, num := range m.Shards {
 			for num >= 1<<7 {
 				dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80)
 				num >>= 7
@@ -1064,16 +1054,16 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) {
 		i = encodeVarintPublic(dAtA, i, uint64(len(m.Index)))
 		i += copy(dAtA[i:], m.Index)
 	}
-	if len(m.Frame) > 0 {
+	if len(m.Field) > 0 {
 		dAtA[i] = 0x12
 		i++
-		i = encodeVarintPublic(dAtA, i, uint64(len(m.Frame)))
-		i += copy(dAtA[i:], m.Frame)
+		i = encodeVarintPublic(dAtA, i, uint64(len(m.Field)))
+		i += copy(dAtA[i:], m.Field)
 	}
-	if m.Slice != 0 {
+	if m.Shard != 0 {
 		dAtA[i] = 0x18
 		i++
-		i = encodeVarintPublic(dAtA, i, uint64(m.Slice))
+		i = encodeVarintPublic(dAtA, i, uint64(m.Shard))
 	}
 	if len(m.RowIDs) > 0 {
 		dAtA8 := make([]byte, len(m.RowIDs)*10)
@@ -1181,23 +1171,17 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) {
 		i = encodeVarintPublic(dAtA, i, uint64(len(m.Index)))
 		i += copy(dAtA[i:], m.Index)
 	}
-	if len(m.Frame) > 0 {
-		dAtA[i] = 0x12
-		i++
-		i = encodeVarintPublic(dAtA, i, uint64(len(m.Frame)))
-		i += copy(dAtA[i:], m.Frame)
-	}
-	if m.Slice != 0 {
-		dAtA[i] = 0x18
-		i++
-		i = encodeVarintPublic(dAtA, i, uint64(m.Slice))
-	}
 	if len(m.Field) > 0 {
-		dAtA[i] = 0x22
+		dAtA[i] = 0x12
 		i++
 		i = encodeVarintPublic(dAtA, i, uint64(len(m.Field)))
 		i += copy(dAtA[i:], m.Field)
 	}
+	if m.Shard != 0 {
+		dAtA[i] = 0x18
+		i++
+		i = encodeVarintPublic(dAtA, i, uint64(m.Shard))
+	}
 	if len(m.ColumnIDs) > 0 {
 		dAtA14 := make([]byte, len(m.ColumnIDs)*10)
 		var j13 int
@@ -1251,6 +1235,24 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) {
 	return i, nil
 }
 
+func encodeFixed64Public(dAtA []byte, offset int, v uint64) int {
+	dAtA[offset] = uint8(v)
+	dAtA[offset+1] = uint8(v >> 8)
+	dAtA[offset+2] = uint8(v >> 16)
+	dAtA[offset+3] = uint8(v >> 24)
+	dAtA[offset+4] = uint8(v >> 32)
+	dAtA[offset+5] = uint8(v >> 40)
+	dAtA[offset+6] = uint8(v >> 48)
+	dAtA[offset+7] = uint8(v >> 56)
+	return offset + 8
+}
+func encodeFixed32Public(dAtA []byte, offset int, v uint32) int {
+	dAtA[offset] = uint8(v)
+	dAtA[offset+1] = uint8(v >> 8)
+	dAtA[offset+2] = uint8(v >> 16)
+	dAtA[offset+3] = uint8(v >> 24)
+	return offset + 4
+}
 func encodeVarintPublic(dAtA []byte, offset int, v uint64) int {
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
@@ -1392,9 +1394,9 @@ func (m *QueryRequest) Size() (n int) {
 	if l > 0 {
 		n += 1 + l + sovPublic(uint64(l))
 	}
-	if len(m.Slices) > 0 {
+	if len(m.Shards) > 0 {
 		l = 0
-		for _, e := range m.Slices {
+		for _, e := range m.Shards {
 			l += sovPublic(uint64(e))
 		}
 		n += 1 + sovPublic(uint64(l)) + l
@@ -1472,12 +1474,12 @@ func (m *ImportRequest) Size() (n int) {
 	if l > 0 {
 		n += 1 + l + sovPublic(uint64(l))
 	}
-	l = len(m.Frame)
+	l = len(m.Field)
 	if l > 0 {
 		n += 1 + l + sovPublic(uint64(l))
 	}
-	if m.Slice != 0 {
-		n += 1 + sovPublic(uint64(m.Slice))
+	if m.Shard != 0 {
+		n += 1 + sovPublic(uint64(m.Shard))
 	}
 	if len(m.RowIDs) > 0 {
 		l = 0
@@ -1522,17 +1524,13 @@ func (m *ImportValueRequest) Size() (n int) {
 	if l > 0 {
 		n += 1 + l + sovPublic(uint64(l))
 	}
-	l = len(m.Frame)
-	if l > 0 {
-		n += 1 + l + sovPublic(uint64(l))
-	}
-	if m.Slice != 0 {
-		n += 1 + sovPublic(uint64(m.Slice))
-	}
 	l = len(m.Field)
 	if l > 0 {
 		n += 1 + l + sovPublic(uint64(l))
 	}
+	if m.Shard != 0 {
+		n += 1 + sovPublic(uint64(m.Shard))
+	}
 	if len(m.ColumnIDs) > 0 {
 		l = 0
 		for _, e := range m.ColumnIDs {
@@ -2335,8 +2333,15 @@ func (m *Attr) Unmarshal(dAtA []byte) error {
 			if (iNdEx + 8) > l {
 				return io.ErrUnexpectedEOF
 			}
-			v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:]))
 			iNdEx += 8
+			v = uint64(dAtA[iNdEx-8])
+			v |= uint64(dAtA[iNdEx-7]) << 8
+			v |= uint64(dAtA[iNdEx-6]) << 16
+			v |= uint64(dAtA[iNdEx-5]) << 24
+			v |= uint64(dAtA[iNdEx-4]) << 32
+			v |= uint64(dAtA[iNdEx-3]) << 40
+			v |= uint64(dAtA[iNdEx-2]) << 48
+			v |= uint64(dAtA[iNdEx-1]) << 56
 			m.FloatValue = float64(math.Float64frombits(v))
 		default:
 			iNdEx = preIndex
@@ -2515,7 +2520,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
 						break
 					}
 				}
-				m.Slices = append(m.Slices, v)
+				m.Shards = append(m.Shards, v)
 			} else if wireType == 2 {
 				var packedLen int
 				for shift := uint(0); ; shift += 7 {
@@ -2555,10 +2560,10 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
 							break
 						}
 					}
-					m.Slices = append(m.Slices, v)
+					m.Shards = append(m.Shards, v)
 				}
 			} else {
-				return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field Shards", wireType)
 			}
 		case 3:
 			if wireType != 0 {
@@ -3067,7 +3072,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
 			iNdEx = postIndex
 		case 2:
 			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType)
 			}
 			var stringLen uint64
 			for shift := uint(0); ; shift += 7 {
@@ -3092,13 +3097,13 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.Frame = string(dAtA[iNdEx:postIndex])
+			m.Field = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
 		case 3:
 			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType)
 			}
-			m.Slice = 0
+			m.Shard = 0
 			for shift := uint(0); ; shift += 7 {
 				if shift >= 64 {
 					return ErrIntOverflowPublic
@@ -3108,7 +3113,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Slice |= (uint64(b) & 0x7F) << shift
+				m.Shard |= (uint64(b) & 0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -3437,54 +3442,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
 			m.Index = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
 		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPublic
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthPublic
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Frame = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		case 3:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType)
-			}
-			m.Slice = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPublic
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Slice |= (uint64(b) & 0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		case 4:
 			if wireType != 2 {
 				return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType)
 			}
@@ -3513,6 +3470,25 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
 			}
 			m.Field = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType)
+			}
+			m.Shard = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPublic
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Shard |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
 		case 5:
 			if wireType == 0 {
 				var v uint64
@@ -3795,50 +3771,49 @@ var (
 func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
 
 var fileDescriptorPublic = []byte{
-	// 709 bytes of a gzipped FileDescriptorProto
+	// 701 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c,
-	0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0xe8, 0xfb, 0x8a, 0x85, 0x50, 0xb0, 0x2c,
-	0x84, 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0x60, 0x5a, 0x8a, 0x58, 0xba,
-	0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x0c, 0xc4,
-	0xaa, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x7b, 0xc7, 0x13, 0xbb, 0xa9, 0x04, 0x2c, 0xd8, 0xcd,
-	0x39, 0x67, 0xe6, 0x66, 0xce, 0xdc, 0x73, 0x1d, 0xd8, 0xc8, 0xca, 0xb3, 0x24, 0x3e, 0x9f, 0x64,
-	0xb9, 0xd2, 0x8a, 0xf7, 0xe2, 0x54, 0xcb, 0x3c, 0x8d, 0x92, 0xe0, 0x2d, 0x38, 0x42, 0x2d, 0xb8,
-	0x07, 0xdd, 0x5d, 0x95, 0x94, 0xf3, 0xb4, 0xf0, 0x98, 0xef, 0x84, 0xae, 0xb0, 0x90, 0x3f, 0x82,
-	0xf6, 0x73, 0xad, 0xf3, 0xc2, 0x6b, 0xf9, 0x4e, 0x38, 0x98, 0x8e, 0x26, 0xf6, 0xe8, 0x04, 0x69,
-	0x61, 0x44, 0xce, 0xc1, 0x3d, 0x94, 0xcb, 0xc2, 0x73, 0x7c, 0x27, 0xec, 0x0b, 0x5a, 0x07, 0x4f,
-	0xc1, 0x7d, 0x19, 0xc5, 0x39, 0x1f, 0x41, 0x6b, 0xb6, 0xe7, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd,
-	0xf6, 0xf8, 0x7f, 0xd0, 0xde, 0x55, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x4d, 0x70, 0x0e,
-	0xe5, 0xd2, 0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0x4e, 0xa3, 0x64, 0xa5, 0x9e,
-	0x46, 0x09, 0x15, 0x71, 0x04, 0x2e, 0x6f, 0x57, 0x71, 0xaa, 0x2a, 0xc1, 0x6b, 0x70, 0x76, 0x62,
-	0x8d, 0xa2, 0x50, 0x8b, 0xd5, 0xaf, 0x1a, 0xc0, 0xef, 0x43, 0xcf, 0xb8, 0x9a, 0xed, 0x55, 0xbf,
-	0xbd, 0xc2, 0xfc, 0x01, 0xf4, 0x4f, 0xe2, 0xb9, 0x2c, 0x74, 0x34, 0xcf, 0xe8, 0x12, 0x8e, 0xa8,
-	0x89, 0xe0, 0x0d, 0x0c, 0xcd, 0x4e, 0x74, 0x7b, 0x2c, 0xf5, 0x1d, 0x4f, 0x7f, 0xf6, 0x4a, 0x77,
-	0x3d, 0x7e, 0x62, 0xe0, 0xa2, 0x66, 0x25, 0xb6, 0x92, 0xf0, 0x49, 0x4f, 0x96, 0x99, 0xac, 0x6e,
-	0x4a, 0x6b, 0xee, 0xc3, 0xe0, 0x58, 0xe7, 0x71, 0x7a, 0x79, 0x1a, 0x25, 0xa5, 0xac, 0x0a, 0x35,
-	0x29, 0xf4, 0x38, 0x4b, 0xb5, 0x91, 0x5d, 0xb2, 0xb1, 0xc2, 0xe8, 0x71, 0x47, 0xa9, 0xc4, 0x88,
-	0x6d, 0x9f, 0x85, 0x3d, 0x51, 0x13, 0x7c, 0x0c, 0x70, 0x90, 0xa8, 0xa8, 0x3a, 0xdb, 0xf1, 0x59,
-	0xc8, 0x44, 0x83, 0x09, 0xb6, 0xa1, 0x8b, 0x37, 0x7d, 0x11, 0x65, 0xb5, 0x5b, 0xf6, 0x0b, 0xb7,
-	0xc1, 0x35, 0x83, 0x8d, 0x57, 0xa5, 0xcc, 0x97, 0x42, 0xbe, 0x2f, 0x65, 0x41, 0x5d, 0x21, 0x5c,
-	0xb9, 0x34, 0x80, 0x6f, 0x41, 0xe7, 0x38, 0x89, 0xcf, 0xa5, 0x79, 0x3b, 0x57, 0x54, 0x08, 0xbd,
-	0xd6, 0x6f, 0x5e, 0x90, 0xd7, 0x9e, 0x68, 0x52, 0x78, 0x52, 0xc8, 0xb9, 0xd2, 0xd6, 0x4c, 0x85,
-	0x78, 0x08, 0xff, 0xee, 0x5f, 0x9d, 0x27, 0xe5, 0x85, 0x14, 0x6a, 0x61, 0x4e, 0x77, 0x68, 0xc3,
-	0x3a, 0xcd, 0x1f, 0xc3, 0xa8, 0xa2, 0x6c, 0xfa, 0xbb, 0xb4, 0x71, 0x8d, 0x0d, 0x3e, 0x30, 0x18,
-	0x56, 0x56, 0x8a, 0x4c, 0xa5, 0x85, 0xc4, 0x7e, 0xed, 0xe7, 0xb9, 0xed, 0xd7, 0x7e, 0x9e, 0xf3,
-	0x6d, 0xe8, 0x0a, 0x59, 0x94, 0x89, 0xb6, 0x21, 0xf8, 0xbf, 0x7e, 0x16, 0x7b, 0xb6, 0x4c, 0xb4,
-	0xb0, 0xbb, 0xf8, 0x33, 0x18, 0xdd, 0x0a, 0x95, 0x99, 0x9e, 0xc1, 0xf4, 0x5e, 0x7d, 0xee, 0x96,
-	0x2e, 0xd6, 0xb6, 0x07, 0x9f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x43, 0x9a, 0x65, 0xba, 0xd3, 0x60,
-	0x3a, 0xac, 0xab, 0x08, 0xb5, 0x10, 0x34, 0xe5, 0x1b, 0xc0, 0x8e, 0xaa, 0x3c, 0xb1, 0x23, 0xec,
-	0x22, 0xce, 0xa7, 0xfd, 0xd9, 0x46, 0x17, 0x91, 0x16, 0x46, 0xa4, 0x2f, 0xc3, 0xbb, 0x28, 0xbd,
-	0x94, 0x17, 0x94, 0xa7, 0x9e, 0xb0, 0x90, 0x4f, 0xea, 0xf9, 0xa4, 0x06, 0x0c, 0xa6, 0xbc, 0x2e,
-	0x61, 0x15, 0x51, 0xcf, 0xb0, 0x0d, 0x34, 0xf6, 0x62, 0x68, 0x02, 0x1d, 0x7c, 0x67, 0x30, 0x9c,
-	0xcd, 0x33, 0x95, 0xeb, 0x46, 0x48, 0x66, 0xe9, 0x85, 0xbc, 0xb2, 0x21, 0x21, 0x80, 0xec, 0x41,
-	0x1e, 0xcd, 0xcd, 0x34, 0xf4, 0x85, 0x01, 0xc8, 0x52, 0x58, 0x28, 0x1c, 0xae, 0x30, 0x80, 0x62,
-	0x81, 0xf3, 0x5e, 0x78, 0xae, 0x09, 0x94, 0x41, 0x18, 0x7f, 0x3b, 0xee, 0x85, 0xd7, 0x26, 0xa9,
-	0x26, 0x30, 0xfe, 0xab, 0x79, 0xc7, 0xbc, 0x38, 0xa1, 0x23, 0x1a, 0x0c, 0xbe, 0x83, 0x50, 0x0b,
-	0xfa, 0xc8, 0x75, 0xe9, 0x23, 0x67, 0x21, 0x9e, 0x34, 0x65, 0x48, 0xec, 0x91, 0xd8, 0x60, 0x82,
-	0x2f, 0x0c, 0xb8, 0xf1, 0x48, 0x83, 0xf4, 0xf7, 0x8c, 0xe2, 0xde, 0x58, 0x26, 0xa6, 0x31, 0xb8,
-	0x17, 0xc1, 0x6f, 0x6c, 0x6e, 0x41, 0x87, 0x6e, 0x61, 0x2d, 0x56, 0x68, 0xcd, 0x44, 0x77, 0xdd,
-	0xc4, 0xce, 0xe6, 0xf5, 0xcd, 0x98, 0x7d, 0xbd, 0x19, 0xb3, 0x6f, 0x37, 0x63, 0xf6, 0xf1, 0xc7,
-	0xf8, 0x9f, 0xb3, 0x0e, 0xfd, 0x95, 0x3c, 0xf9, 0x19, 0x00, 0x00, 0xff, 0xff, 0x03, 0x56, 0xc7,
-	0xa4, 0x5a, 0x06, 0x00, 0x00,
+	0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0x08, 0x8a, 0x85, 0x50, 0xb0, 0x2c, 0x84,
+	0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0xe0, 0xb6, 0x14, 0xb1, 0x74, 0x9b,
+	0x51, 0x1b, 0xc9, 0xf1, 0x18, 0x7b, 0xac, 0x34, 0xcf, 0xc1, 0x86, 0x47, 0x60, 0xc1, 0x43, 0xb0,
+	0xec, 0x92, 0x47, 0x80, 0xf2, 0x22, 0x68, 0xee, 0x78, 0x62, 0x37, 0x95, 0x2a, 0x16, 0xec, 0xe6,
+	0x9c, 0x33, 0x73, 0x67, 0xce, 0xcc, 0xb9, 0x36, 0x6c, 0xa4, 0xc5, 0x59, 0x3c, 0x3b, 0x1f, 0xa5,
+	0x99, 0x54, 0x92, 0x77, 0x66, 0x89, 0x12, 0x59, 0x12, 0xc5, 0xc1, 0x47, 0x70, 0x50, 0x2e, 0xb8,
+	0x07, 0xed, 0x5d, 0x19, 0x17, 0xf3, 0x24, 0xf7, 0x98, 0xef, 0x84, 0x2e, 0x5a, 0xc8, 0x9f, 0x41,
+	0xf3, 0xb5, 0x52, 0x59, 0xee, 0x35, 0x7c, 0x27, 0xec, 0x8d, 0x07, 0x23, 0xbb, 0x74, 0xa4, 0x69,
+	0x34, 0x22, 0xe7, 0xe0, 0x1e, 0x8a, 0x65, 0xee, 0x39, 0xbe, 0x13, 0x76, 0x91, 0xc6, 0xc1, 0x4b,
+	0x70, 0xdf, 0x46, 0xb3, 0x8c, 0x0f, 0xa0, 0x31, 0xd9, 0xf3, 0x98, 0xcf, 0x42, 0x17, 0x1b, 0x93,
+	0x3d, 0xfe, 0x00, 0x9a, 0xbb, 0xb2, 0x48, 0x94, 0xd7, 0x20, 0xca, 0x00, 0xbe, 0x09, 0xce, 0xa1,
+	0x58, 0x7a, 0x8e, 0xcf, 0xc2, 0x2e, 0xea, 0x61, 0x30, 0x86, 0xce, 0x69, 0x14, 0xaf, 0xd4, 0xd3,
+	0x28, 0xa6, 0x22, 0x0e, 0xea, 0xe1, 0xed, 0x2a, 0x4e, 0x59, 0x25, 0x78, 0x0f, 0xce, 0xce, 0x4c,
+	0x69, 0x11, 0xe5, 0x62, 0xb5, 0xab, 0x01, 0xfc, 0x31, 0x74, 0x8c, 0xab, 0xc9, 0x5e, 0xb9, 0xf7,
+	0x0a, 0xf3, 0x27, 0xd0, 0x3d, 0x99, 0xcd, 0x45, 0xae, 0xa2, 0x79, 0x4a, 0x87, 0x70, 0xb0, 0x22,
+	0x82, 0x0f, 0xd0, 0x37, 0x33, 0xb5, 0xdb, 0x63, 0xa1, 0xee, 0x78, 0xfa, 0xbb, 0x5b, 0xba, 0xeb,
+	0xf1, 0x2b, 0x03, 0x57, 0x6b, 0x56, 0x62, 0x2b, 0x49, 0x5f, 0xe9, 0xc9, 0x32, 0x15, 0xe5, 0x49,
+	0x69, 0xcc, 0x7d, 0xe8, 0x1d, 0xab, 0x6c, 0x96, 0x5c, 0x9c, 0x46, 0x71, 0x21, 0xca, 0x42, 0x75,
+	0x4a, 0x7b, 0x9c, 0x24, 0xca, 0xc8, 0x2e, 0xd9, 0x58, 0x61, 0xed, 0x71, 0x47, 0xca, 0xd8, 0x88,
+	0x4d, 0x9f, 0x85, 0x1d, 0xac, 0x08, 0x3e, 0x04, 0x38, 0x88, 0x65, 0x54, 0xae, 0x6d, 0xf9, 0x2c,
+	0x64, 0x58, 0x63, 0x82, 0x6d, 0x68, 0xeb, 0x93, 0xbe, 0x89, 0xd2, 0xca, 0x2d, 0xbb, 0xc7, 0x6d,
+	0x70, 0xcd, 0x60, 0xe3, 0x5d, 0x21, 0xb2, 0x25, 0x8a, 0x4f, 0x85, 0xc8, 0xe9, 0x55, 0x08, 0x97,
+	0x2e, 0x0d, 0xe0, 0x5b, 0xd0, 0x3a, 0xbe, 0x8c, 0xb2, 0xa9, 0xb9, 0x3b, 0x17, 0x4b, 0xa4, 0xbd,
+	0x56, 0x77, 0x9e, 0x93, 0xd7, 0x0e, 0xd6, 0x29, 0xbd, 0x12, 0xc5, 0x5c, 0x2a, 0x6b, 0xa6, 0x44,
+	0x3c, 0x84, 0xff, 0xf7, 0xaf, 0xce, 0xe3, 0x62, 0x2a, 0x50, 0x2e, 0xcc, 0xea, 0x16, 0x4d, 0x58,
+	0xa7, 0xf9, 0x73, 0x18, 0x94, 0x94, 0x4d, 0x7f, 0x9b, 0x26, 0xae, 0xb1, 0xc1, 0x67, 0x06, 0xfd,
+	0xd2, 0x4a, 0x9e, 0xca, 0x24, 0x17, 0xfa, 0xbd, 0xf6, 0xb3, 0xcc, 0xbe, 0xd7, 0x7e, 0x96, 0xf1,
+	0x6d, 0x68, 0xa3, 0xc8, 0x8b, 0x58, 0xd9, 0x10, 0x3c, 0xac, 0xae, 0xc5, 0xae, 0x2d, 0x62, 0x85,
+	0x76, 0x16, 0x7f, 0x05, 0x83, 0x5b, 0xa1, 0x32, 0xdd, 0xd3, 0x1b, 0x3f, 0xaa, 0xd6, 0xdd, 0xd2,
+	0x71, 0x6d, 0x7a, 0xf0, 0x9d, 0x41, 0xaf, 0x56, 0x99, 0x3f, 0xa5, 0x5e, 0xa6, 0x33, 0xf5, 0xc6,
+	0xfd, 0xaa, 0x0a, 0xca, 0x05, 0x52, 0x97, 0x6f, 0x00, 0x3b, 0x2a, 0xf3, 0xc4, 0x8e, 0xf4, 0x2b,
+	0xea, 0xfe, 0xb4, 0xdb, 0xd6, 0x5e, 0x51, 0xd3, 0x68, 0x44, 0xfa, 0x32, 0x5c, 0x46, 0xc9, 0x85,
+	0x98, 0x52, 0x9e, 0x3a, 0x68, 0x21, 0x1f, 0x55, 0xfd, 0x49, 0x0f, 0xd0, 0x1b, 0xf3, 0xaa, 0x84,
+	0x55, 0xb0, 0xea, 0x61, 0x1b, 0x68, 0xfd, 0x16, 0x7d, 0x13, 0xe8, 0xe0, 0x17, 0x83, 0xfe, 0x64,
+	0x9e, 0xca, 0x4c, 0xd5, 0x42, 0x32, 0x49, 0xa6, 0xe2, 0xca, 0x86, 0x84, 0x80, 0x66, 0x0f, 0x66,
+	0x22, 0x9e, 0xd2, 0xe9, 0xbb, 0x68, 0x80, 0x66, 0x29, 0x2c, 0x14, 0x0e, 0x17, 0x0d, 0xa0, 0x58,
+	0xe8, 0x7e, 0xcf, 0x3d, 0xd7, 0x04, 0xca, 0x20, 0x1d, 0x7f, 0xdb, 0xee, 0xb9, 0xd7, 0x24, 0xa9,
+	0x22, 0x74, 0xfc, 0x57, 0xfd, 0xae, 0xf3, 0xe2, 0x84, 0x0e, 0xd6, 0x18, 0x7d, 0x0f, 0x28, 0x17,
+	0xf4, 0x91, 0x6b, 0xd3, 0x47, 0xce, 0x42, 0xbd, 0xd2, 0x94, 0x21, 0xb1, 0x43, 0x62, 0x8d, 0x09,
+	0xbe, 0x31, 0xe0, 0xc6, 0x23, 0x35, 0xd2, 0xbf, 0x33, 0x7a, 0xbf, 0xa1, 0x2d, 0x68, 0xd1, 0x7e,
+	0xd6, 0x4c, 0x89, 0xd6, 0x8e, 0xdb, 0x5e, 0x3f, 0xee, 0xce, 0xe6, 0xf5, 0xcd, 0x90, 0xfd, 0xb8,
+	0x19, 0xb2, 0x9f, 0x37, 0x43, 0xf6, 0xe5, 0xf7, 0xf0, 0xbf, 0xb3, 0x16, 0xfd, 0x34, 0x5e, 0xfc,
+	0x09, 0x00, 0x00, 0xff, 0xff, 0x67, 0xca, 0x55, 0x5d, 0x44, 0x06, 0x00, 0x00,
 }

internal/public.proto

@@ -46,7 +46,7 @@ message AttrMap {
 
 message QueryRequest {
 	string Query = 1;
-	repeated uint64 Slices = 2;
+	repeated uint64 Shards = 2;
 	bool ColumnAttrs = 3;
 	bool Remote = 5;
 	bool ExcludeRowAttrs = 6;
@@ -70,8 +70,8 @@ message QueryResult {
 
 message ImportRequest {
 	string Index = 1;
-	string Frame = 2;
-	uint64 Slice = 3;
+	string Field = 2;
+	uint64 Shard = 3;
 	repeated uint64 RowIDs = 4;
 	repeated uint64 ColumnIDs = 5;
 	repeated string RowKeys = 7;
@@ -81,9 +81,8 @@ message ImportRequest {
 
 message ImportValueRequest {
 	string Index = 1;
-	string Frame = 2;
-	uint64 Slice = 3;
-	string Field = 4;
+	string Field = 2;
+	uint64 Shard = 3;
 	repeated uint64 ColumnIDs = 5;
 	repeated string ColumnKeys = 7;
 	repeated int64 Values = 6;

internal/test/querygenerator.go

@@ -0,0 +1,190 @@
+package test
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/pilosa/pilosa/pql"
+)
+
+type Args map[string]interface{}
+
+type Calls []*pql.Call
+
+func PQL(calls ...*pql.Call) *pql.Query {
+	return &pql.Query{Calls: calls}
+}
+
+func Row(frame string, row int) *pql.Call {
+	return &pql.Call{
+		Name: "Row",
+		Args: Args{
+			"frame": frame,
+			"row":   row,
+		},
+	}
+}
+
+func mutationArgs(args ...interface{}) Args {
+	rargs := make(Args)
+	for _, arg := range args {
+		switch v := arg.(type) {
+		case int:
+			rargs["column"] = v
+		case string:
+			if strings.Contains(v, "=") {
+				parts := strings.Split(v, "=")
+				rargs["frame"] = parts[0]
+				i, _ := strconv.ParseInt(parts[1], 10, 64)
+				rargs["value"] = i
+			} else {
+				rargs["timestamp"] = v
+			}
+		default:
+			fmt.Printf("wat %T!\n", v)
+		}
+	}
+
+	return rargs
+}
+
+func Set(args ...interface{}) *pql.Call {
+	return &pql.Call{Name: "Set", Args: mutationArgs(args...)}
+}
+
+func Clear(args ...interface{}) *pql.Call {
+	return &pql.Call{Name: "Clear", Args: mutationArgs(args...)}
+}
+
+func magic(args ...interface{}) (Args, Calls) {
+	var (
+		rargs Args
+		calls Calls
+	)
+
+	for _, arg := range args {
+		switch v := arg.(type) {
+		case Args:
+			rargs = v
+		case []*pql.Call:
+			calls = append(calls, v...)
+		default:
+			fmt.Printf("wat %T!\n", v)
+		}
+	}
+
+	return rargs, calls
+}
+func Count(args ...*pql.Call) *pql.Call {
+	kvargs, children := magic(args)
+	return &pql.Call{Name: "Count", Args: kvargs, Children: children}
+}
+
+func Union(args ...*pql.Call) *pql.Call {
+	kvargs, children := magic(args)
+	return &pql.Call{Name: "Union", Args: kvargs, Children: children}
+}
+
+func Intersect(args ...*pql.Call) *pql.Call {
+	kvargs, children := magic(args)
+	return &pql.Call{Name: "Intersect", Args: kvargs, Children: children}
+}
+
+func Difference(args ...*pql.Call) *pql.Call {
+	kvargs, children := magic(args)
+	return &pql.Call{Name: "Difference", Args: kvargs, Children: children}
+}
+func Xor(args ...*pql.Call) *pql.Call {
+	kvargs, children := magic(args)
+	return &pql.Call{Name: "Xor", Args: kvargs, Children: children}
+}
+
+func Between(frame string, min, max int) *pql.Call {
+	return &pql.Call{
+		Name: "Range",
+		Args: Args{
+			"Op":    pql.BETWEEN,
+			"Value": []int{min, max},
+		},
+	}
+}
+func Lt(frame string, column int) *pql.Call {
+	return &pql.Call{
+		Name: "Range",
+		Args: Args{
+			"Op":    pql.LT,
+			"Value": column,
+		},
+	}
+}
+func Lte(frame string, column int) *pql.Call {
+	return &pql.Call{
+		Name: "Range",
+		Args: Args{
+			"Op":    pql.LTE,
+			"Value": column,
+		},
+	}
+}
+func Gt(frame string, column int) *pql.Call {
+	return &pql.Call{
+		Name: "Range",
+		Args: Args{
+			"Op":    pql.GT,
+			"Value": column,
+		},
+	}
+}
+
+func Gte(frame string, column int) *pql.Call {
+	return &pql.Call{
+		Name: "Range",
+		Args: Args{
+			"Op":    pql.GTE,
+			"Value": column,
+		},
+	}
+}
+func CompareCall(a, b *pql.Call) bool {
+	if a.Name != b.Name {
+		return false
+	}
+	for k, i := range a.Args {
+		switch v := i.(type) {
+		case []int:
+			bside := b.Args[k]
+			for j := range v {
+				if v[j] != bside.([]int)[j] {
+					return false
+				}
+
+			}
+		default:
+			if b.Args[k] != i {
+				return false
+			}
+		}
+	}
+
+	if len(a.Children) == len(b.Children) {
+		for i := range a.Children {
+			if !CompareCall(a.Children[i], b.Children[i]) {
+				return false
+			}
+		}
+	} else {
+		return false
+	}
+	return true
+}
+
+func Compare(a, b *pql.Query) bool {
+	for i := range a.Calls {
+		if !CompareCall(a.Calls[i], b.Calls[i]) {
+			return false
+		}
+
+	}
+	return true
+}

internal/test/querygenerator_test.go

@@ -0,0 +1,219 @@
+package test
+
+import (
+	"testing"
+	"github.com/pilosa/pilosa/pql"
+)
+
+
+func TestPQL_Generator(t *testing.T) {
+	t.Run("pql.Query generator", func(t *testing.T) {
+		for _, u := range []struct {
+			pql  string
+			calc *pql.Query
+			exp  *pql.Query
+		}{
+			{
+				pql:  "Union(Row(aaa=10),Row(bbb=9))",
+				calc: PQL(Union(Row("aaa", 10), Row("bbb", 9))),
+				exp: &pql.Query{
+					Calls: []*pql.Call{
+						{
+							Name: "Union",
+							Args: map[string]interface{}{},
+							Children: []*pql.Call{
+								{
+									Name: "Row",
+									Args: map[string]interface{}{"frame": "aaa", "row": 10},
+								},
+								{
+									Name: "Row",
+									Args: map[string]interface{}{"frame": "bbb", "row": 9},
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				pql:  "Intersect(Row(aaa=10),Row(bbb=9))",
+				calc: PQL(Intersect(Row("aaa", 10), Row("bbb", 9))),
+				exp: &pql.Query{
+					Calls: []*pql.Call{
+						{
+							Name: "Intersect",
+							Args: map[string]interface{}{},
+							Children: []*pql.Call{
+								{
+									Name: "Row",
+									Args: map[string]interface{}{"frame": "aaa", "row": 10},
+								},
+								{
+									Name: "Row",
+									Args: map[string]interface{}{"frame": "bbb", "row": 9},
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				pql:  "Difference(Row(aaa=10),Row(bbb=9))",
+				calc: PQL(Difference(Row("aaa", 10), Row("bbb", 9))),
+				exp: &pql.Query{
+					Calls: []*pql.Call{
+						{
+							Name: "Difference",
+							Args: map[string]interface{}{},
+							Children: []*pql.Call{
+								{
+									Name: "Row",
+									Args: map[string]interface{}{"frame": "aaa", "row": 10},
+								},
+								{
+									Name: "Row",
+									Args: map[string]interface{}{"frame": "bbb", "row": 9},
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				pql:  "Range(bbb > 20)",
+				calc: PQL(Gt("bbb", 20)),
+				exp: &pql.Query{
+					Calls: []*pql.Call{
+						{
+							Name: "Range",
+							Args: map[string]interface{}{
+								"Op":    pql.GT,
+								"Value": 20,
+							},
+						},
+					},
+				},
+			},
+			{
+				pql:  "Range(10 < bbb < 20)",
+				calc: PQL(Between("bbb", 10, 20)),
+				exp: &pql.Query{
+					Calls: []*pql.Call{
+						{
+							Name: "Range",
+							Args: map[string]interface{}{
+								"Op":    pql.BETWEEN,
+								"Value": []int{10, 20},
+							},
+						},
+					},
+				},
+			},
+			{
+				pql:  "Set(10, aaa=9)",
+				calc: PQL(Set(10, "aaa=9")),
+				exp: &pql.Query{
+					Calls: []*pql.Call{
+						{
+							Name: "Set",
+							Args: map[string]interface{}{
+								"frame":  "aaa",
+								"value":  int64(9),
+								"column": 10,
+							},
+						},
+					},
+				},
+			},
+			{
+				pql:  "Clear(10, aaa=10)",
+				calc: PQL(Clear(10, "aaa=9")),
+				exp: &pql.Query{
+					Calls: []*pql.Call{
+						{
+							Name: "Clear",
+							Args: map[string]interface{}{
+								"frame":  "aaa",
+								"value":  int64(9),
+								"column": 10,
+							},
+						},
+					},
+				},
+			},
+			{
+				pql:  `Set(10, aaa=10, "2017-03-02T03:00")`,
+				calc: PQL(Set(10, "aaa=9", "2017-03-02T03:00")),
+				exp: &pql.Query{
+					Calls: []*pql.Call{
+						{
+							Name: "Set",
+							Args: map[string]interface{}{
+								"frame":     "aaa",
+								"value":     int64(9),
+								"column":    10,
+								"timestamp": "2017-03-02T03:00",
+							},
+						},
+					},
+				},
+			},
+			{
+				pql:  `Count(Row(aaa=10))`,
+				calc: PQL(Count(Row("aaa", 10))),
+				exp: &pql.Query{
+					Calls: []*pql.Call{
+						{
+							Name: "Count",
+							Args: map[string]interface{}{},
+							Children: []*pql.Call{
+								{
+									Name: "Row",
+									Args: map[string]interface{}{"frame": "aaa", "row": 10},
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				pql:  "Intersect(Union(Row(aaa=10),Row(bbb=9)), Row(aaa=12))",
+				calc: PQL(Intersect(Union(Row("aaa", 10), Row("bbb", 9)), Row("aaa", 12))),
+				exp: &pql.Query{
+					Calls: []*pql.Call{
+						{
+							Name: "Intersect",
+							Args: map[string]interface{}{},
+							Children: []*pql.Call{
+								{
+									Name: "Union",
+									Args: map[string]interface{}{},
+									Children: []*pql.Call{
+										{
+											Name: "Row",
+											Args: map[string]interface{}{"frame": "aaa", "row": 10},
+										},
+										{
+											Name: "Row",
+											Args: map[string]interface{}{"frame": "bbb", "row": 9},
+										},
+									},
+								},
+								{
+									Name: "Row",
+									Args: map[string]interface{}{"frame": "aaa", "row": 12},
+								},
+							},
+						},
+					},
+				},
+			},
+		} {
+
+			if !Compare(u.calc, u.exp) {
+				t.Fatalf("Not Equal. expected: %v, got %v for %s", u.exp, u.calc, u.pql)
+			}
+		}
+	})
+
+}

iterator.go

@@ -20,37 +20,37 @@ import (
 	"github.com/pilosa/pilosa/roaring"
 )
 
-// Iterator is an interface for looping over row/column pairs.
-type Iterator interface {
+// iterator is an interface for looping over row/column pairs.
+type iterator interface {
 	Seek(rowID, columnID uint64)
 	Next() (rowID, columnID uint64, eof bool)
 }
 
-// BufIterator wraps an iterator to provide the ability to unread values.
-type BufIterator struct {
+// bufIterator wraps an iterator to provide the ability to unread values.
+type bufIterator struct {
 	buf struct {
 		rowID    uint64
 		columnID uint64
 		eof      bool
 		full     bool
 	}
-	itr Iterator
+	itr iterator
 }
 
-// NewBufIterator returns a buffered iterator that wraps itr.
-func NewBufIterator(itr Iterator) *BufIterator {
-	return &BufIterator{itr: itr}
+// newBufIterator returns a buffered iterator that wraps itr.
+func newBufIterator(itr iterator) *bufIterator {
+	return &bufIterator{itr: itr}
 }
 
 // Seek moves to the first pair equal to or greater than pseek/bseek.
-func (itr *BufIterator) Seek(rowID, columnID uint64) {
+func (itr *bufIterator) Seek(rowID, columnID uint64) {
 	itr.buf.full = false
 	itr.itr.Seek(rowID, columnID)
 }
 
 // Next returns the next pair in the row.
 // If a value has been buffered then it is returned and the buffer is cleared.
-func (itr *BufIterator) Next() (rowID, columnID uint64, eof bool) {
+func (itr *bufIterator) Next() (rowID, columnID uint64, eof bool) {
 	if itr.buf.full {
 		itr.buf.full = false
 		return itr.buf.rowID, itr.buf.columnID, itr.buf.eof
@@ -63,7 +63,7 @@ func (itr *BufIterator) Next() (rowID, columnID uint64, eof bool) {
 }
 
 // Peek reads the next value but leaves it on the buffer.
-func (itr *BufIterator) Peek() (rowID, columnID uint64, eof bool) {
+func (itr *bufIterator) Peek() (rowID, columnID uint64, eof bool) {
 	rowID, columnID, eof = itr.Next()
 	itr.Unread()
 	return
@@ -71,25 +71,25 @@ func (itr *BufIterator) Peek() (rowID, columnID uint64, eof bool) {
 
 // Unread pushes previous pair on to the buffer.
 // Panics if the buffer is already full.
-func (itr *BufIterator) Unread() {
+func (itr *bufIterator) Unread() {
 	if itr.buf.full {
 		panic("pilosa.BufIterator: buffer full")
 	}
 	itr.buf.full = true
 }
 
-// LimitIterator wraps an Iterator and limits it to a max column/row pair.
-type LimitIterator struct {
-	itr         Iterator
+// limitIterator wraps an Iterator and limits it to a max column/row pair.
+type limitIterator struct {
+	itr         iterator
 	maxRowID    uint64
 	maxColumnID uint64
 
 	eof bool
 }
 
-// NewLimitIterator returns a new LimitIterator.
-func NewLimitIterator(itr Iterator, maxRowID, maxColumnID uint64) *LimitIterator {
-	return &LimitIterator{
+// newLimitIterator returns a new LimitIterator.
+func newLimitIterator(itr iterator, maxRowID, maxColumnID uint64) *limitIterator {
+	return &limitIterator{
 		itr:         itr,
 		maxRowID:    maxRowID,
 		maxColumnID: maxColumnID,
@@ -97,11 +97,11 @@ func NewLimitIterator(itr Iterator, maxRowID, maxColumnID uint64) *LimitIterator
 }
 
 // Seek moves the underlying iterator to a column/row pair.
-func (itr *LimitIterator) Seek(rowID, columnID uint64) { itr.itr.Seek(rowID, columnID) }
+func (itr *limitIterator) Seek(rowID, columnID uint64) { itr.itr.Seek(rowID, columnID) }
 
 // Next returns the next row/column ID pair.
 // If the underlying iterator returns a pair higher than the max then EOF is returned.
-func (itr *LimitIterator) Next() (rowID, columnID uint64, eof bool) {
+func (itr *limitIterator) Next() (rowID, columnID uint64, eof bool) {
 	// Always return EOF once it is reached by limit or the underlying iterator.
 	if itr.eof {
 		return 0, 0, true
@@ -118,22 +118,22 @@ func (itr *LimitIterator) Next() (rowID, columnID uint64, eof bool) {
 	return rowID, columnID, false
 }
 
-// SliceIterator iterates over a pair of row/column ID slices.
-type SliceIterator struct {
+// sliceIterator iterates over a pair of row/column ID slices.
+type sliceIterator struct {
 	rowIDs    []uint64
 	columnIDs []uint64
 
 	i, n int
 }
 
-// NewSliceIterator returns an iterator to iterate over a set of row/column ID pairs.
+// newSliceIterator returns an iterator to iterate over a set of row/column ID pairs.
 // Both slices MUST have an equal length. Otherwise the function will panic.
-func NewSliceIterator(rowIDs, columnIDs []uint64) *SliceIterator {
+func newSliceIterator(rowIDs, columnIDs []uint64) *sliceIterator {
 	if len(columnIDs) != len(rowIDs) {
 		panic(fmt.Sprintf("pilosa.SliceIterator: pair length mismatch: %d != %d", len(rowIDs), len(columnIDs)))
 	}
 
-	return &SliceIterator{
+	return &sliceIterator{
 		rowIDs:    rowIDs,
 		columnIDs: columnIDs,
 
@@ -143,7 +143,7 @@ func NewSliceIterator(rowIDs, columnIDs []uint64) *SliceIterator {
 
 // Seek moves the cursor to a given pair.
 // If the pair is not found, the iterator seeks to the next pair.
-func (itr *SliceIterator) Seek(bseek, pseek uint64) {
+func (itr *sliceIterator) Seek(bseek, pseek uint64) {
 	for i := 0; i < itr.n; i++ {
 		rowID := itr.rowIDs[i]
 		columnID := itr.columnIDs[i]
@@ -159,7 +159,7 @@ func (itr *SliceIterator) Seek(bseek, pseek uint64) {
 }
 
 // Next returns the next row/column ID pair.
-func (itr *SliceIterator) Next() (rowID, columnID uint64, eof bool) {
+func (itr *sliceIterator) Next() (rowID, columnID uint64, eof bool) {
 	if itr.i >= itr.n {
 		return 0, 0, true
 	}
@@ -171,24 +171,24 @@ func (itr *SliceIterator) Next() (rowID, columnID uint64, eof bool) {
 	return rowID, columnID, false
 }
 
-// RoaringIterator converts a roaring.Iterator to output column/row pairs.
-type RoaringIterator struct {
+// roaringIterator converts a roaring.Iterator to output column/row pairs.
+type roaringIterator struct {
 	itr *roaring.Iterator
 }
 
-// NewRoaringIterator returns a new iterator wrapping itr.
-func NewRoaringIterator(itr *roaring.Iterator) *RoaringIterator {
-	return &RoaringIterator{itr: itr}
+// newRoaringIterator returns a new iterator wrapping itr.
+func newRoaringIterator(itr *roaring.Iterator) *roaringIterator {
+	return &roaringIterator{itr: itr}
 }
 
 // Seek moves the cursor to a pair matching bseek/pseek.
 // If the pair is not found then it moves to the next pair.
-func (itr *RoaringIterator) Seek(bseek, pseek uint64) {
-	itr.itr.Seek((bseek * SliceWidth) + pseek)
+func (itr *roaringIterator) Seek(bseek, pseek uint64) {
+	itr.itr.Seek((bseek * ShardWidth) + pseek)
 }
 
 // Next returns the next column/row ID pair.
-func (itr *RoaringIterator) Next() (rowID, columnID uint64, eof bool) {
+func (itr *roaringIterator) Next() (rowID, columnID uint64, eof bool) {
 	v, eof := itr.itr.Next()
-	return v / SliceWidth, v % SliceWidth, eof
+	return v / ShardWidth, v % ShardWidth, eof
 }

iterator_internal_test.go

@@ -12,19 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package pilosa_test
+package pilosa
 
 import (
 	"reflect"
 	"testing"
-
-	"github.com/pilosa/pilosa"
 )
 
 // Ensure slice iterator and iterate over a set of pairs.
 func TestSliceIterator(t *testing.T) {
 	// Initialize iterator.
-	itr := pilosa.NewSliceIterator(
+	itr := newSliceIterator(
 		[]uint64{0, 0, 2, 4},
 		[]uint64{0, 1, 0, 10},
 	)
@@ -48,7 +46,7 @@ func TestSliceIterator(t *testing.T) {
 
 // Ensure buffered iterator can unread values on to the buffer.
 func TestBufIterator(t *testing.T) {
-	itr := pilosa.NewBufIterator(pilosa.NewSliceIterator(
+	itr := newBufIterator(newSliceIterator(
 		[]uint64{0, 0, 1, 2},
 		[]uint64{1, 3, 0, 100},
 	))
@@ -77,7 +75,7 @@ func TestBufIterator_DoubleFillPanic(t *testing.T) {
 	func() {
 		defer func() { v = recover() }()
 
-		itr := pilosa.NewBufIterator(pilosa.NewSliceIterator(nil, nil))
+		itr := newBufIterator(newSliceIterator(nil, nil))
 		itr.Unread()
 		itr.Unread()
 	}()