Merge pull request #1005 from codysoyland/release-v0.8.3

Release v0.8.3

.travis.yml

@@ -20,6 +20,9 @@ deploy:
     skip_cleanup: true
     on:
       branch: master
+matrix:
+  allow_failures:
+    - go: master
 notifications:
   slack:
     secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI="

CHANGELOG.md

@@ -5,6 +5,76 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.8.3] - 2017-12-12
+
+This version contains 1 contribution from 1 contributor. There are 2 files changed, 59 insertions, and 42 deletions.
+
+### Fixed
+
+- Protect against accessing pointers to memory which was unmapped ([#1000](https://github.com/pilosa/pilosa/pull/1000))
+
+## [0.8.2] - 2017-12-05
+
+This version contains 1 contribution from 1 contributor. There are 15 files changed, 127 insertions, and 98 deletions.
+
+### Fixed
+
+- Modify initialization of HTTP client so only one instance is created ([#994](https://github.com/pilosa/pilosa/pull/994))
+
+## [0.8.1] - 2017-11-15
+
+This version contains 2 contributions from 2 contributors. There are 4 files changed, 27 insertions, and 14 deletions.
+
+### Fixed
+
+- Fix CountOpenFiles() fatal crash ([#969](https://github.com/pilosa/pilosa/pull/969))
+- Fix version check when local is greater than pilosa.com ([#968](https://github.com/pilosa/pilosa/pull/968))
+
+## [0.8.0] - 2017-11-15
+
+This version contains 31 contributions from 8 contributors. There are 84 files changed, 3,732 insertions, and 1,428 deletions.
+
+### Added
+
+- Diagnostics ([#895](https://github.com/pilosa/pilosa/pull/895))
+- Add docker-build make target for repeatable Docker-based builds ([#933](https://github.com/pilosa/pilosa/pull/933))
+- Add documentation on importing field values; fixes #924 ([#938](https://github.com/pilosa/pilosa/pull/938))
+- Add flag documentation and tests, remove "plugins.path" ([#942](https://github.com/pilosa/pilosa/pull/942))
+- Add TLS support ([#867](https://github.com/pilosa/pilosa/pull/867))
+- Add TLS cluster how to ([#898](https://github.com/pilosa/pilosa/pull/898))
+- Add support for gossip encryption ([#889](https://github.com/pilosa/pilosa/pull/889))
+- Add Recalculate Caches endpoint ([#881](https://github.com/pilosa/pilosa/pull/881))
+- Add search-friendly documentation for BSI range query syntax ([#955](https://github.com/pilosa/pilosa/pull/955))
+
+### Changed
+
+- Remove unneeded Gopkg.toml constraints and update all dependencies ([#943](https://github.com/pilosa/pilosa/pull/943))
+- Remove row and column labels in webUI ([#884](https://github.com/pilosa/pilosa/pull/884))
+- Internal Client refactoring ([#892](https://github.com/pilosa/pilosa/pull/892))
+- Remove column/row labels for input definition ([#945](https://github.com/pilosa/pilosa/pull/945))
+- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878))
+
+### Fixed
+
+- Skip permissions test when run as root. Fixes #940 ([#941](https://github.com/pilosa/pilosa/pull/941))
+- Address "connection reset" issues in client ([#934](https://github.com/pilosa/pilosa/pull/934))
+- Fix field value import: Use signed int and respect field minimum ([#919](https://github.com/pilosa/pilosa/pull/919))
+- Constrain BoltDB to version rather than specific revision ([#887](https://github.com/pilosa/pilosa/pull/887))
+- Fix bug in environment variable format ([#882](https://github.com/pilosa/pilosa/pull/882))
+- Fix overflow in differenceRunBitmap ([#949](https://github.com/pilosa/pilosa/pull/949))
+
+### Performance
+
+- Use FieldNotNull to improve efficiency of BETWEEN queries ([#874](https://github.com/pilosa/pilosa/pull/874))
+
+## [0.7.2] - 2017-11-15
+
+This version contains 1 contribution from 1 contributor. There is 1 file changed, 16 insertions, and 1 deletion.
+
+### Changed
+
+- Bump HTTP client's MaxIdleConns and MaxIdleConnsPerHost ([#920](https://github.com/pilosa/pilosa/pull/920))
+
 ## [0.7.1] - 2017-10-09
 
 This version contains 3 contributions from 3 contributors. There are 14 files changed, 221 insertions, and 52 deletions.
@@ -163,3 +233,6 @@ This version contains 53 contributions from 13 contributors (including 4 volunte
 [Unreleased]: https://github.com/pilosa/pilosa/compare/v0.5...HEAD
 [0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4
 [0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5
+[0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6
+[0.7.0]: https://github.com/pilosa/pilosa/compare/v0.6...v0.7
+[0.8.0]: https://github.com/pilosa/pilosa/compare/v0.7...v0.8

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.9.1 as builder
+FROM golang:1.9.2 as builder
 
 ARG ldflags=''
 

client.go

@@ -25,7 +25,6 @@ import (
 	"io/ioutil"
 	"log"
 	"math/rand"
-	"net"
 	"net/http"
 	"net/url"
 	"sort"
@@ -46,14 +45,13 @@ type ClientOptions struct {
 // InternalHTTPClient represents a client to the Pilosa cluster.
 type InternalHTTPClient struct {
 	defaultURI *URI
-	options    *ClientOptions
 
 	// The client to use for HTTP communication.
 	HTTPClient *http.Client
 }
 
 // NewInternalHTTPClient returns a new instance of InternalHTTPClient to connect to host.
-func NewInternalHTTPClient(host string, options *ClientOptions) (*InternalHTTPClient, error) {
+func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTTPClient, error) {
 	if host == "" {
 		return nil, ErrHostRequired
 	}
@@ -63,34 +61,14 @@ func NewInternalHTTPClient(host string, options *ClientOptions) (*InternalHTTPCl
 		return nil, err
 	}
 
-	client := NewInternalHTTPClientFromURI(uri, options)
+	client := NewInternalHTTPClientFromURI(uri, remoteClient)
 	return client, nil
 }
 
-func NewInternalHTTPClientFromURI(defaultURI *URI, options *ClientOptions) *InternalHTTPClient {
+func NewInternalHTTPClientFromURI(defaultURI *URI, remoteClient *http.Client) *InternalHTTPClient {
-	if options == nil {
-		options = &ClientOptions{}
-	}
-	transport := &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
-			DualStack: true,
-		}).DialContext,
-		MaxIdleConns:          1000,
-		MaxIdleConnsPerHost:   200,
-		IdleConnTimeout:       90 * time.Second,
-		TLSHandshakeTimeout:   10 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
-	}
-	if options.TLS != nil {
-		transport.TLSClientConfig = options.TLS
-	}
-	client := &http.Client{Transport: transport}
 	return &InternalHTTPClient{
 		defaultURI: defaultURI,
-		HTTPClient: client,
+		HTTPClient: remoteClient,
 	}
 }
 

client_test.go

@@ -18,6 +18,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"net/http"
 	"reflect"
 	"testing"
 
@@ -43,6 +44,13 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) {
 	return server, hldr
 }
 
+var defaultClient *http.Client
+
+func init() {
+	defaultClient = pilosa.GetHTTPClient(nil)
+
+}
+
 // Test distributed TopN Row count across 3 nodes.
 func TestClient_MultiNode(t *testing.T) {
 	cluster := test.NewCluster(3)
@@ -54,7 +62,7 @@ func TestClient_MultiNode(t *testing.T) {
 	}
 
 	s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
-		e := pilosa.NewExecutor(nil)
+		e := pilosa.NewExecutor(defaultClient)
 		e.Holder = hldr[0].Holder
 		e.Scheme = cluster.Nodes[0].Scheme
 		e.Host = cluster.Nodes[0].Host
@@ -62,7 +70,7 @@ func TestClient_MultiNode(t *testing.T) {
 		return e.Execute(ctx, index, query, slices, opt)
 	}
 	s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
-		e := pilosa.NewExecutor(nil)
+		e := pilosa.NewExecutor(defaultClient)
 		e.Holder = hldr[1].Holder
 		e.Scheme = cluster.Nodes[1].Scheme
 		e.Host = cluster.Nodes[1].Host
@@ -70,7 +78,7 @@ func TestClient_MultiNode(t *testing.T) {
 		return e.Execute(ctx, index, query, slices, opt)
 	}
 	s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
-		e := pilosa.NewExecutor(nil)
+		e := pilosa.NewExecutor(defaultClient)
 		e.Holder = hldr[2].Holder
 		e.Scheme = cluster.Nodes[2].Scheme
 		e.Host = cluster.Nodes[2].Host
@@ -135,9 +143,9 @@ func TestClient_MultiNode(t *testing.T) {
 
 	// Connect to each node to compare results.
 	client := make([]*test.Client, 3)
-	client[0] = test.MustNewClient(s[0].Host())
+	client[0] = test.MustNewClient(s[0].Host(), defaultClient)
-	client[1] = test.MustNewClient(s[1].Host())
+	client[1] = test.MustNewClient(s[1].Host(), defaultClient)
-	client[2] = test.MustNewClient(s[2].Host())
+	client[2] = test.MustNewClient(s[2].Host(), defaultClient)
 
 	topN := 4
 	queryRequest := &internal.QueryRequest{
@@ -218,7 +226,7 @@ func TestClient_Import(t *testing.T) {
 	s.Handler.Holder = hldr.Holder
 
 	// Send import request.
-	c := test.MustNewClient(s.Host())
+	c := test.MustNewClient(s.Host(), defaultClient)
 	if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
 		{RowID: 0, ColumnID: 1},
 		{RowID: 0, ColumnID: 5},
@@ -269,7 +277,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) {
 	s.Handler.Holder = hldr.Holder
 
 	// Send import request.
-	c := test.MustNewClient(s.Host())
+	c := test.MustNewClient(s.Host(), defaultClient)
 	if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
 		{RowID: 0, ColumnID: 1},
 		{RowID: 0, ColumnID: 5},
@@ -318,7 +326,7 @@ func TestClient_ImportValue(t *testing.T) {
 	s.Handler.Holder = hldr.Holder
 
 	// Send import request.
-	c := test.MustNewClient(s.Host())
+	c := test.MustNewClient(s.Host(), defaultClient)
 	if err := c.ImportValue(context.Background(), "i", "f", fld.Name, 0, []pilosa.FieldValue{
 		{ColumnID: 1, Value: -10},
 		{ColumnID: 2, Value: 20},
@@ -355,7 +363,7 @@ func TestClient_BackupRestore(t *testing.T) {
 	s.Handler.Cluster.Nodes[0].Host = s.Host()
 	s.Handler.Holder = hldr.Holder
 
-	c := test.MustNewClient(s.Host())
+	c := test.MustNewClient(s.Host(), defaultClient)
 
 	// Backup from frame.
 	var buf bytes.Buffer
@@ -420,7 +428,7 @@ func TestClient_BackupInverseView(t *testing.T) {
 	s.Handler.Cluster.Nodes[0].Host = s.Host()
 	s.Handler.Holder = hldr.Holder
 
-	c := test.MustNewClient(s.Host())
+	c := test.MustNewClient(s.Host(), defaultClient)
 
 	// Backup from frame.
 	var buf bytes.Buffer
@@ -457,7 +465,7 @@ func TestClient_BackupInvalidView(t *testing.T) {
 	s.Handler.Cluster.Nodes[0].Host = s.Host()
 	s.Handler.Holder = hldr.Holder
 
-	c := test.MustNewClient(s.Host())
+	c := test.MustNewClient(s.Host(), defaultClient)
 
 	// Backup from frame.
 	var buf bytes.Buffer
@@ -487,7 +495,7 @@ func TestClient_FragmentBlocks(t *testing.T) {
 	s.Handler.Holder = hldr.Holder
 
 	// Retrieve blocks.
-	c := test.MustNewClient(s.Host())
+	c := test.MustNewClient(s.Host(), defaultClient)
 	blocks, err := c.FragmentBlocks(context.Background(), "i", "f", pilosa.ViewStandard, 0)
 	if err != nil {
 		t.Fatal(err)

ctl/common.go

@@ -2,6 +2,7 @@ package ctl
 
 import (
 	"crypto/tls"
+
 	"github.com/pilosa/pilosa"
 	"github.com/spf13/pflag"
 )
@@ -22,19 +23,18 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP
 // CommandClient returns a pilosa.InternalHTTPClient for the command
 func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error) {
 	tlsConfig := cmd.TLSConfiguration()
-	var clientOptions *pilosa.ClientOptions
+	var TLSConfig *tls.Config
 	if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" {
 		cert, err := tls.LoadX509KeyPair(tlsConfig.CertificatePath, tlsConfig.CertificateKeyPath)
 		if err != nil {
 			return nil, err
 		}
-		TLSConfig := &tls.Config{
+		TLSConfig = &tls.Config{
 			Certificates:       []tls.Certificate{cert},
 			InsecureSkipVerify: tlsConfig.SkipVerify,
 		}
-		clientOptions = &pilosa.ClientOptions{TLS: TLSConfig}
 	}
-	client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), clientOptions)
+	client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), pilosa.GetHTTPClient(TLSConfig))
 	if err != nil {
 		return nil, err
 	}

diagnostics/diagnostics.go

@@ -172,9 +172,9 @@ func (d *Diagnostics) CompareVersion(value string) error {
 
 	if localVersion[0] < currentVersion[0] { //Major
 		return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value)
-	} else if localVersion[1] < currentVersion[1] { // Minor
+	} else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor
 		return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value)
-	} else if localVersion[2] < currentVersion[2] { // Patch
+	} else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch
 		return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value)
 	}
 

diagnostics/diagnostics_test.go

@@ -96,6 +96,11 @@ func TestDiagnosticsVersion_Compare(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Versions should match")
 	}
+	d.SetVersion("v1.7.0")
+	err = d.CompareVersion("0.7.2")
+	if err != nil {
+		t.Fatalf("Local version is greater")
+	}
 }
 
 func TestDiagnosticsVersion_Check(t *testing.T) {

docs/administration.md

@@ -128,21 +128,21 @@ Note: This will only work when the replication factor is >= 2
 
 Each Pilosa cluster is configured by default to share anonymous usage details with Pilosa Corp. These metrics allow us to understand how Pilosa is used by the community and improve the technology to suit your needs. Diagnostics are sent to Pilosa every hour. Each of the metrics are detailed below as well as opt-out instructions.
 
-<strong id="version">Version:</strong> Version string of the build.
+- **Version:** Version string of the build.
-<strong id="host">Host:</strong> Host URI.
+- **Host:** Host URI.
-<strong id="cluster">Cluster:</strong> List of nodes in the Cluster. 
+- **Cluster:** List of nodes in the Cluster.
-<strong id="num_nodes">NumNodes:</strong> Number of nodes in the Cluster.
+- **NumNodes:** Number of nodes in the Cluster.
-<strong id="num_cpu">NumCPU:</strong> Number of Cores per Node
+- **NumCPU:** Number of Cores per Node
-<strong id="bsa_enabled">BSIEnabled:</strong> Bit Slice Index Frames in use.
+- **BSIEnabled:** Bit Slice Index Frames in use.
-<strong id="time_quantum_enabled">TimeQuantumEnabled:</strong> Time Quantum Frames in use.
+- **TimeQuantumEnabled:** Time Quantum Frames in use.
-<strong id="inverse_enabled">InverseEnabled:</strong> Inverse Frames in use.
+- **InverseEnabled:** Inverse Frames in use.
-<strong id="num_indexes">NumIndexes:</strong> Number of Indexes in the Cluster.
+- **NumIndexes:** Number of Indexes in the Cluster.
-<strong id="num_frames">NumFrames:</strong> Number of Frames in the Cluster.
+- **NumFrames:** Number of Frames in the Cluster.
-<strong id="num_slices">NumSlices:</strong> Number of Slices in the Cluster.
+- **NumSlices:** Number of Slices in the Cluster.
-<strong id="num_views">NumViews:</strong> Number of Views in the Cluster.
+- **NumViews:** Number of Views in the Cluster.
-<strong id="open_files">OpenFiles:</strong> Open file handle count.
+- **OpenFiles:** Open file handle count.
-<strong id="go_routines">GoRoutines:</strong> Go routine count.
+- **GoRoutines:** Go routine count.
-
+ 
 You can opt-out of the Pilosa diagnostics reporting by setting either the command line configuration option `--metric.diagnostics=false`, use the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option.
 
 #### Metrics
@@ -166,42 +166,23 @@ StatsD Tags adhere to the DataDog format (key:value), and we tag the following:
 ##### Events
 We currently track the following events
 
-<strong id="index">Index:</strong> The creation of a new Index.
+- **Index:** The creation of a new Index.
-
+- **Frame:** The creation of a new Frame.
-<strong id="frame">Frame:</strong> The creation of a new Frame.
+- **MaxSlice:** The Creation of a new Slice.
-
+- **SetBit:** Count of set bits.
-<strong id="maxSlice">MaxSlice:</strong> The Creation of a new Slice.
+- **ClearBit:** Count of cleared bits.
-
+- **ImportBit:** During a bulk data import this represents the count of bits created.
-<strong id="setbit">SetBit:</strong> Count of set bits.
+- **SetRowAttrs:** Count of Attributes set per row.
-
+- **SetColumnAttrs:** Count of Attributes set per collumn.
-<strong id="clearbit">ClearBit:</strong> Count of cleared bits.
+- **Bitmap:** Count of Bitmap queries.
-
+- **TopN:** Count of TopN queries.
-<strong id="importbit">ImportBit:</strong> During a bulk data import this represents the count of bits created.
+- **Union:** Count of Union queries.
-
+- **Intersection:** Count of Intersection queries.
-<strong id="setrowattrs">SetRowAttrs:</strong> Count of Attributes set per row.
+- **Difference:** Count of Difference queries.
-
+- **Count:** Count of Count queries.
-<strong id="setcollumnattrs">SetColumnAttrs:</strong> Count of Attributes set per collumn.
+- **Range:** Count of Range queries.
-
+- **Snapshot:** Event count when the snapshot process is triggered.
-<strong id="bitmap">Bitmap:</strong> Count of Bitmap queries.
+- **BlockRepair:** Count of data blocks that were out of sync and repaired.
-
+- **Garbage Collection:** Event count when Garbage Collection occurs.
-<strong id="topn">TopN:</strong> Count of TopN queries.
+- **Goroutines:** Number of running Goroutines.
-
+- **OpenFiles:** Number of open file handles associated with running Pilosa process ID.
-<strong id="union">Union:</strong> Count of Union queries.
-
-<strong id="intersection">Intersection:</strong> Count of Intersection queries.
-
-<strong id="difference">Difference:</strong> Count of Difference queries.
-
-<strong id="count">Count:</strong> Count of Count queries.
-
-<strong id="range">Range:</strong> Count of Range queries.
-
-<strong id="snapshot">Snapshot:</strong> Event count when the snapshot process is triggered.
-
-<strong id="blockrepair">BlockRepair:</strong> Count of data blocks that were out of sync and repaired.
-
-<strong id="garbage_collection">Garbage Collection:</strong> Event count when Garbage Collection occurs.
-
-<strong id="goroutines">Goroutines:</strong> Number of running Goroutines.
-
-<strong id="openfiles">OpenFiles:</strong> Number of open file handles associated with running Pilosa process ID.

docs/examples.md

@@ -0,0 +1,347 @@
++++
+title = "Examples"
+weight = 4
+nav = [
+    "Transportation",
+    "Chemical similarity search",
+]
++++
+
+## Examples
+
+### Transportation
+
+#### Introduction
+
+New York City released an extremely detailed data set of over 1 billion taxi rides taken in the city - this data has become a popular target for analysis by tech bloggers and has been very well studied. For this reason, we thought it would be interesting to import this data to Pilosa in order to compare with other data stores and techniques on the exact same data set.
+
+Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions).
+
+We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step.
+
+After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa.
+
+#### Data Model
+
+The NYC taxi data is comprised of a number of csv files listed here: http://www.nyc.gov/html/tlc/html/about/trip_record_data.shtml. These data files have around 20 columns, about half of which are relevant to the benchmark queries we're looking at:
+
+* Distance: miles, floating point
+* Fare: dollars, floating point
+* Number of passengers: integer
+* Dropoff location: latitude and longitude, floating point
+* Pickup location: latitude and longitude, floating point
+* Dropoff time: timestamp
+* Pickup time: timestamp
+
+We import these fields, creating one or more Pilosa frames from each of them:
+
+frame	|mapping
+------------|---------------------
+cab_type	|direct map of enum int → row ID
+dist_miles	|round(dist) → row ID
+total_amount_dollars	|round(dist) → row ID
+passenger_count	|direct map of integer value → row ID
+drop_grid_id	|(lat, lon) → 100x100 rectangular grid → cell ID
+drop_year	|year(timestamp) → row ID
+drop_month	|month(timestamp) → row ID
+drop_day	|day(timestamp) → row ID
+drop_time	|time of day mapped to one of 48 half-hour buckets
+pickup_grid_id	|(lat, lon) → 100x100 rectangular grid → cell ID
+pickup_year	|year(timestamp) → row ID
+pickup_month	|month(timestamp) → row ID
+pickup_day	|day(timestamp) → row ID
+pickup_time	|time of day mapped to one of 48 half-hour buckets → row ID
+
+We also created two extra frames that represent the duration and average speed of each ride:
+
+frame	|mapping
+--------------------|-------------
+duration_minutes	|round(drop_timestamp - pickup_timestamp) → row ID
+speed_mph	|round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID
+
+#### Mapping
+
+Each column that we want to use must be mapped to a combination of frames and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities.
+
+##### 0 columns → 1 frame
+
+cab_type: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this frame. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this frame are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type frame is constant.
+
+##### 1 column → 1 frame
+
+The following three frames are mapped in a simple direct way from single columns of the original data.
+
+dist_miles: each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times.
+
+In PDK parlance, we define a Mapper, which is simply a function that returns integer row IDs. PDK has a number of predefined mappers that can be described with a few parameters. One of these is LinearFloatMapper, which applies a linear function to the input, and casts it to an integer, so the rounding is handled implicitly. In code:
+```go
+lfm := pdk.LinearFloatMapper{
+    Min: -0.5,
+    Max: 3600.5,
+    Res: 3601,
+}
+```
+
+`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three.
+
+This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the frame to use (`Frame`).
+```go
+pdk.BitMapper{
+    Frame:   "dist_miles",
+    Mapper:  lfm,
+    Parsers: []pdk.Parser{pdk.FloatParser{}},
+    Fields:  []int{fields["trip_distance"]},
+},
+```
+
+These same objects are represented in the JSON definition file:
+```go
+{
+    "Fields": {
+        "Trip_distance": 10
+    },
+    "Mappers": [
+        {
+            "Name": "lfm0",
+            "Min": -0.5,
+            "Max": 3600.5,
+            "Res": 3600
+        }
+    ],
+    "BitMappers": [
+        {
+            "Frame": "dist_miles",
+            "Mapper": {
+                "Name": "lfm0"
+            },
+            "Parsers": [
+                {"Name": "FloatParser"}
+            ],
+            "Fields": "Trip_distance"
+        }
+    ]
+}
+```
+
+Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of BitMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names to column indices. We use these names in the BitMapper definitions to keep things human-readable.
+
+**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The BitMapper definition is very similar to the previous one.
+
+**passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID.
+
+##### 1 column → multiple frames
+
+When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis.
+
+We do this by storing time data in four separate frames for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of frame "year", row 6 of frame "month", and row 24 of frame "day". 
+
+We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of frame "time_of_day".
+
+We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total frames for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time.
+
+##### Multiple columns → 1 frame
+
+The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID.
+
+We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two frames for two locations: pickup_grid_id, drop_grid_id.
+
+Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient.
+
+##### Complex mappings
+
+We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the frame `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the frame `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work:
+```go
+durm := pdk.CustomMapper{
+    Func: func(fields ...interface{}) interface{} {
+        start := fields[0].(time.Time)
+        end := fields[1].(time.Time)
+        return end.Sub(start).Minutes()
+    },
+    Mapper: lfm,
+}
+```
+
+#### Import process
+
+After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process.
+
+#### Queries
+
+Now we can run some example queries.
+
+Count per cab type can be retrieved, sorted, with a single PQL call.
+
+```
+TopN(frame=cab_type)
+```
+
+High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs.
+
+```
+TopN(frame=pickup_grid_id)
+```
+
+Average of total_amount per passenger_count can be computed with some postprocessing. We use a small number of `TopN` calls to retrieve counts of rides by passenger_count, then use those counts to compute an average.
+
+```python
+queries = ''
+pcounts = range(10)
+for i in pcounts:
+    queries += "TopN(Bitmap(id=%d, frame='passenger_count'), frame=total_amount_dollars)" % i
+resp = requests.post(qurl, data=queries)
+
+average_amounts = []
+for pcount, topn in zip(pcounts, resp.json()['results']):
+    wsum = sum([r['count'] * r['key'] for r in topn])
+    count = sum([r['count'] for r in topn])
+    average_amounts.append(float(wsum)/count)
+```
+
+For more examples and details, see this [ipython notebook](https://github.com/pilosa/notebooks/blob/master/taxi-use-case.ipynb).
+
+### Chemical similarity search
+
+#### Overview
+
+The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals.
+
+We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set.
+
+Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient.
+```
+T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B))
+```
+
+A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical.
+
+All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase
+
+#### Data model
+
+We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. 
+
+Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions.
+
+Given a SMILES encoded molecule and a similarity threshold, we want to retrieve all molecule ids (or SMILES) that have a similarity percentage greater than or equal to the similarity threshold. For example, given a molecule with:
+```
+SMILES = "IC=C1/CCC(C(=O)O1)c2cccc3ccccc23"
+threshold = 90
+```
+
+return the set of molecules that have at least a 90% similarity with the given molecule.
+
+The Inverse view swaps the rows and columns automatically to enable queries over either the chembl_id or fingerprint.
+
+Standard View is used to calculate similarity
+```
+Index: mole
+    View: Standard
+        Col: chembl_id
+            Frame: fingerprint
+                Row: position_id ("on" bit positions of a fingerprint)
+```
+
+Inverse View is used for finding chembl_id based on given SMILES. 
+From a given SMILES, we use RDKit to convert it to fingerprints with "on" bit position. From "on" bit positions,  we can search a list of chembl_ids that match the bit positions. To choose the right chembl_id, we need another query to Standard View then choose the right chembl_id which has the length that matches the given fingerprint's length after using RDKit to convert SMILES to fingerprint.
+```
+Index: mole
+    View: Inverse
+        Col: position_id ("on" bit positions of a fingerprint)
+            Frame: fingerprint
+                Row: chembl_id
+```
+
+After retrieving chembl_id from the Inverse View, we can use the Tanimoto coefficient to compare chembl_id with the entire data set of molecules. The result of this comparison is the list of `chembl_id`s that have a Tanimoto coefficient greater than the given threshold.
+
+#### Import process
+
+To import data into Pilosa, we need to get chembl_id and SMILES from SD files, convert SMILES to Morgan fingerprints, and then write chembl_id and fingerprint to Pilosa. The fastest way is to extracted chembl_id and SMILES from SD file to csv file, then use the `pilosa import` command to import the csv file into Pilosa. Since chembl_id in the SD file is always paired with CHEMBL, e.g CHEMBL6329, and because Pilosa doesn't support string keys, we will ignore CHEMBL and instead use chembl_id as an integer key.
+
+For the `mole` index, each row in the csv file has the format 'chembl_id, position_id' by running the following command from Chem-usecase:
+```
+python import_from_sdf.py -p <path_to_sdf_file> -file id_fingerprint.csv
+```
+
+
+First, follow the instruction in the [getting started]({{< ref "getting-started.md" >}}) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above.
+The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393
+```
+curl localhost:10101/index/mole \
+     -X POST \
+     -d '{"options": {"columnLabel": "position_id"}}'
+
+curl localhost:10101/index/mole/frame/fingerprint \
+     -X POST \
+     -d '{"options": {"rowLabel": "chembl_id", "inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}'
+
+```
+
+Run the following commands to import the csv data into the `mole` index:
+```
+pilosa import -d mole -f fingerprint id_fingerprint.csv
+```
+
+#### Queries
+
+Get chembl_id from a given SMILES:
+```
+python get_mol_fr_smile.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23"
+```
+
+Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get all chemlb_id that have positions are on, which following these steps:
+
+* Convert SMILES to fingerprint bit "on" positions
+
+    ```python
+    from rdkit import Chem
+    from rdkit.Chem import AllChem
+    mol=Chem.MolFromSmiles("I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23")
+    fp = list(AllChem.GetMorganFingerprintAsBitVect(mol, 2, nBits=4096).GetOnBits())
+    ```
+        
+* Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id
+
+    ```python
+    bit_maps = ["Bitmap(position_id=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp]
+    bitmap_string = ', '.join(bit_maps)
+    intersection = "Intersect(%s)" % bitmap_string
+    mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"]
+    ```
+
+* From list of chembl_id, query all "on" position from mol index, if the length of array of "on" position is matched to len(fp) then return that chembl_id, otherwise the given SMILES does not exist.
+
+    ```python    
+    for m in mole_ids:
+        mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(chembl_id=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"]
+        existed_mol = False
+        if len(mol) == len(fp):
+            found = m
+            existed_mol = True
+            break
+    ```
+
+Retrieve molecule_ids that have similarity with SMILES="I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" and similarity threshold = 70%
+```
+python similar.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" -t 70
+```
+
+Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN query to get all chemlb_id that have position is on, which following these steps:
+
+* Get chembl_id from a SMILES (steps discussed above)
+
+* Query Pilosa’s TopN to get list of similarity chembl_id
+    ```python
+    query_string = 'TopN(Bitmap(chembl_id=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)'
+    topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string)
+    ```
+
+#### Benchmark
+
+To run benchmark for specific chembl_id for different similarity threshold at percentage of [50, 70, 75, 80, 85, 90], run following command:
+```
+python benchmarks.py -id 6223
+```
+
+As Matt Swain’s blog post also did a great job using mongoDB for chemical similarity search, we compared benchmark on 500000 molecules between mongoDB aggregation framework with Pilosa.
+
+Both using the same molecule, Morgan fingerprint folded to fixed lengths of 4096 bits and were run on a MacBook Pro with a 2.8 GHz 2-core Intel Core i7 processor, memory of 16 GB 1600 MHz DDR3, single host cluster

docs/getting-started.md

@@ -155,4 +155,4 @@ curl localhost:10101/index/repository/query \
 
 ### What's Next?
 
-You can jump to [Data Model](../data-model/) for an in-depth look at Pilosa's data model, or [Query Language](../query-language/) for more details about **PQL**, the query language of Pilosa. Check out the [Tutorials](../tutorials/) for example implementations of real world use cases for Pilosa. Ready to get going in your favorite language? Have a peek at our small but expanding set of official [Client Libraries](../client-libraries/).
+You can jump to [Data Model](../data-model/) for an in-depth look at Pilosa's data model, or [Query Language](../query-language/) for more details about **PQL**, the query language of Pilosa. Check out the [Examples](../examples/) page for example implementations of real world use cases for Pilosa. Ready to get going in your favorite language? Have a peek at our small but expanding set of official [Client Libraries](../client-libraries/).

docs/howtos.md

@@ -1,214 +0,0 @@
-+++
-title = "How Tos"
-weight = 4
-nav = [
-    "how-to-setup-a-secure-pilosa-cluster"
-]
-+++
-
-## How Tos
-
-### How To Setup a Secure Pilosa Cluster
-
-#### Introduction
-
-Pilosa supports encrypting the communication between and to nodes in a cluster using TLS. In this tutorial, we will be setting up a three node Pilosa cluster running on the same computer. The same steps can be used for a multi-computer cluster but that requires setting up firewalls and other platform-specific configuration which is out of the scope of this tutorial.
-
-This tutorial assumes that you are using a UNIX-like system, such as Linux or MacOS. [Windows Subsystem for Linux (WSL)](https://msdn.microsoft.com/en-us/commandline/wsl/about) works equally well on Windows 10 systems.
-
-#### Installing Pilosa and Creating the Directory Structure
-
-If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](https://www.pilosa.com/docs/latest/installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](https://www.pilosa.com/docs/latest/installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](https://www.pilosa.com/docs/latest/installation/#build-from-source).
-
-After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line:
-```
-pilosa --help
-```
-
-Let's create a directory for the tutorial to put all of our files and switch to that directory:
-```
-mkdir $HOME/pilosa-tls-tutorial && cd $_
-```
-
-#### Creating the TLS Certificate and Gossip Key
-
-Securing a Pilosa cluster consists of securing the communication between nodes using TLS and Gossip encryption. [Pilosa Enterprise](https://www.pilosa.com/enterprise/) additionally supports authentication and other security features, but those are not covered in this tutorial.
-
-The first step is acquiring an SSL certificate. You can buy a commercial certificate or retrieve a Let's Encrypt certificiate but we will be using a self signed certificate for practical reasons. Using self-signed certificates is not recommended in production, since it makes man in the middle attacks easy.
-
-The following command creates a 2048bit self-signed wildcard certificate for `*.pilosa.local` which expires 10 years later.
-
-```
-openssl req -x509 -newkey rsa:2048 -keyout pilosa.local.key -out pilosa.local.crt -days 3650 -nodes -subj "/C=US/ST=Texas/L=Austin/O=Pilosa/OU=Com/CN=*.pilosa.local"
-```
-
-The command above creates two files in the current directory:
-* `pilosa.local.crt` is the SSL certificate.
-* `pilosa.local.key` is the private key file which must be kept as secret.
-
-Having created the SSL certificate, we can now create the gossip encryption key. Gossip encryption key file must be exactly 16, 24, or 32 bytes to select one of AES-128, AES-192, or AES-256 encryption. Reading random bytes from cryptographically secure `/dev/random` serves our purpose very well:
-```
-head -c 32 /dev/random > pilosa.local.gossip32
-```
-
-We now should have `pilosa.local.gossip32` in the current directory with 32 random bytes.
-
-#### Creating the Configuration Files
-
-Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes.
-
-Create `node1.config.toml` in the project directory and paste the following in it:
-
-```toml
-# node1.config.toml
-
-data-dir = "node1_data"
-bind = "https://01.pilosa.local:10501"
-
-[cluster]
-hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"]
-
-[tls]
-certificate = "pilosa.local.crt"
-key = "pilosa.local.key"
-skip-verify = true
-
-[gossip]
-seed = "01.pilosa.local:15000"
-port = 15000
-key = "pilosa.local.gossip32"
-```
-
-Create `node2.config.toml` in the project directory and paste the following in it:
-
-```toml
-# node2.config.toml
-
-data-dir = "node2_data"
-bind = "https://02.pilosa.local:10502"
-
-[cluster]
-hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"]
-
-[tls]
-certificate = "pilosa.local.crt"
-key = "pilosa.local.key"
-skip-verify = true
-
-[gossip]
-seed = "01.pilosa.local:15000"
-port = 16000
-key = "pilosa.local.gossip32"
-```
-
-Create `node3.config.toml` in the project directory and paste the following in it:
-
-```toml
-# node3.config.toml
-
-data-dir = "node3_data"
-bind = "https://03.pilosa.local:10503"
-
-[cluster]
-hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"]
-
-[tls]
-certificate = "pilosa.local.crt"
-key = "pilosa.local.key"
-skip-verify = true
-
-[gossip]
-seed = "01.pilosa.local:15000"
-port = 17000
-key = "pilosa.local.gossip32"
-```
-
-Here is some explanation of the configuration items:
-* `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it.
-* `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes.
-* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](https://www.pilosa.com/docs/latest/configuration/#cluster-hosts) for other settings.
-* `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers.
-* `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before.
-
-#### Final Touches Before Running the Cluster
-
-Before running the cluster, let's make sure that `01.pilosa.local`, `02.pilosa.local` and `03.pilosa.local` resolve to an IP address. If you are running the cluster on your computer, it is adequate to add them to your `/etc/hosts`. Below is one of the many ways of doing that (mind the `>>`):
-```
-sudo sh -c 'printf "\n127.0.0.1 01.pilosa.local 02.pilosa.local 03.pilosa.local\n" >> /etc/hosts'
-```
-
-Ensure we can access the hosts in the cluster:
-```
-ping -c 1 01.pilosa.local
-ping -c 1 02.pilosa.local
-ping -c 1 03.pilosa.local
-```
-
-If any of the commands above return `ping: unknown host`, make sure your `/etc/hosts` contains the failed hostname.
-
-#### Running the Cluster
-
-Let's open three terminal windows and run each node in its window. This will enable us to better observe what's happening on which node.
-
-Switch to the first terminal window, change to the project directory and start the first node:
-```
-cd $HOME/pilosa-tls-tutorial
-pilosa server -c node1.config.toml
-```
-
-Switch to the second terminal window, change to the project directory and start the second node:
-```
-cd $HOME/pilosa-tls-tutorial
-pilosa server -c node2.config.toml
-```
-
-Switch to the third terminal window, change to the project directory and start the third node:
-```
-cd $HOME/pilosa-tls-tutorial
-pilosa server -c node3.config.toml
-```
-
-Let's ensure that all three Pilosa servers are runnning and they are connected:
-```
-curl -k --ipv4 https://01.pilosa.local:10501/status
-```
-
-The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL.
-
-All nodes should be in the `UP` state:
-```
-{"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}}
-```
-
-#### Running Queries
-
-Having confirmed that our cluster is running OK, let's run a few queries. But before that, we need to create an index and a frame:
-```
-curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index -d ''
-```
-
-This will create index `sample-index` with default options. Let's create the frame now:
-```
-curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/frame/sample-frame -d ''
-```
-
-We just created frame `sample-frame` with default options.
-
-Let's run a `SetBit` query:
-```
-curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)'
-```
-
-Confirm that the bit was indeed set:
-```
-curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)'
-```
-
-The same response should be returned when querying other nodes in the cluster:
-```
-curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)'
-```
-
-#### What's Next?
-
-Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa.

docs/input-definition.md

@@ -46,7 +46,7 @@ curl localhost:10101/index/repository/input-definition/stargazer \
                  {
                      "name": "repo_id", 
                      "primaryKey": true
-                 }, 
+                 },
                  {
                      "actions": [
                          {
@@ -105,8 +105,8 @@ curl localhost:10101/index/repository/input/stargazer \
      -X POST \
      -d '[
              {
-                 "language_id": "Go", 
                  "repo_id": 91720568, 
+                 "language_id": "Go", 
                  "stargazer_id": 513114,
                  "time_value": "2017-05-18T20:40"
              }, 

docs/installation.md

@@ -74,19 +74,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
 
 1. Download the latest release:
     ```
-    curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.7.1/pilosa-v0.7.1-darwin-amd64.tar.gz
+    curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-darwin-amd64.tar.gz
     ```
 
     Other releases can be downloaded from our Releases page on Github.
 
 2. Extract the binary:
     ```
-    tar xfz pilosa-v0.7.1-darwin-amd64.tar.gz
+    tar xfz pilosa-v0.8.3-darwin-amd64.tar.gz
     ```
 
 3. Move the binary into your PATH so you can run `pilosa` from any shell:
     ```
-    cp -i pilosa-v0.7.1-darwin-amd64/pilosa /usr/local/bin
+    cp -i pilosa-v0.8.3-darwin-amd64/pilosa /usr/local/bin
     ```
 
 4. Make sure Pilosa is installed successfully:
@@ -228,19 +228,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
 
 1. To install the latest version of Pilosa, download the latest release:
     ```
-    curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.7.1/pilosa-v0.7.1-linux-amd64.tar.gz
+    curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-linux-amd64.tar.gz
     ```
 
     Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github.
 
 2. Extract the binary:
     ```
-    tar xfz pilosa-v0.7.1-linux-amd64.tar.gz
+    tar xfz pilosa-v0.8.3-linux-amd64.tar.gz
     ```
 
 3. Move the binary into your PATH so you can run `pilosa` from any shell:
     ```
-    cp -i pilosa-v0.7.1-linux-amd64/pilosa /usr/local/bin
+    cp -i pilosa-v0.8.3-linux-amd64/pilosa /usr/local/bin
     ```
 
 4. Make sure Pilosa is installed successfully:

docs/query-language.md

@@ -427,7 +427,7 @@ Returns `{{"attrs":{},"bits":[10]}`
 **Spec:**
 
 ```
-Range(<frame=STRING>, <FIELD_NAME, COMPARISON_OPERATOR, integer> )
+Range(<frame=STRING>, <FIELD_NAME, COMPARISON_OPERATOR, COMPARISON_VALUE> )
 ```
 
 **Description:**
@@ -441,7 +441,7 @@ Returns bits that are true for the comparison operator.
 **Examples:**
 
 In our source data, commitactivity was counted over the last year.
-The following Range query returns all repositories having more than 100 commits.
+The following greater-than Range query returns all repositories having more than 100 commits.
 
 ```
 Range(frame="stats", commitactivity > 100)
@@ -451,6 +451,25 @@ Returns `{{"attrs":{},"bits":[10]}`
 
 * bits are repositories which had at least 100 commits in the last year.
 
+BSI range queries support the following operators:
+
+ Operator | Name                          | Value              
+----------|-------------------------------|--------------------
+ `>`      | greater-than, GT              | integer            
+ `<`      | less-than, LT                 | integer            
+ `<=`     | less-than-or-equal-to, LTE    | integer            
+ `>=`     | greater-than-or-equal-to, GTE | integer            
+ `==`     | equal-to, EQ                  | integer            
+ `!=`     | not-equal-to, NEQ             | integer or `null`  
+ `><`     | between, BETWEEN              | [integer, integer] 
+
+The `BETWEEN` query specifies an interval with both bounds, using `><` operator, and a two-element list containing the lower and upper bounds of the interval:
+
+```
+Range(frame="stats", commitactivity >< [100, 200])
+```
+
+This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way.
 
 #### Sum
 
@@ -499,4 +518,4 @@ SetFieldValue returns `null` upon success.
 Set the number of pull requests of repository 10.
 ```
 SetFieldValue(col=10, frame="stats", pullrequests=2)
-```
+```

docs/tutorials.md

@@ -2,346 +2,213 @@
 title = "Tutorials"
 weight = 4
 nav = [
-    "Transportation",
+    "How To Setup a Secure Cluster",
-    "Chemical similarity search",
 ]
 +++
 
 ## Tutorials
 
-### Transportation
+### How To Setup a Secure Cluster
 
 #### Introduction
 
-New York City released an extremely detailed data set of over 1 billion taxi rides taken in the city - this data has become a popular target for analysis by tech bloggers and has been very well studied. For this reason, we thought it would be interesting to import this data to Pilosa in order to compare with other data stores and techniques on the exact same data set.
+Pilosa supports encrypting the communication between and to nodes in a cluster using TLS. In this tutorial, we will be setting up a three node Pilosa cluster running on the same computer. The same steps can be used for a multi-computer cluster but that requires setting up firewalls and other platform-specific configuration which is out of the scope of this tutorial.
 
-Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions).
+This tutorial assumes that you are using a UNIX-like system, such as Linux or MacOS. [Windows Subsystem for Linux (WSL)](https://msdn.microsoft.com/en-us/commandline/wsl/about) works equally well on Windows 10 systems.
 
-We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step.
+#### Installing Pilosa and Creating the Directory Structure
 
-After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa.
+If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](https://www.pilosa.com/docs/latest/installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](https://www.pilosa.com/docs/latest/installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](https://www.pilosa.com/docs/latest/installation/#build-from-source).
 
-#### Data Model
+After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line:
-
+```
-The NYC taxi data is comprised of a number of csv files listed here: http://www.nyc.gov/html/tlc/html/about/trip_record_data.shtml. These data files have around 20 columns, about half of which are relevant to the benchmark queries we're looking at:
+pilosa --help
-
-* Distance: miles, floating point
-* Fare: dollars, floating point
-* Number of passengers: integer
-* Dropoff location: latitude and longitude, floating point
-* Pickup location: latitude and longitude, floating point
-* Dropoff time: timestamp
-* Pickup time: timestamp
-
-We import these fields, creating one or more Pilosa frames from each of them:
-
-frame	|mapping
-------------|---------------------
-cab_type	|direct map of enum int → row ID
-dist_miles	|round(dist) → row ID
-total_amount_dollars	|round(dist) → row ID
-passenger_count	|direct map of integer value → row ID
-drop_grid_id	|(lat, lon) → 100x100 rectangular grid → cell ID
-drop_year	|year(timestamp) → row ID
-drop_month	|month(timestamp) → row ID
-drop_day	|day(timestamp) → row ID
-drop_time	|time of day mapped to one of 48 half-hour buckets
-pickup_grid_id	|(lat, lon) → 100x100 rectangular grid → cell ID
-pickup_year	|year(timestamp) → row ID
-pickup_month	|month(timestamp) → row ID
-pickup_day	|day(timestamp) → row ID
-pickup_time	|time of day mapped to one of 48 half-hour buckets → row ID
-
-We also created two extra frames that represent the duration and average speed of each ride:
-
-frame	|mapping
---------------------|-------------
-duration_minutes	|round(drop_timestamp - pickup_timestamp) → row ID
-speed_mph	|round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID
-
-#### Mapping
-
-Each column that we want to use must be mapped to a combination of frames and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities.
-
-##### 0 columns → 1 frame
-
-cab_type: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this frame. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this frame are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type frame is constant.
-
-##### 1 column → 1 frame
-
-The following three frames are mapped in a simple direct way from single columns of the original data.
-
-dist_miles: each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times.
-
-In PDK parlance, we define a Mapper, which is simply a function that returns integer row IDs. PDK has a number of predefined mappers that can be described with a few parameters. One of these is LinearFloatMapper, which applies a linear function to the input, and casts it to an integer, so the rounding is handled implicitly. In code:
-```go
-lfm := pdk.LinearFloatMapper{
-    Min: -0.5,
-    Max: 3600.5,
-    Res: 3601,
-}
 ```
 
-`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three.
+Let's create a directory for the tutorial to put all of our files and switch to that directory:
-
+```
-This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the frame to use (`Frame`).
+mkdir $HOME/pilosa-tls-tutorial && cd $_
-```go
-pdk.BitMapper{
-    Frame:   "dist_miles",
-    Mapper:  lfm,
-    Parsers: []pdk.Parser{pdk.FloatParser{}},
-    Fields:  []int{fields["trip_distance"]},
-},
 ```
 
-These same objects are represented in the JSON definition file:
+#### Creating the TLS Certificate and Gossip Key
-```go
-{
-    "Fields": {
-        "Trip_distance": 10
-    },
-    "Mappers": [
-        {
-            "Name": "lfm0",
-            "Min": -0.5,
-            "Max": 3600.5,
-            "Res": 3600
-        }
-    ],
-    "BitMappers": [
-        {
-            "Frame": "dist_miles",
-            "Mapper": {
-                "Name": "lfm0"
-            },
-            "Parsers": [
-                {"Name": "FloatParser"}
-            ],
-            "Fields": "Trip_distance"
-        }
-    ]
-}
-```
 
-Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of BitMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names to column indices. We use these names in the BitMapper definitions to keep things human-readable.
+Securing a Pilosa cluster consists of securing the communication between nodes using TLS and Gossip encryption. [Pilosa Enterprise](https://www.pilosa.com/enterprise/) additionally supports authentication and other security features, but those are not covered in this tutorial.
 
-**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The BitMapper definition is very similar to the previous one.
+The first step is acquiring an SSL certificate. You can buy a commercial certificate or retrieve a Let's Encrypt certificiate but we will be using a self signed certificate for practical reasons. Using self-signed certificates is not recommended in production, since it makes man in the middle attacks easy.
 
-**passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID.
+The following command creates a 2048bit self-signed wildcard certificate for `*.pilosa.local` which expires 10 years later.
-
-##### 1 column → multiple frames
-
-When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis.
-
-We do this by storing time data in four separate frames for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of frame "year", row 6 of frame "month", and row 24 of frame "day". 
-
-We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of frame "time_of_day".
-
-We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total frames for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time.
-
-##### Multiple columns → 1 frame
-
-The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID.
-
-We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two frames for two locations: pickup_grid_id, drop_grid_id.
-
-Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient.
-
-##### Complex mappings
-
-We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the frame `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the frame `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work:
-```go
-durm := pdk.CustomMapper{
-    Func: func(fields ...interface{}) interface{} {
-        start := fields[0].(time.Time)
-        end := fields[1].(time.Time)
-        return end.Sub(start).Minutes()
-    },
-    Mapper: lfm,
-}
-```
-
-#### Import process
-
-After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process.
-
-#### Queries
-
-Now we can run some example queries.
-
-Count per cab type can be retrieved, sorted, with a single PQL call.
 
 ```
-TopN(frame=cab_type)
+openssl req -x509 -newkey rsa:2048 -keyout pilosa.local.key -out pilosa.local.crt -days 3650 -nodes -subj "/C=US/ST=Texas/L=Austin/O=Pilosa/OU=Com/CN=*.pilosa.local"
 ```
 
-High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs.
+The command above creates two files in the current directory:
+* `pilosa.local.crt` is the SSL certificate.
+* `pilosa.local.key` is the private key file which must be kept as secret.
 
+Having created the SSL certificate, we can now create the gossip encryption key. Gossip encryption key file must be exactly 16, 24, or 32 bytes to select one of AES-128, AES-192, or AES-256 encryption. Reading random bytes from cryptographically secure `/dev/random` serves our purpose very well:
 ```
-TopN(frame=pickup_grid_id)
+head -c 32 /dev/random > pilosa.local.gossip32
 ```
 
-Average of total_amount per passenger_count can be computed with some postprocessing. We use a small number of `TopN` calls to retrieve counts of rides by passenger_count, then use those counts to compute an average.
+We now should have `pilosa.local.gossip32` in the current directory with 32 random bytes.
 
-```python
+#### Creating the Configuration Files
-queries = ''
-pcounts = range(10)
-for i in pcounts:
-    queries += "TopN(Bitmap(id=%d, frame='passenger_count'), frame=total_amount_dollars)" % i
-resp = requests.post(qurl, data=queries)
 
-average_amounts = []
+Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes.
-for pcount, topn in zip(pcounts, resp.json()['results']):
+
-    wsum = sum([r['count'] * r['key'] for r in topn])
+Create `node1.config.toml` in the project directory and paste the following in it:
-    count = sum([r['count'] for r in topn])
+
-    average_amounts.append(float(wsum)/count)
+```toml
+# node1.config.toml
+
+data-dir = "node1_data"
+bind = "https://01.pilosa.local:10501"
+
+[cluster]
+hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"]
+
+[tls]
+certificate = "pilosa.local.crt"
+key = "pilosa.local.key"
+skip-verify = true
+
+[gossip]
+seed = "01.pilosa.local:15000"
+port = 15000
+key = "pilosa.local.gossip32"
 ```
 
-For more examples and details, see this [ipython notebook](https://github.com/pilosa/notebooks/blob/master/taxi-use-case.ipynb).
+Create `node2.config.toml` in the project directory and paste the following in it:
 
-### Chemical similarity search
+```toml
+# node2.config.toml
 
-#### Overview
+data-dir = "node2_data"
+bind = "https://02.pilosa.local:10502"
 
-The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals.
+[cluster]
+hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"]
 
-We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set.
+[tls]
+certificate = "pilosa.local.crt"
+key = "pilosa.local.key"
+skip-verify = true
 
-Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient.
+[gossip]
-```
+seed = "01.pilosa.local:15000"
-T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B))
+port = 16000
+key = "pilosa.local.gossip32"
 ```
 
-A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical.
+Create `node3.config.toml` in the project directory and paste the following in it:
 
-All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase
+```toml
+# node3.config.toml
 
-#### Data model
+data-dir = "node3_data"
+bind = "https://03.pilosa.local:10503"
 
-We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. 
+[cluster]
+hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"]
 
-Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions.
+[tls]
+certificate = "pilosa.local.crt"
+key = "pilosa.local.key"
+skip-verify = true
 
-Given a SMILES encoded molecule and a similarity threshold, we want to retrieve all molecule ids (or SMILES) that have a similarity percentage greater than or equal to the similarity threshold. For example, given a molecule with:
+[gossip]
-```
+seed = "01.pilosa.local:15000"
-SMILES = "IC=C1/CCC(C(=O)O1)c2cccc3ccccc23"
+port = 17000
-threshold = 90
+key = "pilosa.local.gossip32"
 ```
 
-return the set of molecules that have at least a 90% similarity with the given molecule.
+Here is some explanation of the configuration items:
+* `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it.
+* `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes.
+* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](https://www.pilosa.com/docs/latest/configuration/#cluster-hosts) for other settings.
+* `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers.
+* `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before.
 
-The Inverse view swaps the rows and columns automatically to enable queries over either the chembl_id or fingerprint.
+#### Final Touches Before Running the Cluster
 
-Standard View is used to calculate similarity
+Before running the cluster, let's make sure that `01.pilosa.local`, `02.pilosa.local` and `03.pilosa.local` resolve to an IP address. If you are running the cluster on your computer, it is adequate to add them to your `/etc/hosts`. Below is one of the many ways of doing that (mind the `>>`):
 ```
-Index: mole
+sudo sh -c 'printf "\n127.0.0.1 01.pilosa.local 02.pilosa.local 03.pilosa.local\n" >> /etc/hosts'
-    View: Standard
-        Col: chembl_id
-            Frame: fingerprint
-                Row: position_id ("on" bit positions of a fingerprint)
 ```
 
-Inverse View is used for finding chembl_id based on given SMILES. 
+Ensure we can access the hosts in the cluster:
-From a given SMILES, we use RDKit to convert it to fingerprints with "on" bit position. From "on" bit positions,  we can search a list of chembl_ids that match the bit positions. To choose the right chembl_id, we need another query to Standard View then choose the right chembl_id which has the length that matches the given fingerprint's length after using RDKit to convert SMILES to fingerprint.
 ```
-Index: mole
+ping -c 1 01.pilosa.local
-    View: Inverse
+ping -c 1 02.pilosa.local
-        Col: position_id ("on" bit positions of a fingerprint)
+ping -c 1 03.pilosa.local
-            Frame: fingerprint
-                Row: chembl_id
 ```
 
-After retrieving chembl_id from the Inverse View, we can use the Tanimoto coefficient to compare chembl_id with the entire data set of molecules. The result of this comparison is the list of `chembl_id`s that have a Tanimoto coefficient greater than the given threshold.
+If any of the commands above return `ping: unknown host`, make sure your `/etc/hosts` contains the failed hostname.
 
-#### Import process
+#### Running the Cluster
 
-To import data into Pilosa, we need to get chembl_id and SMILES from SD files, convert SMILES to Morgan fingerprints, and then write chembl_id and fingerprint to Pilosa. The fastest way is to extracted chembl_id and SMILES from SD file to csv file, then use the `pilosa import` command to import the csv file into Pilosa. Since chembl_id in the SD file is always paired with CHEMBL, e.g CHEMBL6329, and because Pilosa doesn't support string keys, we will ignore CHEMBL and instead use chembl_id as an integer key.
+Let's open three terminal windows and run each node in its window. This will enable us to better observe what's happening on which node.
 
-For the `mole` index, each row in the csv file has the format 'chembl_id, position_id' by running the following command from Chem-usecase:
+Switch to the first terminal window, change to the project directory and start the first node:
 ```
-python import_from_sdf.py -p <path_to_sdf_file> -file id_fingerprint.csv
+cd $HOME/pilosa-tls-tutorial
+pilosa server -c node1.config.toml
 ```
 
-
+Switch to the second terminal window, change to the project directory and start the second node:
-First, follow the instruction in the [getting started]({{< ref "getting-started.md" >}}) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above.
-The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393
 ```
-curl localhost:10101/index/mole \
+cd $HOME/pilosa-tls-tutorial
-     -X POST \
+pilosa server -c node2.config.toml
-     -d '{"options": {"columnLabel": "position_id"}}'
-
-curl localhost:10101/index/mole/frame/fingerprint \
-     -X POST \
-     -d '{"options": {"rowLabel": "chembl_id", "inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}'
-
 ```
 
-Run the following commands to import the csv data into the `mole` index:
+Switch to the third terminal window, change to the project directory and start the third node:
 ```
-pilosa import -d mole -f fingerprint id_fingerprint.csv
+cd $HOME/pilosa-tls-tutorial
+pilosa server -c node3.config.toml
 ```
 
-#### Queries
+Let's ensure that all three Pilosa servers are runnning and they are connected:
-
-Get chembl_id from a given SMILES:
 ```
-python get_mol_fr_smile.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23"
+curl -k --ipv4 https://01.pilosa.local:10501/status
 ```
 
-Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get all chemlb_id that have positions are on, which following these steps:
+The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL.
 
-* Convert SMILES to fingerprint bit "on" positions
+All nodes should be in the `UP` state:
-
-    ```python
-    from rdkit import Chem
-    from rdkit.Chem import AllChem
-    mol=Chem.MolFromSmiles("I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23")
-    fp = list(AllChem.GetMorganFingerprintAsBitVect(mol, 2, nBits=4096).GetOnBits())
-    ```
-        
-* Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id
-
-    ```python
-    bit_maps = ["Bitmap(position_id=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp]
-    bitmap_string = ', '.join(bit_maps)
-    intersection = "Intersect(%s)" % bitmap_string
-    mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"]
-    ```
-
-* From list of chembl_id, query all "on" position from mol index, if the length of array of "on" position is matched to len(fp) then return that chembl_id, otherwise the given SMILES does not exist.
-
-    ```python    
-    for m in mole_ids:
-        mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(chembl_id=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"]
-        existed_mol = False
-        if len(mol) == len(fp):
-            found = m
-            existed_mol = True
-            break
-    ```
-
-Retrieve molecule_ids that have similarity with SMILES="I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" and similarity threshold = 70%
 ```
-python similar.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" -t 70
+{"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}}
 ```
 
-Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN query to get all chemlb_id that have position is on, which following these steps:
+#### Running Queries
 
-* Get chembl_id from a SMILES (steps discussed above)
+Having confirmed that our cluster is running OK, let's run a few queries. But before that, we need to create an index and a frame:
-
-* Query Pilosa’s TopN to get list of similarity chembl_id
-    ```python
-    query_string = 'TopN(Bitmap(chembl_id=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)'
-    topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string)
-    ```
-
-#### Benchmark
-
-To run benchmark for specific chembl_id for different similarity threshold at percentage of [50, 70, 75, 80, 85, 90], run following command:
 ```
-python benchmarks.py -id 6223
+curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index -d ''
 ```
 
-As Matt Swain’s blog post also did a great job using mongoDB for chemical similarity search, we compared benchmark on 500000 molecules between mongoDB aggregation framework with Pilosa.
+This will create index `sample-index` with default options. Let's create the frame now:
+```
+curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/frame/sample-frame -d ''
+```
 
-Both using the same molecule, Morgan fingerprint folded to fixed lengths of 4096 bits and were run on a MacBook Pro with a 2.8 GHz 2-core Intel Core i7 processor, memory of 16 GB 1600 MHz DDR3, single host cluster
+We just created frame `sample-frame` with default options.
+
+Let's run a `SetBit` query:
+```
+curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)'
+```
+
+Confirm that the bit was indeed set:
+```
+curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)'
+```
+
+The same response should be returned when querying other nodes in the cluster:
+```
+curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)'
+```
+
+#### What's Next?
+
+Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa.

executor.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"sort"
 	"time"
 
@@ -51,12 +52,9 @@ type Executor struct {
 }
 
 // NewExecutor returns a new instance of Executor.
-func NewExecutor(clientOptions *ClientOptions) *Executor {
+func NewExecutor(remoteClient *http.Client) *Executor {
-	if clientOptions == nil {
-		clientOptions = &ClientOptions{}
-	}
 	return &Executor{
-		client: NewInternalHTTPClientFromURI(nil, clientOptions),
+		client: NewInternalHTTPClientFromURI(nil, remoteClient),
 	}
 }
 
@@ -968,7 +966,7 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql
 		}
 
 		// Forward call to remote node otherwise.
-		if res, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
+		if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
 			return false, err
 		} else {
 			ret = res[0].(bool)
@@ -1074,7 +1072,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C
 		}
 
 		// Forward call to remote node otherwise.
-		if res, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
+		if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
 			return false, err
 		} else {
 			ret = res[0].(bool)
@@ -1141,7 +1139,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq
 	resp := make(chan error, len(nodes))
 	for _, node := range nodes {
 		go func(node *Node) {
-			_, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
+			_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
 			resp <- err
 		}(node)
 	}
@@ -1199,7 +1197,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
 	resp := make(chan error, len(nodes))
 	for _, node := range nodes {
 		go func(node *Node) {
-			_, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
+			_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
 			resp <- err
 		}(node)
 	}
@@ -1286,7 +1284,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
 	resp := make(chan error, len(nodes))
 	for _, node := range nodes {
 		go func(node *Node) {
-			_, err := e.exec(ctx, node, index, &pql.Query{Calls: calls}, nil, opt)
+			_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil, opt)
 			resp <- err
 		}(node)
 	}
@@ -1345,7 +1343,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
 	resp := make(chan error, len(nodes))
 	for _, node := range nodes {
 		go func(node *Node) {
-			_, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
+			_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
 			resp <- err
 		}(node)
 	}
@@ -1361,7 +1359,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
 }
 
 // exec executes a PQL query remotely for a set of slices on a node.
-func (e *Executor) exec(ctx context.Context, node *Node, index string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) {
+func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) {
 	// Encode request object.
 	pbreq := &internal.QueryRequest{
 		Query:  q.String(),
@@ -1511,7 +1509,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
 			if n.Host == e.Host {
 				resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn)
 			} else if !opt.Remote {
-				results, err := e.exec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt)
+				results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt)
 				if len(results) > 0 {
 					resp.result = results[0]
 				}

fragment.go

@@ -28,6 +28,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"sort"
 	"sync"
@@ -1677,9 +1678,9 @@ func (h *blockHasher) WriteValue(v uint64) {
 type FragmentSyncer struct {
 	Fragment *Fragment
 
-	Host          string
+	Host         string
-	Cluster       *Cluster
+	Cluster      *Cluster
-	ClientOptions *ClientOptions
+	RemoteClient *http.Client
 
 	Closing <-chan struct{}
 }
@@ -1714,7 +1715,7 @@ func (s *FragmentSyncer) SyncFragment() error {
 		}
 
 		// Retrieve remote blocks.
-		client, err := NewInternalHTTPClient(node.Host, s.ClientOptions)
+		client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
 		if err != nil {
 			return err
 		}
@@ -1793,7 +1794,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
 			return nil
 		}
 
-		client, err := NewInternalHTTPClient(node.Host, s.ClientOptions)
+		client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
 		if err != nil {
 			return err
 		}

handler.go

@@ -56,9 +56,9 @@ type Handler struct {
 	StatusHandler StatusHandler
 
 	// Local hostname & cluster configuration.
-	URI           *URI
+	URI          *URI
-	Cluster       *Cluster
+	Cluster      *Cluster
-	ClientOptions *ClientOptions
+	RemoteClient *http.Client
 
 	Router *mux.Router
 
@@ -1506,7 +1506,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
 	}
 
 	// Create a client for the remote cluster.
-	client := NewInternalHTTPClientFromURI(host, h.ClientOptions)
+	client := NewInternalHTTPClientFromURI(host, h.RemoteClient)
 
 	// Determine the maximum number of slices.
 	maxSlices, err := client.MaxSliceByIndex(r.Context())
@@ -1760,8 +1760,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	// Validation the input definition with the curent index's ColumnLabel.
+	if err := req.Validate(); err != nil {
-	if err := req.Validate(index.ColumnLabel()); err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
@@ -1908,10 +1907,9 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index,
 	for _, field := range inputDef.Fields() {
 		validFields[field.Name] = true
 		if field.PrimaryKey {
-			columnLabel := field.Name
+			value, ok := req[field.Name]
-			value, ok := req[columnLabel]
 			if !ok {
-				return nil, fmt.Errorf("columnLabel required")
+				return nil, fmt.Errorf("primary key does not exist")
 			}
 			rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float
 			if !ok {

handler_test.go

@@ -1309,7 +1309,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) {
 		t.Fatalf("unexpected body: %s", body)
 	}
 
-	// Eusure throwing error if primary field's name doesn't match columnLabel
+	// Ensure throwing error if there's no primary key
 	hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{ColumnLabel: "id"})
 	unmatchColumnBody := []byte(`
 			{
@@ -1323,8 +1323,17 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) {
 			}],
 			"fields": [
 				{
-					"name": "columnID",
+					"name": "foo",
-					"primaryKey": true
+					"actions": [
+						{
+							"frame": "cab-type",
+							"valueDestination": "mapping",
+							"valueMap": {
+								"Green": 1,
+								"Yellow": 2
+							}
+						}
+					]
 				}
 			]
 		}`)
@@ -1333,7 +1342,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) {
 	h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i1/input-definition/input1", bytes.NewBuffer(unmatchColumnBody)))
 	if w.Code != http.StatusBadRequest {
 		t.Fatalf("unexpected status code: %d", w.Code)
-	} else if body := w.Body.String(); body != pilosa.ErrInputDefinitionColumnLabel.Error()+"\n" {
+	} else if body := w.Body.String(); body != pilosa.ErrInputDefinitionHasPrimaryKey.Error()+"\n" {
 		t.Fatalf("unexpected body: %s", body)
 	}
 
@@ -1677,7 +1686,7 @@ func TestInput_JSON(t *testing.T) {
 				"distanceMiles": 8,
 				"withPet": true
 				}]`,
-			err: "columnLabel required"},
+			err: "primary key does not exist"},
 		{json: `[{
 				"id": 1,
 				"cabType": "yellow",

holder.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/http"
 	"os"
 	"path/filepath"
 	"sort"
@@ -430,9 +431,9 @@ func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.Lstd
 type HolderSyncer struct {
 	Holder *Holder
 
-	URI           *URI
+	URI          *URI
-	Cluster       *Cluster
+	Cluster      *Cluster
-	ClientOptions *ClientOptions
+	RemoteClient *http.Client
 
 	// Signals that the sync should stop.
 	Closing <-chan struct{}
@@ -518,7 +519,7 @@ func (s *HolderSyncer) syncIndex(index string) error {
 
 	// Sync with every other host.
 	for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) {
-		client, err := NewInternalHTTPClient(node.Host, s.ClientOptions)
+		client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
 		if err != nil {
 			return err
 		}
@@ -563,7 +564,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
 
 	// Sync with every other host.
 	for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) {
-		client, err := NewInternalHTTPClient(node.Host, s.ClientOptions)
+		client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
 		if err != nil {
 			return err
 		}
@@ -616,11 +617,11 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err
 
 	// Sync fragments together.
 	fs := FragmentSyncer{
-		Fragment:      frag,
+		Fragment:     frag,
-		Host:          s.URI.HostPort(),
+		Host:         s.URI.HostPort(),
-		Cluster:       s.Cluster,
+		Cluster:      s.Cluster,
-		Closing:       s.Closing,
+		Closing:      s.Closing,
-		ClientOptions: s.ClientOptions,
+		RemoteClient: s.RemoteClient,
 	}
 	if err := fs.SyncFragment(); err != nil {
 		return err

holder_test.go

@@ -320,7 +320,7 @@ func TestHolder_DeleteIndex(t *testing.T) {
 // Ensure holder can sync with a remote holder.
 func TestHolderSyncer_SyncHolder(t *testing.T) {
 	cluster := test.NewCluster(2)
-
+	client := pilosa.GetHTTPClient(nil)
 	// Create a local holder.
 	hldr0 := test.MustOpenHolder()
 	defer hldr0.Close()
@@ -332,7 +332,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 	defer s.Close()
 	s.Handler.Holder = hldr1.Holder
 	s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
-		e := pilosa.NewExecutor(nil)
+		e := pilosa.NewExecutor(client)
 		e.Holder = hldr1.Holder
 		e.Scheme = cluster.Nodes[1].Scheme
 		e.Host = cluster.Nodes[1].Host
@@ -400,9 +400,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 		t.Fatal(err)
 	}
 	syncer := pilosa.HolderSyncer{
-		Holder:  hldr0.Holder,
+		Holder:       hldr0.Holder,
-		URI:     uri,
+		URI:          uri,
-		Cluster: cluster,
+		Cluster:      cluster,
+		RemoteClient: pilosa.GetHTTPClient(nil),
 	}
 
 	if err := syncer.SyncHolder(); err != nil {

index.go

@@ -687,7 +687,8 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin
 
 	for _, fr := range pb.Frames {
 		opt := FrameOptions{
-			RowLabel:       fr.Meta.RowLabel,
+			// Deprecating row labels per #810. So, setting the default row label here.
+			RowLabel:       DefaultRowLabel,
 			InverseEnabled: fr.Meta.InverseEnabled,
 			CacheType:      fr.Meta.CacheType,
 			CacheSize:      fr.Meta.CacheSize,

index_test.go

@@ -309,14 +309,14 @@ func TestIndex_CreateInputDefinition(t *testing.T) {
 	// Create Input Definition.
 	frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
 	action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
-	fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
+	field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
-	def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
+	def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}}
 	inputDef, err := index.CreateInputDefinition(&def)
 	if err != nil {
 		t.Fatal(err)
 	} else if inputDef.Frames()[0].Name != frames.Name {
 		t.Fatalf("unexpected input definition frames %v", inputDef.Frames())
-	} else if inputDef.Fields()[0].Name != fields.Name {
+	} else if inputDef.Fields()[0].Name != field.Name {
 		t.Fatalf("unexpected input definition actions %v", inputDef.Fields())
 	}
 }

input_definition.go

@@ -90,7 +90,8 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error {
 		inputFrame := InputFrame{
 			Name: fr.Name,
 			Options: FrameOptions{
-				RowLabel:       frameMeta.RowLabel,
+				// Deprecating row labels per #810. So, setting the default row label here.
+				RowLabel:       DefaultRowLabel,
 				InverseEnabled: frameMeta.InverseEnabled,
 				CacheSize:      frameMeta.CacheSize,
 				CacheType:      frameMeta.CacheType,
@@ -100,6 +101,8 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error {
 		i.frames = append(i.frames, inputFrame)
 	}
 
+	primaryKeyGiven := false
+
 	for _, field := range pb.Fields {
 		var actions []Action
 		for _, action := range field.InputDefinitionActions {
@@ -111,6 +114,10 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error {
 			})
 		}
 
+		if field.PrimaryKey {
+			primaryKeyGiven = true
+		}
+
 		inputField := InputDefinitionField{
 			Name:       field.Name,
 			PrimaryKey: field.PrimaryKey,
@@ -119,6 +126,10 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error {
 		i.fields = append(i.fields, inputField)
 	}
 
+	if len(pb.Fields) > 0 && !primaryKeyGiven {
+		return ErrInputDefinitionHasPrimaryKey
+	}
+
 	return nil
 }
 
@@ -265,7 +276,7 @@ type InputDefinitionInfo struct {
 }
 
 // Validate the InputDefinitionInfo data.
-func (i *InputDefinitionInfo) Validate(columnLabel string) error {
+func (i *InputDefinitionInfo) Validate() error {
 	numPrimaryKey := 0
 	accountRowID := make(map[string]uint64)
 
@@ -281,6 +292,9 @@ func (i *InputDefinitionInfo) Validate(columnLabel string) error {
 
 	// Validate columnLabel and duplicate primaryKey.
 	for _, field := range i.Fields {
+		if field.Name == "" {
+			return ErrInputDefinitionNameRequired
+		}
 		for _, action := range field.Actions {
 			if err := action.Validate(); err != nil {
 				return err
@@ -298,9 +312,6 @@ func (i *InputDefinitionInfo) Validate(columnLabel string) error {
 		}
 		if field.PrimaryKey {
 			numPrimaryKey++
-			if field.Name != columnLabel {
-				return ErrInputDefinitionColumnLabel
-			}
 		} else if len(field.Actions) == 0 {
 			return ErrInputDefinitionActionRequired
 		}

input_definition_test.go

@@ -110,14 +110,14 @@ func TestActionValidation(t *testing.T) {
 	action := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}}
 	field := pilosa.InputDefinitionField{Name: "id", PrimaryKey: false, Actions: []pilosa.Action{action}}
 	info := pilosa.InputDefinitionInfo{Fields: []pilosa.InputDefinitionField{field}}
-	err := info.Validate("id")
+	err := info.Validate()
 	if err != pilosa.ErrInputDefinitionAttrsRequired {
 		t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionAttrsRequired, err)
 	}
 
 	frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}}
 	info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
-	err = info.Validate("id")
+	err = info.Validate()
 	if !strings.Contains(err.Error(), "rowID required for single-row-boolean") {
 		t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err)
 	}
@@ -126,7 +126,7 @@ func TestActionValidation(t *testing.T) {
 	action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
 	field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
 	info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
-	err = info.Validate("id")
+	err = info.Validate()
 	if err != pilosa.ErrName {
 		t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrName, err)
 	}
@@ -135,23 +135,15 @@ func TestActionValidation(t *testing.T) {
 	action = pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
 	field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
 	info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
-	err = info.Validate("id")
+	err = info.Validate()
 	if err != pilosa.ErrFrameRequired {
 		t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrFrameRequired, err)
 	}
 
-	action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
-	field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
-	info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
-	err = info.Validate("test")
-	if err != pilosa.ErrInputDefinitionColumnLabel {
-		t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionColumnLabel, err)
-	}
-
 	action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
 	field = pilosa.InputDefinitionField{Name: "x", PrimaryKey: false, Actions: []pilosa.Action{action}}
 	info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
-	err = info.Validate("id")
+	err = info.Validate()
 	if err != pilosa.ErrInputDefinitionHasPrimaryKey {
 		t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionHasPrimaryKey, err)
 	}
@@ -159,7 +151,7 @@ func TestActionValidation(t *testing.T) {
 	action = pilosa.Action{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}}
 	field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
 	info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
-	err = info.Validate("id")
+	err = info.Validate()
 	if !strings.Contains(err.Error(), "invalid ValueDestination") {
 		t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err)
 	}
@@ -167,7 +159,7 @@ func TestActionValidation(t *testing.T) {
 	action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputMapping, RowID: &rowID}
 	field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
 	info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
-	err = info.Validate("id")
+	err = info.Validate()
 	if err != pilosa.ErrInputDefinitionValueMap {
 		t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionValueMap, err)
 	}
@@ -177,7 +169,7 @@ func TestActionValidation(t *testing.T) {
 	action1 := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
 	field1 := pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action1}}
 	info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field, field1}}
-	err = info.Validate("id")
+	err = info.Validate()
 	if !strings.Contains(err.Error(), "duplicate rowID with other field") {
 		t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err)
 	}
@@ -185,7 +177,7 @@ func TestActionValidation(t *testing.T) {
 	field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true}
 	field1 = pilosa.InputDefinitionField{Name: "test", PrimaryKey: false}
 	info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field, field1}}
-	err = info.Validate("id")
+	err = info.Validate()
 	if err != pilosa.ErrInputDefinitionActionRequired {
 		t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionActionRequired, err)
 	}

roaring/roaring.go

@@ -665,13 +665,19 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		c := b.containers[i]
 		switch c.container_type {
 		case ContainerRun:
+			c.array = nil
+			c.bitmap = nil
 			runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize])
 			c.runs = (*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount]
 			opsOffset = int(offset) + runCountHeaderSize + len(c.runs)*interval16Size
 		case ContainerArray:
+			c.runs = nil
+			c.bitmap = nil
 			c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n]
 			opsOffset = int(offset) + len(c.array)*2 // sizeof(uint32)
 		case ContainerBitmap:
+			c.array = nil
+			c.runs = nil
 			c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN]
 			opsOffset = int(offset) + len(c.bitmap)*8 // sizeof(uint64)
 		}
@@ -1019,17 +1025,16 @@ func (c *container) unmap() {
 		return
 	}
 
-	if c.array != nil {
+	switch c.container_type {
+	case ContainerArray:
 		tmp := make([]uint16, len(c.array))
 		copy(tmp, c.array)
 		c.array = tmp
-	}
+	case ContainerBitmap:
-	if c.bitmap != nil {
 		tmp := make([]uint64, len(c.bitmap))
 		copy(tmp, c.bitmap)
 		c.bitmap = tmp
-	}
+	case ContainerRun:
-	if c.runs != nil {
 		tmp := make([]interval16, len(c.runs))
 		copy(tmp, c.runs)
 		c.runs = tmp
@@ -1614,21 +1619,17 @@ func (c *container) runToArray() {
 func (c *container) clone() *container {
 	other := &container{n: c.n, container_type: c.container_type}
 
-	if c.array != nil {
+	switch c.container_type {
+	case ContainerArray:
 		other.array = make([]uint16, len(c.array))
 		copy(other.array, c.array)
-	}
+	case ContainerBitmap:
-
-	if c.bitmap != nil {
 		other.bitmap = make([]uint64, len(c.bitmap))
 		copy(other.bitmap, c.bitmap)
-	}
+	case ContainerRun:
-
-	if c.runs != nil {
 		other.runs = make([]interval16, len(c.runs))
 		copy(other.runs, c.runs)
 	}
-
 	return other
 }
 
@@ -2508,17 +2509,25 @@ func differenceRunArray(a, b *container) *container {
 // differenceRunBitmap computes the difference of an run from a bitmap.
 func differenceRunBitmap(a, b *container) *container {
 	// If a is full, difference is the flip of b.
-	if a.runs[0].start == 0 && a.runs[0].last == 65535 {
+	if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 {
 		return b.flipBitmap()
 	}
 	output := &container{container_type: ContainerRun}
 	output.n = a.n
+	if len(a.runs) == 0 {
+		return output
+	}
 	for j := 0; j < len(a.runs); j++ {
 		run := a.runs[j]
+		add := true
 		for bit := a.runs[j].start; bit <= a.runs[j].last; bit++ {
 			if b.bitmapContains(bit) {
 				output.n--
 				if run.start == bit {
+					if bit == 65535 { //overflow
+						add = false
+					}
+
 					run.start++
 				} else if bit == run.last {
 					run.last--
@@ -2534,10 +2543,15 @@ func differenceRunBitmap(a, b *container) *container {
 					break
 				}
 			}
+
+			if bit == 65535 { //overflow
+				break
+			}
 		}
 		if run.start <= run.last {
-			output.runs = append(output.runs, run)
+			if add {
-
+				output.runs = append(output.runs, run)
+			}
 		}
 	}
 

roaring/roaring_internal_test.go

@@ -347,8 +347,8 @@ func TestRunMax(t *testing.T) {
 }
 
 func TestIntersectionCountArrayRun(t *testing.T) {
-	a := &container{array: []uint16{1, 5, 10, 11, 12}}
+	a := &container{container_type: ContainerArray, array: []uint16{1, 5, 10, 11, 12}}
-	b := &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
+	b := &container{container_type: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
 
 	ret := intersectionCountArrayRun(a, b)
 	if ret != 3 {
@@ -357,16 +357,16 @@ func TestIntersectionCountArrayRun(t *testing.T) {
 }
 
 func TestIntersectionCountBitmapRun(t *testing.T) {
-	a := &container{bitmap: []uint64{0x8000000000000000}}
+	a := &container{container_type: ContainerBitmap, bitmap: []uint64{0x8000000000000000}}
-	b := &container{runs: []interval16{{start: 63, last: 64}}}
+	b := &container{container_type: ContainerRun, runs: []interval16{{start: 63, last: 64}}}
 
 	ret := intersectionCountBitmapRun(a, b)
 	if ret != 1 {
 		t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap, b.runs, ret)
 	}
 
-	a = &container{bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}}
+	a = &container{container_type: ContainerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}}
-	b = &container{runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}}
+	b = &container{container_type: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}}
 
 	ret = intersectionCountBitmapRun(a, b)
 	if ret != 14 {
@@ -414,6 +414,8 @@ func TestIntersectionCountRunRun(t *testing.T) {
 			bruns: []interval16{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6},
 	}
 	for i, test := range tests {
+		a.container_type = ContainerRun
+		b.container_type = ContainerRun
 		a.runs = test.aruns
 		b.runs = test.bruns
 		ret := intersectionCountRunRun(a, b)
@@ -454,6 +456,8 @@ func TestIntersectArrayRun(t *testing.T) {
 	}
 
 	for i, test := range tests {
+		a.container_type = ContainerArray
+		b.container_type = ContainerRun
 		a.array = test.array
 		b.runs = test.runs
 		ret := intersectArrayRun(a, b)
@@ -510,6 +514,8 @@ func TestIntersectRunRun(t *testing.T) {
 		},
 	}
 	for i, test := range tests {
+		a.container_type = ContainerRun
+		b.container_type = ContainerRun
 		a.runs = test.aruns
 		b.runs = test.bruns
 		ret := intersectRunRun(a, b)
@@ -573,6 +579,8 @@ func TestIntersectBitmapRunBitmap(t *testing.T) {
 		for i, v := range test.exp {
 			exp[i] = v
 		}
+		a.container_type = ContainerBitmap
+		b.container_type = ContainerRun
 		ret := intersectBitmapRun(a, b)
 		if ret.isArray() {
 			ret.arrayToBitmap()
@@ -632,6 +640,8 @@ func TestIntersectBitmapRunArray(t *testing.T) {
 			a.bitmap[i] = v
 		}
 		b.runs = test.runs
+		a.container_type = ContainerBitmap
+		b.container_type = ContainerRun
 		ret := intersectBitmapRun(a, b)
 		if !reflect.DeepEqual(ret.array, test.exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
@@ -873,6 +883,8 @@ func TestUnionRunRun(t *testing.T) {
 	for i, test := range tests {
 		a.runs = test.aruns
 		b.runs = test.bruns
+		a.container_type = ContainerRun
+		b.container_type = ContainerRun
 		ret := unionRunRun(a, b)
 		if !reflect.DeepEqual(ret.runs, test.exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
@@ -913,6 +925,8 @@ func TestUnionArrayRun(t *testing.T) {
 	for i, test := range tests {
 		a.array = test.array
 		b.runs = test.runs
+		a.container_type = ContainerArray
+		b.container_type = ContainerRun
 		ret := unionArrayRun(a, b)
 		if !reflect.DeepEqual(ret.array, test.exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
@@ -921,7 +935,7 @@ func TestUnionArrayRun(t *testing.T) {
 }
 
 func TestBitmapSetRange(t *testing.T) {
-	c := &container{bitmap: make([]uint64, bitmapN)}
+	c := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
 	tests := []struct {
 		bitmap []uint64
 		start  uint64
@@ -961,7 +975,7 @@ func TestBitmapSetRange(t *testing.T) {
 }
 
 func TestArrayToBitmap(t *testing.T) {
-	a := &container{}
+	a := &container{container_type: ContainerArray}
 	tests := []struct {
 		array []uint16
 		exp   []uint64
@@ -992,7 +1006,7 @@ func TestArrayToBitmap(t *testing.T) {
 }
 
 func TestBitmapToArray(t *testing.T) {
-	a := &container{}
+	a := &container{container_type: ContainerBitmap}
 	tests := []struct {
 		bitmap []uint64
 		exp    []uint16
@@ -1023,7 +1037,7 @@ func TestBitmapToArray(t *testing.T) {
 }
 
 func TestRunToBitmap(t *testing.T) {
-	a := &container{}
+	a := &container{container_type: ContainerRun}
 	tests := []struct {
 		runs []interval16
 		exp  []uint64
@@ -1077,7 +1091,7 @@ func getFullBitmap() []uint64 {
 }
 
 func TestBitmapToRun(t *testing.T) {
-	a := &container{}
+	a := &container{container_type: ContainerBitmap}
 	tests := []struct {
 		bitmap []uint64
 		exp    []interval16
@@ -1155,7 +1169,7 @@ func TestBitmapToRun(t *testing.T) {
 }
 
 func TestArrayToRun(t *testing.T) {
-	a := &container{}
+	a := &container{container_type: ContainerArray}
 	tests := []struct {
 		array []uint16
 		exp   []interval16
@@ -1189,7 +1203,7 @@ func TestArrayToRun(t *testing.T) {
 }
 
 func TestRunToArray(t *testing.T) {
-	a := &container{}
+	a := &container{container_type: ContainerRun}
 	tests := []struct {
 		runs []interval16
 		exp  []uint16
@@ -1223,7 +1237,7 @@ func TestRunToArray(t *testing.T) {
 }
 
 func TestBitmapZeroRange(t *testing.T) {
-	c := &container{bitmap: make([]uint64, bitmapN)}
+	c := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
 	tests := []struct {
 		bitmap []uint64
 		start  uint64
@@ -1267,8 +1281,8 @@ func TestBitmapZeroRange(t *testing.T) {
 }
 
 func TestUnionBitmapRun(t *testing.T) {
-	a := &container{bitmap: make([]uint64, bitmapN)}
+	a := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
-	b := &container{}
+	b := &container{container_type: ContainerRun}
 	tests := []struct {
 		bitmap []uint64
 		runs   []interval16
@@ -1288,6 +1302,7 @@ func TestUnionBitmapRun(t *testing.T) {
 		}
 		a.n = a.bitmapCountRange(0, 65535)
 		b.runs = test.runs
+		b.n = b.runCountRange(0, 65535)
 		ret := unionBitmapRun(a, b)
 		if ret.isArray() {
 			ret.arrayToBitmap()
@@ -1305,7 +1320,7 @@ func TestUnionBitmapRun(t *testing.T) {
 }
 
 func TestBitmapCountRuns(t *testing.T) {
-	c := &container{bitmap: make([]uint64, bitmapN)}
+	c := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
 	tests := []struct {
 		bitmap []uint64
 		exp    int
@@ -1355,7 +1370,7 @@ func TestBitmapCountRuns(t *testing.T) {
 }
 
 func TestArrayCountRuns(t *testing.T) {
-	c := &container{}
+	c := &container{container_type: ContainerArray}
 	tests := []struct {
 		array []uint16
 		exp   int
@@ -1396,8 +1411,8 @@ func TestArrayCountRuns(t *testing.T) {
 }
 
 func TestDifferenceArrayRun(t *testing.T) {
-	a := &container{}
+	a := &container{container_type: ContainerArray}
-	b := &container{}
+	b := &container{container_type: ContainerRun}
 	tests := []struct {
 		array []uint16
 		runs  []interval16
@@ -1422,8 +1437,8 @@ func TestDifferenceArrayRun(t *testing.T) {
 }
 
 func TestDifferenceRunArray(t *testing.T) {
-	a := &container{}
+	a := &container{container_type: ContainerRun}
-	b := &container{}
+	b := &container{container_type: ContainerArray}
 	tests := []struct {
 		runs  []interval16
 		array []uint16
@@ -1485,10 +1500,16 @@ func MakeBitmap(start []uint64) []uint64 {
 	}
 	return b
 }
+func MakeLastBitSet() []uint64 {
+	obj := NewBitmap(65535)
+	c := obj.container(0)
+	c.arrayToBitmap()
+	return c.bitmap
+}
 
 func TestDifferenceRunBitmap(t *testing.T) {
-	a := &container{}
+	a := &container{container_type: ContainerRun}
-	b := &container{bitmap: make([]uint64, bitmapN)}
+	b := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
 	tests := []struct {
 		runs   []interval16
 		bitmap []uint64
@@ -1524,14 +1545,24 @@ func TestDifferenceRunBitmap(t *testing.T) {
 			bitmap: MakeBitmap([]uint64{0x0, 0x8000000000000000}),
 			exp:    []interval16{{start: 0, last: 65}},
 		},
+		{
+			runs:   []interval16{{start: 1, last: 65535}},
+			bitmap: MakeBitmap([]uint64{0x0000000000000001}),
+			exp:    []interval16{{start: 1, last: 65535}},
+		},
+		{
+			runs:   []interval16{{start: 0, last: 65533}, {start: 65535, last: 65535}},
+			bitmap: MakeLastBitSet(),
+			exp:    []interval16{{start: 0, last: 65533}},
+		},
 	}
 	for i, test := range tests {
 		a.runs = test.runs
-		a.n = a.runCountRange(0, 100)
+		a.n = a.runCountRange(0, 65536)
 		for i, v := range test.bitmap {
 			b.bitmap[i] = v
 		}
-		b.n = b.bitmapCountRange(0, 100)
+		b.n = b.bitmapCountRange(0, 65536)
 		ret := differenceRunBitmap(a, b)
 		if !reflect.DeepEqual(ret.runs, test.exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
@@ -1540,8 +1571,8 @@ func TestDifferenceRunBitmap(t *testing.T) {
 }
 
 func TestDifferenceBitmapRun(t *testing.T) {
-	a := &container{bitmap: make([]uint64, bitmapN)}
+	a := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
-	b := &container{}
+	b := &container{container_type: ContainerRun}
 	tests := []struct {
 		bitmap []uint64
 		runs   []interval16
@@ -1568,7 +1599,7 @@ func TestDifferenceBitmapRun(t *testing.T) {
 }
 
 func TestDifferenceBitmapArray(t *testing.T) {
-	b := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap}
+	b := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
 	a := &container{container_type: ContainerArray}
 	tests := []struct {
 		bitmap []uint64
@@ -2497,8 +2528,9 @@ func TestSearc64(t *testing.T) {
 }
 
 func TestIntersectArrayBitmap(t *testing.T) {
-	a, b := &container{}, &container{
+	a, b := &container{container_type: ContainerArray}, &container{
-		bitmap: make([]uint64, bitmapN),
+		container_type: ContainerBitmap,
+		bitmap:         make([]uint64, bitmapN),
 	}
 	tests := []struct {
 		array  []uint16

server.go

@@ -58,6 +58,7 @@ type Server struct {
 	Handler           *Handler
 	Broadcaster       Broadcaster
 	BroadcastReceiver BroadcastReceiver
+	RemoteClient      *http.Client
 
 	// Cluster configuration.
 	// Host is replaced with actual host after opening if port is ":0".
@@ -167,10 +168,10 @@ func (s *Server) Open() error {
 	}
 
 	// Create default HTTP client
-	s.createDefaultClient()
+	s.createDefaultClient(s.RemoteClient)
 
 	// Create executor for executing queries.
-	e := NewExecutor(&ClientOptions{TLS: s.TLS})
+	e := NewExecutor(s.RemoteClient)
 	e.Holder = s.Holder
 	e.Scheme = s.URI.Scheme()
 	e.Host = s.URI.HostPort()
@@ -229,6 +230,25 @@ func (s *Server) Addr() net.Addr {
 	}
 	return s.ln.Addr()
 }
+func GetHTTPClient(t *tls.Config) *http.Client {
+	transport := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		MaxIdleConns:          1000,
+		MaxIdleConnsPerHost:   200,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+	if t != nil {
+		transport.TLSClientConfig = t
+	}
+	return &http.Client{Transport: transport}
+}
 
 // Logger returns a logger that writes to LogOutput
 func (s *Server) Logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) }
@@ -256,7 +276,7 @@ func (s *Server) monitorAntiEntropy() {
 		syncer.URI = s.URI
 		syncer.Cluster = s.Cluster
 		syncer.Closing = s.closing
-		syncer.ClientOptions = &ClientOptions{TLS: s.TLS}
+		syncer.RemoteClient = s.RemoteClient
 
 		// Sync holders.
 		if err := syncer.SyncHolder(); err != nil {
@@ -534,7 +554,10 @@ func (s *Server) monitorDiagnostics() {
 		s.diagnostics.Set("NumIndexes", len(s.Holder.Indexes()))
 		s.diagnostics.Set("NumFrames", numFrames)
 		s.diagnostics.Set("NumSlices", numSlices)
-		s.diagnostics.Set("OpenFiles", CountOpenFiles())
+		openFiles, err := CountOpenFiles()
+		if err == nil {
+			s.diagnostics.Set("OpenFiles", openFiles)
+		}
 		s.diagnostics.Set("GoRoutines", runtime.NumGoroutine())
 		s.diagnostics.CheckVersion()
 		s.diagnostics.Flush()
@@ -584,8 +607,11 @@ func (s *Server) monitorRuntime() {
 		// Record the number of go routines.
 		s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0)
 
+		openFiles, err := CountOpenFiles()
 		// Open File handles.
-		s.Holder.Stats.Gauge("OpenFiles", float64(CountOpenFiles()), 1.0)
+		if err == nil {
+			s.Holder.Stats.Gauge("OpenFiles", float64(openFiles), 1.0)
+		}
 
 		// Runtime memory metrics.
 		runtime.ReadMemStats(&m)
@@ -597,35 +623,29 @@ func (s *Server) monitorRuntime() {
 	}
 }
 
-func (s *Server) createDefaultClient() {
+func (s *Server) createDefaultClient(remoteClient *http.Client) {
-	transport := &http.Transport{}
+	s.defaultClient = NewInternalHTTPClientFromURI(nil, remoteClient)
-	if s.TLS != nil {
-		transport.TLSClientConfig = s.TLS
-	}
-	s.defaultClient = NewInternalHTTPClientFromURI(nil, &ClientOptions{TLS: s.TLS})
 }
 
 // CountOpenFiles on operating systems that support lsof.
-func CountOpenFiles() int {
+func CountOpenFiles() (int, error) {
-	count := 0
-
 	switch runtime.GOOS {
 	case "darwin", "linux", "unix", "freebsd":
 		// -b option avoid kernel blocks
 		pid := os.Getpid()
 		out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -b -p %v", pid)).Output()
 		if err != nil {
-			log.Fatal(err)
+			return 0, fmt.Errorf("calling lsof: %s", err)
 		}
 		// only count lines with our pid, avoiding warning messages from -b
 		lines := strings.Split(string(out), strconv.Itoa(pid))
-		count = len(lines)
+		return len(lines), nil
 	case "windows":
 		// TODO: count open file handles on windows
+		return 0, errors.New("CountOpenFiles() on Windows is not supported")
 	default:
-
+		return 0, errors.New("CountOpenFiles() on this OS is not supported")
 	}
-	return count
 }
 
 // StatusHandler specifies two methods which an object must implement to share

server/server.go

@@ -31,10 +31,11 @@ import (
 
 	"crypto/tls"
 
+	"io/ioutil"
+
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/gossip"
 	"github.com/pilosa/pilosa/statsd"
-	"io/ioutil"
 )
 
 func init() {
@@ -161,6 +162,7 @@ func (m *Command) SetupServer() error {
 	m.Server.MaxWritesPerRequest = m.Config.MaxWritesPerRequest
 
 	// Setup TLS
+	var TLSConfig *tls.Config
 	if uri.Scheme() == "https" {
 		if m.Config.TLS.CertificatePath == "" {
 			return errors.New("certificate path is required for TLS sockets")
@@ -176,8 +178,15 @@ func (m *Command) SetupServer() error {
 			Certificates:       []tls.Certificate{cert},
 			InsecureSkipVerify: m.Config.TLS.SkipVerify,
 		}
-		m.Server.Handler.ClientOptions = &pilosa.ClientOptions{TLS: m.Server.TLS}
+
+		// TODO Review this location
+
+		TLSConfig = m.Server.TLS
+
 	}
+	c := pilosa.GetHTTPClient(TLSConfig)
+	m.Server.RemoteClient = c
+	m.Server.Handler.RemoteClient = c
 
 	// Set internal port (string).
 	gossipPortStr := pilosa.DefaultGossipPort

server/server_test.go

@@ -25,7 +25,6 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"path/filepath"
 	"reflect"
 	"runtime"
 	"sort"
@@ -33,11 +32,9 @@ import (
 	"strings"
 	"testing"
 	"testing/quick"
-	"time"
 
 	"github.com/BurntSushi/toml"
 	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/gossip"
 	"github.com/pilosa/pilosa/server"
 	"github.com/pilosa/pilosa/test"
 )
@@ -53,7 +50,7 @@ func TestMain_Set_Quick(t *testing.T) {
 		defer m.Close()
 
 		// Create client.
-		client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), nil)
+		client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -326,7 +323,7 @@ func TestMain_FrameRestore(t *testing.T) {
 	defer m2.Close()
 
 	// Import from first cluster.
-	client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), nil)
+	client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
 	if err != nil {
 		t.Fatal(err)
 	} else if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
@@ -375,30 +372,19 @@ func tempMkdir(t *testing.T) string {
 // Ensure the file handle count is working
 func TestCountOpenFiles(t *testing.T) {
 	// Windows is not supported yet
-	supported := []string{"darwin", "linux", "unix", "freebsd"}
+	if runtime.GOOS == "windows" {
-	sort.Strings(supported)
+		t.Skip("Skipping unsupported CountOpenFiles test on Windows.")
-	i := sort.Search(len(supported),
-		func(i int) bool { return supported[i] >= runtime.GOOS })
-	if i == len(supported) {
-		return
 	}
-
+	count, err := pilosa.CountOpenFiles()
-	// Create directory store temp file
-	testDir := tempMkdir(t)
-	defer os.RemoveAll(testDir)
-
-	count := pilosa.CountOpenFiles()
-	testFile := filepath.Join(testDir, "test.txt")
-	_, err := os.Create(testFile)
 	if err != nil {
-		t.Fatalf("create test file failed: %s", err)
+		t.Errorf("CountOpenFiles failed: %s", err)
 	}
-
+	if count == 0 {
-	if pilosa.CountOpenFiles() < count+1 {
+		t.Error("CountOpenFiles returned invalid value 0.")
-		t.Error("Invalid open file handle count")
 	}
 }
 
+/* TODO: Fix this test. See #951.
 // Ensure program can send/receive broadcast messages.
 func TestMain_SendReceiveMessage(t *testing.T) {
 	m0 := MustRunMain()
@@ -580,6 +566,7 @@ func TestMain_SendReceiveMessage(t *testing.T) {
 		t.Fatal("frame not found")
 	}
 }
+*/
 
 // availablePorts returns a slice of ports that can be used for testing.
 func availablePorts(cnt int) ([]string, error) {
@@ -685,7 +672,7 @@ func (m *Main) URL() string { return "http://" + m.Server.Addr().String() }
 
 // Client returns a client to connect to the program.
 func (m *Main) Client() *pilosa.InternalHTTPClient {
-	client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), nil)
+	client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
 	if err != nil {
 		panic(err)
 	}

test/client.go

@@ -1,6 +1,8 @@
 package test
 
 import (
+	"net/http"
+
 	"github.com/pilosa/pilosa"
 )
 
@@ -10,8 +12,8 @@ type Client struct {
 }
 
 // MustNewClient returns a new instance of Client. Panic on error.
-func MustNewClient(host string) *Client {
+func MustNewClient(host string, h *http.Client) *Client {
-	c, err := pilosa.NewInternalHTTPClient(host, nil)
+	c, err := pilosa.NewInternalHTTPClient(host, h)
 	if err != nil {
 		panic(err)
 	}

test/executor.go

@@ -1,6 +1,7 @@
 package test
 
 import (
+	"net/http"
 	"strings"
 
 	"github.com/pilosa/pilosa"
@@ -12,10 +13,16 @@ type Executor struct {
 	*pilosa.Executor
 }
 
+var remoteClient *http.Client
+
+func init() {
+	remoteClient = pilosa.GetHTTPClient(nil)
+}
+
 // NewExecutor returns a new instance of Executor.
 // The executor always matches the hostname of the first cluster node.
 func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor {
-	executor := pilosa.NewExecutor(nil)
+	executor := pilosa.NewExecutor(remoteClient)
 	e := &Executor{Executor: executor}
 	e.Holder = holder
 	e.Cluster = cluster