test and fix authentication of client certs btwn cluster nodes

This change should have been adding the "GetClientCertificate"
function in server/tlsconfig.go. This is in addition to the
GetCertificate func which is only used by servers. It ended up being
much more involved for a few reasons:

1. We had no way of passing a configured HTTP client into the
translate store stuff.

2. Our cluster tests assumed http, not HTTPS, and didn't have any way
to pass the necessary configuration in.

3. I encountered what turned out to be an unrelated bug in
cmd/server_test.go which is why I moved "close(m.Started)" in
server/server.go. Basically, I was running something on port 10111
which caused the test to fail (because it was trying to bind to that),
but the failure was not immediately caught during server startup
because the m.Started channel got closed which allowed the test code
to fall through to where it called m.Close() which then got a nil
pointer exception because m.Handler had never been set up.

4. Our test code was assuming that it could create clients that
ignored the config, which meant they didn't do TLS. I added an
InternalClient() method to pilosa.Server to expose the configured
client.

api_test.go

@@ -37,14 +37,14 @@ func TestAPI_Import(t *testing.T) {
 				pilosa.OptServerNodeID("node0"),
 				pilosa.OptServerClusterHasher(&offsetModHasher{}),
 				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
-				pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 			)},
 		[]server.CommandOption{
 			server.OptCommandServerOptions(
 				pilosa.OptServerNodeID("node1"),
 				pilosa.OptServerClusterHasher(&offsetModHasher{}),
 				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
-				pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 			)},
 	)
 	defer c.Close()
@@ -189,13 +189,13 @@ func TestAPI_ImportValue(t *testing.T) {
 			server.OptCommandServerOptions(
 				pilosa.OptServerNodeID("node0"),
 				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 			)},
 		[]server.CommandOption{
 			server.OptCommandServerOptions(
 				pilosa.OptServerNodeID("node1"),
 				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 			)},
 	)
 	defer c.Close()

cmd/server_test.go

@@ -42,7 +42,7 @@ func TestServerConfig(t *testing.T) {
 	tests := []commandTest{
 		// TEST 0
 		{
-			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111", "--translation.map-size", "100000"},
+			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--translation.map-size", "100000"},
 			env: map[string]string{
 				"PILOSA_DATA_DIR":                "/tmp/myEnvDatadir",
 				"PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s",
@@ -69,9 +69,9 @@ func TestServerConfig(t *testing.T) {
 			validation: func() error {
 				v := validator{}
 				v.Check(cmd.Server.Config.DataDir, actualDataDir)
-				v.Check(cmd.Server.Config.Bind, "localhost:10111")
+				v.Check(cmd.Server.Config.Bind, "localhost:42454")
 				v.Check(cmd.Server.Config.Cluster.ReplicaN, 2)
-				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"})
+				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:42454", "localhost:10110"})
 				v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90))
 				v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
 				v.Check(cmd.Server.Config.Translation.MapSize, 100000)

executor_test.go

@@ -2717,7 +2717,7 @@ func TestExecutor_Execute_Existence(t *testing.T) {
 		c := test.MustRunCluster(t, 1, []server.CommandOption{
 			server.OptCommandServerOptions(
 				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
-				pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 			),
 		})
 		defer c.Close()

http/translator.go

@@ -27,8 +27,14 @@ import (
 	"github.com/pilosa/pilosa/v2/logger"
 )
 
-func OpenTranslateReader(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap) (pilosa.TranslateEntryReader, error) {
-	r := NewTranslateEntryReader(ctx)
+func GetOpenTranslateReaderFunc(client *http.Client) pilosa.OpenTranslateReaderFunc {
+	return func(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap) (pilosa.TranslateEntryReader, error) {
+		return openTranslateReader(ctx, nodeURL, offsets, client)
+	}
+}
+
+func openTranslateReader(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap, client *http.Client) (pilosa.TranslateEntryReader, error) {
+	r := NewTranslateEntryReader(ctx, client)
 	r.URL = nodeURL + "/internal/translate/data"
 	r.Offsets = offsets
 	if err := r.Open(); err != nil {
@@ -60,8 +66,11 @@ type TranslateEntryReader struct {
 }
 
 // NewTranslateEntryReader returns a new instance of TranslateEntryReader.
-func NewTranslateEntryReader(ctx context.Context) *TranslateEntryReader {
-	r := &TranslateEntryReader{HTTPClient: http.DefaultClient, Logger: logger.NopLogger}
+func NewTranslateEntryReader(ctx context.Context, client *http.Client) *TranslateEntryReader {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	r := &TranslateEntryReader{HTTPClient: client, Logger: logger.NopLogger}
 	r.ctx, r.cancel = context.WithCancel(ctx)
 	return r
 }

http/translator_test.go

@@ -54,7 +54,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 			}
 
 			// Connect to server and stream all available data.
-			r := http.NewTranslateEntryReader(context.Background())
+			r := http.NewTranslateEntryReader(context.Background(), nil)
 			r.URL = primary.URL()
 
 			// Wait to ensure writes make it to translate store

server.go

@@ -385,6 +385,10 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 	return s, nil
 }
 
+func (s *Server) InternalClient() InternalClient {
+	return s.defaultClient
+}
+
 // UpAndDown brings the server up minimally and shuts it down
 // again; basically, it exists for testing holder open and close.
 func (s *Server) UpAndDown() error {

server/cluster_test.go

@@ -24,6 +24,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/pilosa/pilosa/v2/server"
+
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/test"
 	"golang.org/x/sync/errgroup"
@@ -635,6 +637,33 @@ func TestClusterResize_RemoveNode(t *testing.T) {
 	})
 }
 
+func TestClusterMutualTLS(t *testing.T) {
+	commandOpts := make([][]server.CommandOption, 3)
+	configs := make([]*server.Config, 3)
+	for i, _ := range configs {
+		conf := server.NewConfig()
+		configs[i] = conf
+		conf.Bind = "https://localhost:0"
+		conf.TLS.CertificatePath = "./testdata/certs/localhost.crt"
+		conf.TLS.CertificateKeyPath = "./testdata/certs/localhost.key"
+		conf.TLS.CACertPath = "./testdata/certs/pilosa-ca.crt"
+		conf.TLS.EnableClientVerification = true
+		conf.TLS.SkipVerify = false
+		commandOpts[i] = append(commandOpts[i], server.OptCommandConfig(conf))
+	}
+
+	cluster := test.MustRunCluster(t, 3, commandOpts...)
+	defer cluster.Close()
+	m0 := cluster[0]
+
+	client0 := m0.Client()
+	if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
+		t.Fatal(err)
+	} else if err := client0.CreateField(context.Background(), "i", "f"); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // checkClusterState polls a given cluster for its state until it
 // receives a matching state. It polls up to n times before returning.
 func checkClusterState(m *test.Command, state string, n int) bool {

server/handler_test.go

@@ -1037,7 +1037,7 @@ func TestClusterTranslator(t *testing.T) {
 	cluster[1] = test.NewCommandNode(false,
 		server.OptCommandServerOptions(
 			pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
-			pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+			pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 		),
 	)
 	cluster[1].Config.Gossip.Port = "0"

server/server.go

@@ -104,6 +104,13 @@ func OptCommandCloseTimeout(d time.Duration) CommandOption {
 	}
 }
 
+func OptCommandConfig(config *Config) CommandOption {
+	return func(c *Command) error {
+		c.Config = config
+		return nil
+	}
+}
+
 // NewCommand returns a new instance of Main.
 func NewCommand(stdin io.Reader, stdout, stderr io.Writer, opts ...CommandOption) *Command {
 	c := &Command{
@@ -128,7 +135,6 @@ func NewCommand(stdin io.Reader, stdout, stderr io.Writer, opts ...CommandOption
 
 // Start starts the pilosa server - it returns once the server is running.
 func (m *Command) Start() (err error) {
-	defer close(m.Started)
 
 	// Seed random number generator
 	rand.Seed(time.Now().UTC().UnixNano())
@@ -158,6 +164,7 @@ func (m *Command) Start() (err error) {
 
 	m.logger.Printf("listening as %s\n", m.listenURI)
 
+	close(m.Started)
 	return nil
 }
 
@@ -308,7 +315,7 @@ func (m *Command) SetupServer() error {
 		pilosa.OptServerDiagnosticsInterval(diagnosticsInterval),
 		pilosa.OptServerExecutorPoolSize(m.Config.WorkerPoolSize),
 		pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
-		pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+		pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(c)),
 		pilosa.OptServerLogger(m.logger),
 		pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore),
 		pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()),

server/testdata/certs/localhost.crt

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEJDCCAgygAwIBAgIRALCMLhpmYVAnizSYqLkbCfgwDQYJKoZIhvcNAQELBQAw
+FDESMBAGA1UEAxMJcGlsb3NhLWNhMB4XDTE5MTAxODE2MzczMFoXDTIxMDQxODE1
+MzMyN1owFDESMBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAoAczv4jpoqfXgjoSQg86kKgs/2DToOh1fXkuyEuke5j9WN03
+/HggXMsia5wwvvBpHuK0Gzp35lEmvqHRP8deeU0s8y5KLRqMFNd8otvBfeWRrMqL
+CsBkDm3KO3lr1kLlSkoJRpNOlRX1j3ysvq4lCFXd0G6znP1MYsNtTryqOlpRLGY5
+OpqhcfjQQgei3KBhX4v7+Ydh2uDK1YmJ7M+oyTIzg0yRxvcRphFVzdnyj7mi12hG
+/Y3SumdpBYk2Wbr5odHQIfR274BNHJLoUWQUHFNCzCyl8HF1AxpE15oO57yqk5hT
+bH7n0xwUb6KyV6/i8JYKh+MoApswhpxSrGHYbQIDAQABo3EwbzAOBgNVHQ8BAf8E
+BAMCA7gwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMB0GA1UdDgQWBBSQ
+xZWltgD7EALT0Lm/HQXUiSg6azAfBgNVHSMEGDAWgBSWsR19rRmyUYyp5UgJABB8
+3+kLSTANBgkqhkiG9w0BAQsFAAOCAgEAEpe/admY07Czm3HYOz2DtCdafDPxvJc9
+yE2WXXwFnR+ij4LPp3pZ+4KDeREnX92Ohrms23SH6Jkdmw1Am8KN7TKgrGS7G2N4
+cSTAfaDxeuiOkMMUEw7G3awcEa6WZtY7TFQrdDH/Or8bzrEvq6/zrSPK1yx8LXme
+Xq4ieaDeJ6Zro/0EGPPidAB76XM6mfyWjs05nmi/xlEnzdWoRHln7NbYH8QwoG/+
+dwik5Kte0AmE3byaQV9gngJzXhccYqQws/cx/eD2UuXKIJJX/3ActO5nrUok12Te
+lS/xbICOsF1zQSz8gcMw86qJYzEWppoqOwSaOq4PkJXnzKdlK9wQLbHvTGO9D72Q
+/VX++wHdssiwO5OwvB7wfDbB5XqZAy8mWjDylNT44zSVPesxs9ctGn5xS8jTFV/4
+9wk5jywA60NWQ2nmsLKFExTn6rVasPaSm/3ICsD8PR0muPaGKaZjntn1NnUYC0Lo
+l9Gh6KSWn29twaJzJmfGqasC5qbXiGm70YvIC1feEp8i7+EmMBC19luxMz+RM3n3
+i9F5uUApRdFd9ft9YnCsegas9av/QohBds+w2dFu8ooXsg4H9XbdS6mTrJ957trL
+Hze7eJ1WaZwlDANUUypGi8awXoAVNzQRIEnpV7AGO0Ui1GcUw0dW/fTevo3XErbS
+PO6paG+7uQQ=
+-----END CERTIFICATE-----

server/testdata/certs/localhost.csr

@@ -0,0 +1,15 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICWTCCAUECAQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0B
+AQEFAAOCAQ8AMIIBCgKCAQEAoAczv4jpoqfXgjoSQg86kKgs/2DToOh1fXkuyEuk
+e5j9WN03/HggXMsia5wwvvBpHuK0Gzp35lEmvqHRP8deeU0s8y5KLRqMFNd8otvB
+feWRrMqLCsBkDm3KO3lr1kLlSkoJRpNOlRX1j3ysvq4lCFXd0G6znP1MYsNtTryq
+OlpRLGY5OpqhcfjQQgei3KBhX4v7+Ydh2uDK1YmJ7M+oyTIzg0yRxvcRphFVzdny
+j7mi12hG/Y3SumdpBYk2Wbr5odHQIfR274BNHJLoUWQUHFNCzCyl8HF1AxpE15oO
+57yqk5hTbH7n0xwUb6KyV6/i8JYKh+MoApswhpxSrGHYbQIDAQABoAAwDQYJKoZI
+hvcNAQELBQADggEBACKQDweB2A1vD0RWqKeeRr4gQjka6bpCEa9crd3Cw0iaoXdd
+7slhrGakmIRv6DLrInVAqQL4qrT0kznrvndXnVeBSaLXidEzYzr4pIBPe/p9x5Ei
+hS+Ld6WukNF6g3Xq9LQwYidkNSDnOe4WW82QZ/tFMrkhHk1EuPNhs127zHqbDQCx
+cYpXstQVs/2QdQiOhXqgs7Zh0G9TQA1fy+gdJooDvPjG3jTgjfSFRYx0ihq9LXkS
+ksaxraegUZ30l3RsVjAOA+/ri6wiHpRnN0zAJ56UF5BKH4kKbMdp7iOo1xErOfHh
+167s0RZGEL5PBPw+Fh10Nxb1ZAtjmppGPf2VS1U=
+-----END CERTIFICATE REQUEST-----

server/testdata/certs/localhost.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAoAczv4jpoqfXgjoSQg86kKgs/2DToOh1fXkuyEuke5j9WN03
+/HggXMsia5wwvvBpHuK0Gzp35lEmvqHRP8deeU0s8y5KLRqMFNd8otvBfeWRrMqL
+CsBkDm3KO3lr1kLlSkoJRpNOlRX1j3ysvq4lCFXd0G6znP1MYsNtTryqOlpRLGY5
+OpqhcfjQQgei3KBhX4v7+Ydh2uDK1YmJ7M+oyTIzg0yRxvcRphFVzdnyj7mi12hG
+/Y3SumdpBYk2Wbr5odHQIfR274BNHJLoUWQUHFNCzCyl8HF1AxpE15oO57yqk5hT
+bH7n0xwUb6KyV6/i8JYKh+MoApswhpxSrGHYbQIDAQABAoIBAC1LnJQcNrU5eMB9
+m2ljUhpb39CxIXWsFoVbZah/8KCjXyJmC1DE17QsMyPuY8XH2plJZd1HIWTpl2Ph
+/Y5l6hG/jHeQpKI7n2bOz5JxTUVLU35rHgv/BNMotFM9PQ66hJgQeKunSljcjFL+
+4Q5f0E+Zrl0u2FwZPiO+WRQuRqiZMUOYL+JzaGg0S4vu5xWK1aEtYe2qbZiucKsB
+WujL2G88ETCVyjOBA+eZYkGoFSojOSXiQTwbM5l//JmJKR2xwVBLAPPc7yTTRW0c
+9gLEFuOttqzhmvrqwS7ExtC/HM1F2pDgax5hVyWfxlI7f8851r5NRCzX2ruPrCrK
+8i/YWukCgYEAzfHjgbEnqJTjWEY7zQlStcdFwmVjTSlu0bCUJypLgjMXYQzjGYiy
+gFU/oBNePuT9DfNRrEbK1p9ZM0OZ+WjvYoZuOUNNLhDjjmzDl0ozl/pdUdVmfRXK
+4LwnVdGoSpe2g0yOujqyAwsrLKlhUmfucMaX2cAVFWr5b2tnRACJHOMCgYEAxuxT
+B4srjmle1fZMI30bOhlZwFALTiSt4GTTvcIssYr5gHNj1AbM7NUX5sLeh6rzvQkG
+2zZwL4RN21C+7VrarJR4d1ZcY5c67FVpNGHGVf29YbvXhcecD9N+hKCtUokfInb+
+nP4R0fGbHzmTPS1Bdc4xDYE6hertIvko5jmLBm8CgYAWMEqXzZN9SVVZTG/i/m8n
+4Kl2cWAKjz0+YG0F7zulcoXBUhchyCWbSBQL1ipgUmGlYCLbkeEG0LJWoKD1S0YH
+cn5rswGcxfsOfE9yfG8kn7fwOjLJpVBveDYwu98K8l+as+Po0qwdQdXMAQ/YRhRd
+mDgCIPzkitsHEyZUIE6liwKBgQCvXQEpYlAIoO3A8/tVQeIxys1rk2UUA5PhgSGF
+ZNufvOONkKwN69zWYFlMzrf8pAFxwbZwpEIsMZetYzad6n1irRSyJ8kU5upEFFvF
+mPiPMDy3HzC1uC3lac5cP4XsgAOZj4uQfQ7apQLaeRhLU00k/JwTyrv7+QoKzdp3
+c30TMwKBgQCspDRlf6uqcAqYFAPcKixZlQYdiIWBa/Xn5aaMxfqlHe8V+nljuD3w
+KNu7Hh04+o6h/87w+pnFpo8DjrFgQph4vdnL11NRJb1/cFK1dGi307NkH4jshMjc
++yJxZh1HTrJcXkhaZbGZIm3O/wznprWkGb5tEk+u29GwFAkqsfCogw==
+-----END RSA PRIVATE KEY-----

server/testdata/certs/pilosa-ca.crl

@@ -0,0 +1,16 @@
+-----BEGIN X509 CRL-----
+MIICgzBtAgEBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNVBAMTCXBpbG9zYS1jYRcN
+MTkxMDE4MTUzMzM0WhcNMjEwNDE4MTUzMzI4WjAAoCMwITAfBgNVHSMEGDAWgBSW
+sR19rRmyUYyp5UgJABB83+kLSTANBgkqhkiG9w0BAQsFAAOCAgEAikN4zcHGbM4X
+ntWIa1oMGePL2EKXiyOyzVHenNECevDrEginPt7xKJPt8t1kdE1uvndlRvMVUEgt
+8tpzWHXPVPiNzfVMdriwJm7keEy+ZbImBSGMBlZJQNT/HVEEwgejspT01eC3FGDV
+MXzuDG+4I9ZeAeQOe9vW2RrEGBlNi5QTboV3PO6gn26YTxrLRRvlzWJhelKtfxox
+L/YTn1D5tM2DFrytm/6Ho0s+0yT2jImurOzju/dXMvXBMeVrj5FbqIKd/fjRGP8P
+f9uVRTSWlGh4LK4oIyhe4aF82u81VTLiag6ZQgWvH0guUO3hG5DqpUvl5w0Vt9PG
+Q89QO00hXUrnyuD5aW0WoT/+Th7Tl5FYQUc2nzFznhp1B2uAgpRp6LI6uStNuI7p
+iYCM87ecVA2l58afS+s/bLwfZ2+TwiR2IT1LAj7Ixa203jFeMkKjDukcKQqZIpMu
+OFP6haOispeXuO3AxoKifbkcnWLTcWupiTvnUt2pFSw94SWRkukIvwndAtnXHz1X
+QrrbDNhNXPyUl8dyW4iq4rPxx9VKGebbDiyR6E+sniTUdJ17odI5OWty3Lzu0X1C
+8WA4ca4aw7cP1z50IWFRD+izcp1DJjqWHyP5uUx2BKJXeInHvTt7+js9md0UFN80
+pODkgMZdWLpFZFl/03JPDyfv3EsHygw=
+-----END X509 CRL-----

server/testdata/certs/pilosa-ca.crt

@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIE6DCCAtCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDEwlwaWxv
+c2EtY2EwHhcNMTkxMDE4MTUzMzM0WhcNMjEwNDE4MTUzMzI4WjAUMRIwEAYDVQQD
+EwlwaWxvc2EtY2EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDEMNPJ
+fCBOvGS51gTY9xjIG67IVLEdqIvH83Oju8P25CoASuRbr9teX4AmquOH17Eo9TvV
+j/f/7b1tsl0F0Rf6Xj1qReKVMtv0JQ3vTguGgNrCfYuhax0gulH9toHjsqst0mK0
+TNNz81WOh4XRUROKm43X7gnwtxakMXwSgk94V5p9z4iH2w/fDg8+9vIRd9nWqMBu
+2HTrJt/BtYJP+WiZhk2tJkBTbAHwnFajfzaZTb5zTDAI+fwgL393LdIambZDvcUN
+UCI6fDoaolM42uqLVKgqimxIcKVBbLEccITNEkEsFJXf4aktzH4dwZ+m99/F1Bjo
+NNDOtSKTwqvu92ywAbEfp1tuoPgJafhaq9QsVj8Yyq6km3zvU/2R5fOei+5CUTG4
+Kp5fwwc+549g/S/DlHaoriq1Gso3jWwJdZM+5ywO2gtf6+QPXLJIQXYu4HQ3HgSW
+V79RkxG4LWcxIwdbr+59HQE9fVIEpNWAlvm6IiEVgNcAoyFFBvXXE9D1lfwPlnQg
+0yBOCynh5cBPKX0UscPIR+3tXi5AVY9MIT/HT4adUpQdU2+TMycPAlwL8pKKj5Ij
+cxOWHNc8nfGjDDe/9dzewC00+B3oJLSLpmJiLaUUYLhqt0I1J28KNyaKA250qAr0
+Ffm27WPdaB9YuMTRoRsDg5FP8VuNA4mJ+6IQMQIDAQABo0UwQzAOBgNVHQ8BAf8E
+BAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQUlrEdfa0ZslGMqeVI
+CQAQfN/pC0kwDQYJKoZIhvcNAQELBQADggIBACxzyX9SzIsyk9wz1/S/Nq7KzIXu
+vyjC+t3uJmAvoiNE63Fe52jiEps1v7zK970IXcwr3VF12HJ5hjIG+SOXhd2juYmG
+wRTZ9DmhgrJz2PZGxHohylmn0u+SzRCfK9OJiFNdb0pNN0Jja0B0PwLzmDaPQqJd
+guF+kwY0OOB60/lr+Ti0DbcNCsHgkIxiwg7Ci3xsuumrQU8BvRX/tLU1sBxqBDss
+/AJz/3oqBeP6dAZvKfIIEPZun+8+ViPhPtr312QkKBEQam/QSVZnuZkU7IKnkm3O
+b8Z1YgDAS0EJATN8gay7aBCq6WsNpI5HdIUHRn5eI+ivZ4pk8p/R+ESwRheRBX6+
+iLUnTRZg1vNcqtkBAcXQaP2OnbKT8dodmlL2hGjDc0FrfbawtTZyx3O5hUstFOIF
+871fXwupIhqXM360gfqgyKCAGIDHDAMvho2p8NcmAgA/HMzTX4oUwqfvzLAvqbPC
+u47Ffpzwdrh4+JncP0F5lFZAjElHfVNhaDcCSsp8lq0lUIjkODvIDmz74YmFSLg6
+qAHULVlcyq6n4Y6Z6UVbMrm/z4l8+hch5mkEm65v1N3WVmpKoC54HtGGAqJt2e/d
+wDqKEhEqx9PbGQK7eeb4Zuzga59cvK21CNea9FHTbIIzUj2l9B6vNam+OfNg2B51
+qK0Hhfb0uVNy2vwx
+-----END CERTIFICATE-----

server/testdata/certs/pilosa-ca.key

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEAxDDTyXwgTrxkudYE2PcYyBuuyFSxHaiLx/Nzo7vD9uQqAErk
+W6/bXl+AJqrjh9exKPU71Y/3/+29bbJdBdEX+l49akXilTLb9CUN704LhoDawn2L
+oWsdILpR/baB47KrLdJitEzTc/NVjoeF0VETipuN1+4J8LcWpDF8EoJPeFeafc+I
+h9sP3w4PPvbyEXfZ1qjAbth06ybfwbWCT/lomYZNrSZAU2wB8JxWo382mU2+c0ww
+CPn8IC9/dy3SGpm2Q73FDVAiOnw6GqJTONrqi1SoKopsSHClQWyxHHCEzRJBLBSV
+3+GpLcx+HcGfpvffxdQY6DTQzrUik8Kr7vdssAGxH6dbbqD4CWn4WqvULFY/GMqu
+pJt871P9keXznovuQlExuCqeX8MHPuePYP0vw5R2qK4qtRrKN41sCXWTPucsDtoL
+X+vkD1yySEF2LuB0Nx4Elle/UZMRuC1nMSMHW6/ufR0BPX1SBKTVgJb5uiIhFYDX
+AKMhRQb11xPQ9ZX8D5Z0INMgTgsp4eXATyl9FLHDyEft7V4uQFWPTCE/x0+GnVKU
+HVNvkzMnDwJcC/KSio+SI3MTlhzXPJ3xoww3v/Xc3sAtNPgd6CS0i6ZiYi2lFGC4
+ardCNSdvCjcmigNudKgK9BX5tu1j3WgfWLjE0aEbA4ORT/FbjQOJifuiEDECAwEA
+AQKCAgBMniz3kMwlM5m6oYQ/HzDEZVs6zShmRr5gFglhNiDvuGdWuei4jdMD5hPs
+BezZzjhXoH3/hzv47OV/g5qkfOX6VvdxG13Dmpsfm6D3hekmXAgS0GjJX0V+1ITU
+qoPLYqlLOdBmTWJ+89bD1NVDy3/wL5OTcljbcwUalmC38cALZVrmKzhMphZqq9j0
+ZAhhHgcqt7h/EXWHqCTWR4ZeqS5aVoLbTgbtDU8sokGLlFDESgJIkMZpmwcFoscK
+yhA1ZzYw+lmMO3FCmeUvJZmOqJ/UTflms6xaFfPTuxjn3GFgnPJqvpdRWZVyV1wh
+Ts6ErGTto0ke1hF9yGt6fRocNvBwlPPCTozcN7g5QjBTYqGo/wmfdsbH7v8Qv/AO
+X0HgbaJxpnh24lH2fHpPSQiNiKTvbfQeOzSuIk1qbO27EOuZrR3svwITbNAOXaqc
+0qrEkJxCSWvG3XOiSgkA37GUBe641HD3Jl0nEVJtDGyHe1LTRCD4H6Qpgt3W36IN
+3cHmAY1/imkAUFdXilAk5rfn1EgUQOIM3KbNkFRjgvNR3ZU2nXWd1f/09IXukZHm
+l0ofT4UxlBPESr+ua+4dWzw8bBNcezPhoYf33MHNHHoUZrUyKQVyvEsX7K+i2JBW
+Lv5yuOPf8NngbFgGkslEvjgZhS6uV0OC+jHUhep914Pb8W5egQKCAQEA2KwCzrWC
+oRZfqJT1FVoPIfbkA4fZ7nbPdT+ei6HxTDaRyGoqoSYs5pewS7jvzK8rmFeRYAIl
+vy2wM7+2I7+7sNue9o54Ia+rpa6UgiyrCMbpdVigLtMwV2P8r3dLuAb+qRSlwUUs
+7lVHxEM9PjyrJqBsizuN1gc6+i3kcZZ7t7LzWEJC7E3ss9Bgc+oICuLl5+81AQ6t
+WajlPntyxggIBIEVImEU/b3yRrKkXJ1FNRwEZLPcDtvd3E+WQjntaAklds6i6Yk7
+XysHHJ+W3l9iwk4f6AzKgqVsQ/W6qxUFT2uiMp4WQZrxC/NLSmKa5TRpSGs0yYhR
+qTv2O/BLwOiQOQKCAQEA580ggQkyjN7ArTpKnKXjV/v2mXA4yK4vuUkABcv384fs
+8HZwuO6jbE0w/k1ng+WtOI9lQBYFsh3AcLJtTJksQ73IuFl+J/3nqylyMvVXzGLW
+QQv7MFGyOvxNPTen43Ezw/USW98jem1wxVlkBAkcRqhoI1rm26vvoQbJmyENbaoF
+qJgkvq0DPhtb4yKws9FzS3pL8/75sJoK0WSUbl8YL+m2ZyeAE/g22AFZF2ecIji3
+92cbdL3mREkP8xPBKTIEy6uAlJbMR5Kyo05rvcX5iSaLyhdZT6Smt/OsA/hgFMmp
+D82i7v3w7C8PZAFcNfH0ijMn6RkcIZquJSS9vxKPuQKCAQAicWWY2n3Dgbz/HX7A
+QCq7qXWhogzgvX5JEy3u4HVtOhd+lnHkYB8vINnjj6h/DzGM7A6uBmldtLOX5lfb
+4vTBLohchVVPdY+BrPjqoJMnuAcQomAEUYoTuk9GzV36CRKeaivXR1nXITNhXZb0
+t8w1PbdDgtoPNeMINhdkfxl3oDjSxOXCjrib9PoAMynIIT2O2xW/70dbQ1/9yL38
+APbVUHe9mVUu6O0ElA1Qs6d2w9dm8rh+nDDufrBSIsc8jIeqFTsMiMYpYLumry/a
+datLXAPybWYb0pQdUYi0/Yh0kZWdGkhLcYfz8F79miMpYGWEH5aqmsLwdhm0eYRx
+TAbxAoIBAQCaOpR25ny41/kRCSjdo05pmx7PpqhRdfwQ5u5Z7QV0Bqtyrwt/MGCR
+zaZcfXnze1zOtjIi+sQdZ8hD86PfhEa8Z8jmR701tK1w5+mw2CbruZNewi8GTkUt
+E/P8WEyfexF7P1NQx13etnSDCmAOg1VJ+gEYDQJvhK2y0UZQriQ2JBsRuvEv4QO4
+ThHMRaeWbTSuMpfYp3zQsR2jhq1v5lu8fV88sT55ivxKqlLxzKFI774PoY3WFgfg
+xVAUZr9tvncMYHbz5oXS1VuAZswu/0txorD97J0i7aVz3XBFT/v1uNuAxUy0gUfM
+jd9KpdEJn4waa8ERSgC6pyVZV2HcgOgZAoIBAE9PaayTPDUM2hW4dEqxGPfLNXno
+bAx0ZpYJwj+vGRJLT3qBAbGvbpD5XntAB9QHVOjefFOpj+F6r70VR85WjcFHbRjh
+qUH4rDCx+J1YISPZznSZ4vDDSlXxS85gqvklc7Ay9ypPUMUzuA9W2/lfR2d0lnRC
+mHaoy1U3WithudukTDyE8oxTc2KVhJWdJ+gEX1L9GsVvX0StnmRJJKcYorIQEv8Z
+xPVYhI3fy7CVM7bUNsg1PEZXig9HjRlr7BdNShHT/0DYO4awuktuPUDZfBWVUR74
+lF3YLpoNiQWT8pwC7bqXBE/DeMUG/kEnXDfCTSBoHuVi3WO61OmgFzUhozM=
+-----END RSA PRIVATE KEY-----

server/tlsconfig.go

@@ -107,6 +107,14 @@ func (kpr *keypairReloader) GetCertificateFunc() func(*tls.ClientHelloInfo) (*tl
 	}
 }
 
+func (kpr *keypairReloader) GetClientCertificateFunc() func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+	return func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+		kpr.certMu.RLock()
+		defer kpr.certMu.RUnlock()
+		return kpr.cert, nil
+	}
+}
+
 func GetTLSConfig(tlsConfig *TLSConfig, logger *log.Logger) (TLSConfig *tls.Config, err error) {
 	if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" {
 		kpr, err := NewKeypairReloader(tlsConfig.CertificatePath, tlsConfig.CertificateKeyPath, logger)
@@ -118,6 +126,7 @@ func GetTLSConfig(tlsConfig *TLSConfig, logger *log.Logger) (TLSConfig *tls.Conf
 			PreferServerCipherSuites: true,
 			MinVersion:               tls.VersionTLS12,
 			GetCertificate:           kpr.GetCertificateFunc(),
+			GetClientCertificate:     kpr.GetClientCertificateFunc(),
 		}
 		if tlsConfig.CACertPath != "" {
 			b, err := ioutil.ReadFile(tlsConfig.CACertPath)

test/pilosa.go

@@ -69,8 +69,10 @@ func newCommand(opts ...server.CommandOption) *Command {
 	m := &Command{commandOptions: opts}
 	m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, opts...)
 	m.Config.DataDir = path
-	m.Config.Bind = "http://localhost:0"
-	m.Config.Cluster.Disabled = true
+	defaultConf := server.NewConfig()
+	if m.Config.Bind == defaultConf.Bind {
+		m.Config.Bind = "http://localhost:0"
+	}
 	m.Config.Translation.MapSize = 140000
 	m.Config.WorkerPoolSize = 2
 
@@ -94,6 +96,7 @@ func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
 func MustRunCommand() *Command {
 	m := newCommand()
 	m.Config.Metric.Diagnostics = false // Disable diagnostics.
+	m.Config.Gossip.Port = "0"
 	if err := m.Start(); err != nil {
 		panic(err)
 	}
@@ -172,11 +175,7 @@ func (m *Command) URL() string { return m.API.Node().URI.String() }
 
 // Client returns a client to connect to the program.
 func (m *Command) Client() *http.InternalClient {
-	client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil))
-	if err != nil {
-		panic(err)
-	}
-	return client
+	return m.Server.InternalClient().(*http.InternalClient)
 }
 
 // Query executes a query against the program through the HTTP API.