Merge pull request #2081 from jaffee/client-cert-tls

Fix authentication of client certs between cluster nodes

api_test.go

@@ -37,14 +37,14 @@ func TestAPI_Import(t *testing.T) {
 				pilosa.OptServerNodeID("node0"),
 				pilosa.OptServerClusterHasher(&offsetModHasher{}),
 				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
-				pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 			)},
 		[]server.CommandOption{
 			server.OptCommandServerOptions(
 				pilosa.OptServerNodeID("node1"),
 				pilosa.OptServerClusterHasher(&offsetModHasher{}),
 				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
-				pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 			)},
 	)
 	defer c.Close()
@@ -189,13 +189,13 @@ func TestAPI_ImportValue(t *testing.T) {
 			server.OptCommandServerOptions(
 				pilosa.OptServerNodeID("node0"),
 				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 			)},
 		[]server.CommandOption{
 			server.OptCommandServerOptions(
 				pilosa.OptServerNodeID("node1"),
 				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 			)},
 	)
 	defer c.Close()

cmd/server_test.go

@@ -42,7 +42,7 @@ func TestServerConfig(t *testing.T) {
 	tests := []commandTest{
 		// TEST 0
 		{
-			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111", "--translation.map-size", "100000"},
+			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--translation.map-size", "100000"},
 			env: map[string]string{
 				"PILOSA_DATA_DIR":                "/tmp/myEnvDatadir",
 				"PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s",
@@ -69,9 +69,9 @@ func TestServerConfig(t *testing.T) {
 			validation: func() error {
 				v := validator{}
 				v.Check(cmd.Server.Config.DataDir, actualDataDir)
-				v.Check(cmd.Server.Config.Bind, "localhost:10111")
+				v.Check(cmd.Server.Config.Bind, "localhost:42454")
 				v.Check(cmd.Server.Config.Cluster.ReplicaN, 2)
-				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"})
+				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:42454", "localhost:10110"})
 				v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90))
 				v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
 				v.Check(cmd.Server.Config.Translation.MapSize, 100000)

executor_test.go

@@ -2717,7 +2717,7 @@ func TestExecutor_Execute_Existence(t *testing.T) {
 		c := test.MustRunCluster(t, 1, []server.CommandOption{
 			server.OptCommandServerOptions(
 				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
-				pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 			),
 		})
 		defer c.Close()

http/translator.go

@@ -27,8 +27,14 @@ import (
 	"github.com/pilosa/pilosa/v2/logger"
 )
 
-func OpenTranslateReader(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap) (pilosa.TranslateEntryReader, error) {
-	r := NewTranslateEntryReader(ctx)
+func GetOpenTranslateReaderFunc(client *http.Client) pilosa.OpenTranslateReaderFunc {
+	return func(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap) (pilosa.TranslateEntryReader, error) {
+		return openTranslateReader(ctx, nodeURL, offsets, client)
+	}
+}
+
+func openTranslateReader(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap, client *http.Client) (pilosa.TranslateEntryReader, error) {
+	r := NewTranslateEntryReader(ctx, client)
 	r.URL = nodeURL + "/internal/translate/data"
 	r.Offsets = offsets
 	if err := r.Open(); err != nil {
@@ -60,8 +66,11 @@ type TranslateEntryReader struct {
 }
 
 // NewTranslateEntryReader returns a new instance of TranslateEntryReader.
-func NewTranslateEntryReader(ctx context.Context) *TranslateEntryReader {
-	r := &TranslateEntryReader{HTTPClient: http.DefaultClient, Logger: logger.NopLogger}
+func NewTranslateEntryReader(ctx context.Context, client *http.Client) *TranslateEntryReader {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	r := &TranslateEntryReader{HTTPClient: client, Logger: logger.NopLogger}
 	r.ctx, r.cancel = context.WithCancel(ctx)
 	return r
 }

http/translator_test.go

@@ -54,7 +54,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 			}
 
 			// Connect to server and stream all available data.
-			r := http.NewTranslateEntryReader(context.Background())
+			r := http.NewTranslateEntryReader(context.Background(), nil)
 			r.URL = primary.URL()
 
 			// Wait to ensure writes make it to translate store

server.go

@@ -385,6 +385,10 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 	return s, nil
 }
 
+func (s *Server) InternalClient() InternalClient {
+	return s.defaultClient
+}
+
 // UpAndDown brings the server up minimally and shuts it down
 // again; basically, it exists for testing holder open and close.
 func (s *Server) UpAndDown() error {

server/cluster_test.go

@@ -24,6 +24,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/pilosa/pilosa/v2/server"
+
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/test"
 	"golang.org/x/sync/errgroup"
@@ -635,6 +637,33 @@ func TestClusterResize_RemoveNode(t *testing.T) {
 	})
 }
 
+func TestClusterMutualTLS(t *testing.T) {
+	commandOpts := make([][]server.CommandOption, 3)
+	configs := make([]*server.Config, 3)
+	for i := range configs {
+		conf := server.NewConfig()
+		configs[i] = conf
+		conf.Bind = "https://localhost:0"
+		conf.TLS.CertificatePath = "./testdata/certs/localhost.crt"
+		conf.TLS.CertificateKeyPath = "./testdata/certs/localhost.key"
+		conf.TLS.CACertPath = "./testdata/certs/pilosa-ca.crt"
+		conf.TLS.EnableClientVerification = true
+		conf.TLS.SkipVerify = false
+		commandOpts[i] = append(commandOpts[i], server.OptCommandConfig(conf))
+	}
+
+	cluster := test.MustRunCluster(t, 3, commandOpts...)
+	defer cluster.Close()
+	m0 := cluster[0]
+
+	client0 := m0.Client()
+	if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
+		t.Fatal(err)
+	} else if err := client0.CreateField(context.Background(), "i", "f"); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // checkClusterState polls a given cluster for its state until it
 // receives a matching state. It polls up to n times before returning.
 func checkClusterState(m *test.Command, state string, n int) bool {

server/handler_test.go

@@ -1037,7 +1037,7 @@ func TestClusterTranslator(t *testing.T) {
 	cluster[1] = test.NewCommandNode(false,
 		server.OptCommandServerOptions(
 			pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
-			pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+			pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 		),
 	)
 	cluster[1].Config.Gossip.Port = "0"

server/server.go

@@ -104,6 +104,13 @@ func OptCommandCloseTimeout(d time.Duration) CommandOption {
 	}
 }
 
+func OptCommandConfig(config *Config) CommandOption {
+	return func(c *Command) error {
+		c.Config = config
+		return nil
+	}
+}
+
 // NewCommand returns a new instance of Main.
 func NewCommand(stdin io.Reader, stdout, stderr io.Writer, opts ...CommandOption) *Command {
 	c := &Command{
@@ -128,7 +135,6 @@ func NewCommand(stdin io.Reader, stdout, stderr io.Writer, opts ...CommandOption
 
 // Start starts the pilosa server - it returns once the server is running.
 func (m *Command) Start() (err error) {
-	defer close(m.Started)
 
 	// Seed random number generator
 	rand.Seed(time.Now().UTC().UnixNano())
@@ -158,6 +164,7 @@ func (m *Command) Start() (err error) {
 
 	m.logger.Printf("listening as %s\n", m.listenURI)
 
+	close(m.Started)
 	return nil
 }
 
@@ -308,7 +315,7 @@ func (m *Command) SetupServer() error {
 		pilosa.OptServerDiagnosticsInterval(diagnosticsInterval),
 		pilosa.OptServerExecutorPoolSize(m.Config.WorkerPoolSize),
 		pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
-		pilosa.OptServerOpenTranslateReader(http.OpenTranslateReader),
+		pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(c)),
 		pilosa.OptServerLogger(m.logger),
 		pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore),
 		pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()),

server/testdata/certs/README.md

@@ -0,0 +1,12 @@
+
+
+# these test certs were generated with the following commands
+
+certstrap --depot-path certs init --common-name pilosa-ca --expires "100 years"
+certstrap --depot-path certs request-cert --common-name localhost
+certstrap --depot-path certs sign "localhost" --CA pilosa-ca --expires "100 years"
+
+# certstrap version
+dev-25ea708a
+
+(built with go 1.13)

server/testdata/certs/localhost.crt

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEJjCCAg6gAwIBAgIRAIob2aYSPKm0mF9f5V95yxgwDQYJKoZIhvcNAQELBQAw
+FDESMBAGA1UEAxMJcGlsb3NhLWNhMCAXDTE5MTAyMjE3NTA1MFoYDzIxMTkxMDIy
+MTc0OTA4WjAUMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDRZDO1v7pLP7l4tZDjnzx/WclQ4aF+cGQXBvuJJKqF0rCR
+Uu/5qM3r5k6EnMev+0h7RSlVxUFCZ4vwYOxIDQApLq2JXrgmiu4rwmM897vriEuM
+J/0xZFc5GZ/6GWYnkAtjPk0vxw6tVdoO8TY6yz7BHccc0+3dNaOlVAPBqedUSvwY
+YNBZ7Z7Q8HqMym9r0t8m3EsGNuZcF9/jctgZK2eugNY5MRY+SlLLylc6xkAhXHSJ
+QbQYiR2Fk2W+O9i/4KF35W//5HgHdWtdtFBV+JIZaFbYAGxmVj1rlzDpPgjqNT0v
+u4YcBBmsYWIFYnhSh+FX7eW0182r/OH+at96hIfjAgMBAAGjcTBvMA4GA1UdDwEB
+/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwHQYDVR0OBBYE
+FCty6olvApulDylSPnIgaNZR1AY7MB8GA1UdIwQYMBaAFAfLkSFyTGFBezv+E0LW
+4ffyrRVCMA0GCSqGSIb3DQEBCwUAA4ICAQAsiVVW4jikh+EUepYIc1zUo0jx5QuP
+TdozdvxydOYYWnNtSoS5UA0552s+hefml68iDoKaR62A3Jrcwl8CC9orZIa5i3p5
+j7JhsKELyEV4vnAWu0zKmeEAHkn3+KjPHohzSQTSiDJ79WD2Qoia0OzNU/ehYVAf
+5mxsruTiq7tuFEKFbO91U8rA4uGWGnmW7/y2q6meA9H3XOUKw32RnbzoTwITk/xN
+ESiFC5KJTCnmjK+H1cgf8lh9jmsAx5H28mcW/J0rPkmyuaMdZqOPNP8cDvFzebGc
+A01QXEiTyNUzJLJgS+404eRvXgEFiLHIPYnHA/cPhb6nkaw86YJJ8V2Q6OO8kQ+C
+DhLClu28cutqctm0rrE6n2FK1bZxVUKeSOxLB3Q80jD3Ipcjt6ICIjlWOYS44Xue
+1tfPGMZg84jJ4Gxprf5aC4gQSe2nKX00FbgPqs/awjrRHRO0vHB8spunindApZPM
+EMSakHumb9rkgYghzGpZaHFbQgu0AZCTDlKhlydx024aiHXYCaZEL/E/bPcUqaol
+nFt/VGA6esbFXxM/1KsBKhQim9LCKu8TTZmwurcNmBSXT07kUfC0nFsmvTo8UWaX
+nlRIHb1Gia8patGYztByPen1/+VPt3gsDuk+cP7si18w6fMxgcXbm5Tws8VDiSPM
+vNwfeAMIYgZFMA==
+-----END CERTIFICATE-----

server/testdata/certs/localhost.csr

@@ -0,0 +1,15 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICWTCCAUECAQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0B
+AQEFAAOCAQ8AMIIBCgKCAQEA0WQztb+6Sz+5eLWQ4588f1nJUOGhfnBkFwb7iSSq
+hdKwkVLv+ajN6+ZOhJzHr/tIe0UpVcVBQmeL8GDsSA0AKS6tiV64JoruK8JjPPe7
+64hLjCf9MWRXORmf+hlmJ5ALYz5NL8cOrVXaDvE2Oss+wR3HHNPt3TWjpVQDwann
+VEr8GGDQWe2e0PB6jMpva9LfJtxLBjbmXBff43LYGStnroDWOTEWPkpSy8pXOsZA
+IVx0iUG0GIkdhZNlvjvYv+Chd+Vv/+R4B3VrXbRQVfiSGWhW2ABsZlY9a5cw6T4I
+6jU9L7uGHAQZrGFiBWJ4UofhV+3ltNfNq/zh/mrfeoSH4wIDAQABoAAwDQYJKoZI
+hvcNAQELBQADggEBAH8/v2vHh7WxvZPCDHc8LS2kDQ6JYZ2C2dETTfTK1zLF+XGO
+U1V19FIKr8B3riAHuHdKX0Cw59MDCuUCq/kC/LAGmJ4IhXmVRjts1hTYudSLiqlI
+894pN0WVZtjeQaesPyL1leacPw6+TetoB3ToyNzNa2BKfpCwGlB2wlOV+iCi2WcD
+ZMYQ9/NpxJ3oDN6yC1UJ0fM44BywDO+xZgJL3WhksrwA4byGGekLCNgoucFJRIcs
+5K3XOpeGw1/BV9SrRZjMNMX1AOz4ykR9QTUm3nDgMZmrWKJBft/PfMNRCPgQ0hQk
+CAaHV2xlu/3+iGXH0Hifqm2wwXF+PeBw99GHWCI=
+-----END CERTIFICATE REQUEST-----

server/testdata/certs/localhost.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA0WQztb+6Sz+5eLWQ4588f1nJUOGhfnBkFwb7iSSqhdKwkVLv
++ajN6+ZOhJzHr/tIe0UpVcVBQmeL8GDsSA0AKS6tiV64JoruK8JjPPe764hLjCf9
+MWRXORmf+hlmJ5ALYz5NL8cOrVXaDvE2Oss+wR3HHNPt3TWjpVQDwannVEr8GGDQ
+We2e0PB6jMpva9LfJtxLBjbmXBff43LYGStnroDWOTEWPkpSy8pXOsZAIVx0iUG0
+GIkdhZNlvjvYv+Chd+Vv/+R4B3VrXbRQVfiSGWhW2ABsZlY9a5cw6T4I6jU9L7uG
+HAQZrGFiBWJ4UofhV+3ltNfNq/zh/mrfeoSH4wIDAQABAoIBACiwPvgfVJUQwfq5
+1XjV1YytptgHsDkAoAJ1geVpcZVllpf2tNkfuzwO2c0dJhSFUNVbYCfhl8fOlTvV
+qbRqilB2WJRsCn662oM4V85eGW8wYfXPQ30YkXZwC00vKFGhxQS484jINorqNuBW
+cQJFwmEGZ02rxPoKVU1oIP6+BEy1l4Gx7n+FGlSzWfseQfmWZxbRZtM64/xCnhmc
+txwDceq9i2uZJe3ARiEo/C4pKhLthhL6UdDE6T4QbSveGUOUr0r07iLOMq+Lqi3V
+/dWkxTOewEVvUdik+KWFtDQON4Rmz6k5C1+Alft+YnLKciEEvGpAzYQTWc2J/5gB
+GV6nEwkCgYEA6PzgMWOziPS2eD3yfv4OZzc7LmSEwmqjPtYSXTJ3zYumLEXb3pQF
+PbxKw1pJiqCwwI/Q30z0QmcfLx4XARLm4sMJXeDBzVpNwDXsysPQYG8pni606Xb9
+GcHSuq20ykojisxLNDfwCZaX4ABnqCfLzA0XUBkjGA5WbG+B8y1R7TUCgYEA5hKw
+XAhtY48Kfn07/EB9byrHp5rOeuG0Gq9xjWaKJysqyZvBvOyAi+9+WDYDut42up46
+zQ3dD/fDnKNBGnYT658qPvRfH7dCXqMFlWDlDmDF2YZD6EAKp6nhiPPS06t13T7R
+lPXo5NgL2I4ttpZ25zf1PqYpYmU6SYjdhIT9+7cCgYEAwYJGMr88PmQBqZzJ6+/W
+xqN44+9Y1KD9NRNX7VAdcPqr6dmb5CHMuJnK/rXenuUX6NeKA+1GcmBz5g53zMyG
+N5EGFpK2ZHi8Wwhuw3ofd/jb+CWS3NWXIe15uNLNdbZeDsnfLF/JXi1VtxafAWsM
+OU4mLsCyBSjjXuy+YdsiUIECgYEA0G0/3bA4wQe1vw9fB0+5EruTse/ZfJL6IW6t
+FSc+outFUO1M3wT2E098ixcr298xf3CW3mrfyHZxfHadJQaxVzSTBgNGJapUcv2G
+smPcPmFG+5/Z5w4VWPzad8JDTUS4ex1MqoD4YlCisqVjHlVDFIaqIv5O4XfJDRLz
+cvk0ShMCgYAXwJLH2TNOyVfjxGC8EOMdW7+qqWDgUN8mpbwb/1kEsSIuZp3At9xG
+g/v5oX+pWqAcvYxkWfNZMFKJkewecgbMyFO+WkbeYGU91sIG1ex7kjrAN9/mdara
+UxEDpVVY6xaacj0fhIwdXOJkI8k1uq1RoQ4Z92zXb8hp3LMkFm24WA==
+-----END RSA PRIVATE KEY-----

server/testdata/certs/pilosa-ca.crl

@@ -0,0 +1,16 @@
+-----BEGIN X509 CRL-----
+MIIChTBvAgEBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNVBAMTCXBpbG9zYS1jYRcN
+MTkxMDIyMTc0OTEyWhgPMjExOTEwMjIxNzQ5MDlaMACgIzAhMB8GA1UdIwQYMBaA
+FAfLkSFyTGFBezv+E0LW4ffyrRVCMA0GCSqGSIb3DQEBCwUAA4ICAQCEIu2McMTu
+1nHe5XI20lEqJqCmxmFxhJIULyU4AKBQbXtS8QbHUfNdiy3R6W/Al7fhPs0umy5L
+OFLsswhhb0U7QnDlwyo/xU1zVy0T9vpaKNQDAebpIW+6sQWiGQC/ORMsGcQZEr7h
++9gXlFQVZCGo523kr8EeeFaPyOqz9L4ZPudknGJ/z53QtBkDkl5eLF8UuGVMo08B
+rtmt1RnUcmACR1J2xndGIhMQg5Ay17xaHn33TQLcJQNc5PaOO4nuMIKGaNlAmba7
+RszqAbMyz/7fmIMzpg0CrNfXNT81LbB2HPKLhQAvwMPd8fwgL672dGQSSMlN+w4r
+CWPSGHa0RIhZ1MRE2a91tsPwKg/C/+UZ72uPWPRBqU0rkjiKinl5Nwclv1oZgUh1
+cHPpfh3e1HKL+COHdlKm7cz/fLsZpF9AbOoAp6bTYW/xUEA55lD7H2Svie6ykWt7
+9xZMn7dsFahFgq3tD+RJZUGlCMpAdEOzKWQmol/B6TK7hL0oB11fSXvgUWqdnVTb
+8RrPO8aFgJecEPzMbXzXmfdeAcEWpMtUjkzLqzEKz7SV58SPACqp5nOgbcaf+Xuh
+XAALZrAmRacjUPS8fymL1owBZOxyV2VKTKQRf6kmNfknu8EEDFw/LCkQ5TOeaUsI
+uhVRL27GU70Je2v2bhqrhQBvCfH5jhWXBw==
+-----END X509 CRL-----

server/testdata/certs/pilosa-ca.crt

@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIE6jCCAtKgAwIBAgIBATANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDEwlwaWxv
+c2EtY2EwIBcNMTkxMDIyMTc0OTEyWhgPMjExOTEwMjIxNzQ5MDlaMBQxEjAQBgNV
+BAMTCXBpbG9zYS1jYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAL+2
+LL0LDuadKSkDEXkiC4eOfx79vjFuuk+5Q4pJu8d4ORl/yzh8q1twswR+5A0kw/xX
+M5YuSTZrdzuFmCFU9JzdZnBfCptQGcdnjSC8aI42qwkE8K5VW247l/Hu2NGO7z5e
+kO3QEyXzZgzNo5VmBAdK1pBu8aJfAdd/AvnAh7NWoGCsIry9Mx/hhBKoQ2XEDcD2
+kAfilcjR74UQfICxCSxP1YKRnJ0jGU555ClMy/xo2pPfJfy+pkc1jXmqSVC0oyJh
+si8wnru8b3w+1gydZyVC2CuUm1OM8Dbbjh3pRe+QbAokCt89YiH3jTa6g+X+lK85
+iapcE0ILJO3p2IdAhVpLESDXsMBKaVb2Qn5OKRbeqTQWNI+PwQmX9wyPHhEZDsWQ
+5rNGQmRByNF1/RO4xvLAi6KuqITXQTdlB5RoOiRdRJmV01T40MHOw/6DzUCDhH01
+n7Bkda2V/fHZuqQuV4D8lCqNufTFZi7BYAZAGjB4e3dbW/vKt2tHk6Ms3KBdu/jT
+LoqmO9qx2VeuTZKLjUUX015GGQM6+FMABD36ytL2ujkI7a+qY5le+yEsMZt+XeqY
+pmvDtpPK0+kPzvudHv34PUhBqTBZaVBUMtngZSnLndSSc9MwhDhUWEDqmcpi1sd3
+82Yd7WT2pV9tcO9XEDtjMz0Nww5YLucx2HQ9XM6xAgMBAAGjRTBDMA4GA1UdDwEB
+/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBQHy5EhckxhQXs7
+/hNC1uH38q0VQjANBgkqhkiG9w0BAQsFAAOCAgEAl8J9LB1xJ/aho6yERJCAzCF9
+teaA2RWPmoPXUW2c6r0lRX+tlSmka7PFZYTe2wDJ3BAcVsI2Rt6wcNzDIbVu61R+
+yg9TsSV/BiyVhAIxeeKLk9lstCK2kFTF6ovwySRuU51M9PH1AmEZrTSFL/bEx+pf
+4ilApBJFbgzTVO+aRDRKXtXGvUPg3WSYK/j/HnuHjbHIGq/5DUcKF/3jYez9Y1I/
+2DKNr6as4iAijN24Wd42/ZCQpZef0pXgQkDK6gfBgYJYn9+4y8kR81yfAh/SXePg
+OD+O8VTNTF2FzTOpFW7CIHtFU9LimBcuhIu0W+VQ1AwFvHV9ogV5TzBMr80psARr
+5UGixlpG4tuL1G7qSGQ5Pf6WqqF4m/uw8nuped+lE+nWBYIGi5XefKceMYgiSWyX
+5F6PbdQjYwteETcvejy1+oDVQPePrVNh+bsP8dL+ENKcM3wGUURKtiVRKTTtHk2w
+fUF1zPShrcFsiEkEjih8Q/Kye5vh//gSHGH4/YNypOIQ7TQwuVR+GpwDAGLLnjDt
+8+oaLbUo/jR01rHBKRtVJ8VopHBKKhgztI+nCHdO+uoB7zJYhY6Z91uKVqGz6Bbn
+G8IFVFMJl1qA2gjGiv90LKth2QgdWjmXkRpigzOVvnYeWmWT4IzfCexDLh/kGrYd
+Dml2adeFyF6QE+ZXLok=
+-----END CERTIFICATE-----

server/testdata/certs/pilosa-ca.key

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKgIBAAKCAgEAv7YsvQsO5p0pKQMReSILh45/Hv2+MW66T7lDikm7x3g5GX/L
+OHyrW3CzBH7kDSTD/Fczli5JNmt3O4WYIVT0nN1mcF8Km1AZx2eNILxojjarCQTw
+rlVbbjuX8e7Y0Y7vPl6Q7dATJfNmDM2jlWYEB0rWkG7xol8B138C+cCHs1agYKwi
+vL0zH+GEEqhDZcQNwPaQB+KVyNHvhRB8gLEJLE/VgpGcnSMZTnnkKUzL/Gjak98l
+/L6mRzWNeapJULSjImGyLzCeu7xvfD7WDJ1nJULYK5SbU4zwNtuOHelF75BsCiQK
+3z1iIfeNNrqD5f6UrzmJqlwTQgsk7enYh0CFWksRINewwEppVvZCfk4pFt6pNBY0
+j4/BCZf3DI8eERkOxZDms0ZCZEHI0XX9E7jG8sCLoq6ohNdBN2UHlGg6JF1EmZXT
+VPjQwc7D/oPNQIOEfTWfsGR1rZX98dm6pC5XgPyUKo259MVmLsFgBkAaMHh7d1tb
++8q3a0eToyzcoF27+NMuiqY72rHZV65NkouNRRfTXkYZAzr4UwAEPfrK0va6OQjt
+r6pjmV77ISwxm35d6pima8O2k8rT6Q/O+50e/fg9SEGpMFlpUFQy2eBlKcud1JJz
+0zCEOFRYQOqZymLWx3fzZh3tZPalX21w71cQO2MzPQ3DDlgu5zHYdD1czrECAwEA
+AQKCAgBK+tVEmq6ZQ551i92poscfnXvagyN/8e6sf6kKzQvEWCD8+B5jWV8rxEg/
+2eEXwulkKDbk5fV8c0uO3BFD+BYzw+Pszz82v31VoApyYz4BYSBDxsrbZEb+NhCX
+bvMeAulPW20Tle3nSer/1u3D4MJyl7KTitC20LHHtsVTKRnp6lePd8OeTeFQ9sLR
+CT5S7vryHXz3mIc8o98qC+il7JfJ3O+Ewa6fRrr/Fm1GOeMONXRHoV8aGvatoyRS
+EWfbSMsFB3DJRBrREkIINgtBo85TpHEnmleltdm9muFakYHo0HNxA+e8qCVZY5i3
+bvtnWMqOPdK57okZq4ImCu/E9lcw67si+w3ghwrxtTnCkHhYzWofTjTmtH3gqGKT
+vuNUyv4d+RTu1bW4rur5izY/vkfIYEIH2OzYrP98wI8b15pIqubPzhQFWWVFrSaZ
+yqdfAxfZj7bI6wP0BpM/FRjh0BBguB29dFlM1RBgKfcSTQJazVIVO5xC61BqNQ7V
+TnziEQ2Gym9jr5TgQIFU4pFhix4GFPY+W9DKWcbucQdJ07IPK1hv3PRjScyKum+r
+zrROLq62pIMvSM6Kc5zsP3y3uHq929GVrRGW6RImjnDGIYzgarfpnCCze4XlHc33
+IUAgv5PZMjyiELfpPfoTW2Zq+TMH2ond3O8R7ZFNyTYRnNz8gQKCAQEA7QhqWGw6
+Kq3f1VX/v3n0pnrcdcdK5P7+JvZeL23kjh2ccz10F8ue9Ub0PUlB3/iewdw7gDet
+xALASFW0KW0fzvxA24lALMUZfvoj4+EvMy0zhOjWn9XU+col3rUwPLBimVc3kwVh
+AFt4Y1D7Em2DRplqqmsqYdinCGPR5SPiMYmxKIXyVQ9WaNAKj71AnvG8XVzJjKHW
+qE8P5Oa9c+YivilHduh86Sm4h/9cANqx7weIp+/eUQhrQ/GHoGflEysFrchu/ihy
+0ByYoS5o6mNAGcdy+qDfn2lK0Q2Yy+lBroMvlIoe8Ba7+jywgkdkUuVpC2Np3h+U
+K6wsCq1C8ZMuHwKCAQEAzw1cJe5GolWVzv3VtMR0TJsy8wBlLCaB814LCxVKcS9k
+pkLlRHiLZ6MEoV+nxjGoSgHHbnlLJtFiVmyO//Ar0JrbPFQdStIjflPaIfzo8I1V
+cKGC0Zl2tQ78efui1fxIz0dHz/ij7QURMMNV7JbdnhlQoRsFG/kUF6A2nsp5o5I9
+1tyMgIyO8Cvsj6PGSzawK1GiSXQ8codNadbr9nmX9Q30jYDhmCi8BBpjtRYQhXX4
+R7TBviCz2egign06cSjM62F/eCoWwhkbFtG9KZfJLetFlnFRP1qFmxIyrK31DUoT
+1SF8GXG6BppeKi+NQd6/CA3nQMSRN/vn86yutb/JLwKCAQEAlBtsB7AocD6Q3AaO
+dSBJBDVEx7stSwUh3UqX817hLwPFnCRMBApRWsVpG1pI9R+uY0evTiB8TTvC1e/n
+UPNEqtrn08Vg+hh2I+c8wckU6w9VX7j1Uo4UM7hiDVR9vY4hLdtXeGAmCkkdYkFZ
+gFw5aMW35qBrrRPo2s80ec8AKR/2OHHpx8h7n6ZO0x3X4Pj/T/8IWP1O2X/9WHPs
+8RxdnSDB7MS8OBy/lwqyfD4qbEby1qyE53ExRRUIP+xrMJzzQXE6GUlA5FTZ8ezD
+1aEKjpEXgQ9Aa6UpAJNUZCVtyxNhiZj8V9x8vJFrYRwmzb1W86y8m/Rzaw57vngg
+mj3y1QKCAQEAqtPT/juMLW09b41AH6eiaH332EOwNXAD8X3a0KcdRysZ+I7ix85I
+/hWiwV3Bd6UKuxyipUQPvBQu9S0WApXuIGtTjH+9tziPLjASnc4jhiylnIq0+yeJ
+qDKYHjB+FdzTWaju1+jFns16dGgFYXDLoiAyxOkdGwozCJcX2YUmPM1wlfBbP8g4
+OnGS+KPf+7hAO01b/wyWvsthFHzNLAsAU5/S8FiwSDD7hssbDpIIyYQed1LsMmia
+2qNKraySZkCKaX2EhOj84+GA1x/u2P0xb+BYBpqxtPqDlwC5rHzN3MwMduDsmbWa
+DM6Z+GZJPlp/9C48aNSFF9KPeh0z/CIi4wKCAQEAlmGPCrWd+4pXgCur3G13+lDp
+HP11+8nPsJXsvVaW40GeeRldW+sXVKBQpmZzJD8RenAqKR0Qz6jouQGeeqC/hKEo
+oUbUtyLb8wKXk+nCJayQnGKD8jhXK9cp6DDbI8jJ6CxEZogqWET2jQa1G3dHdD1l
+AKekHho4xVUqur3FtLDdjJp17wiwSMXT19CGJcjjtw/DASmft6v5WpC0bAkttd++
+sNBON4HAHV7I6QZ3NyZoDfN3Y1FNfwN84Jc/y9/Q+GFtKM1XDugTxwdWSuCCMxnA
+IXDPonWf+yiOMFNM3JbzdmniIN8dJUD/+t2SImeXYDVCIQRBlKFBeuWNEH/h+Q==
+-----END RSA PRIVATE KEY-----

server/tlsconfig.go

@@ -107,6 +107,14 @@ func (kpr *keypairReloader) GetCertificateFunc() func(*tls.ClientHelloInfo) (*tl
 	}
 }
 
+func (kpr *keypairReloader) GetClientCertificateFunc() func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+	return func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+		kpr.certMu.RLock()
+		defer kpr.certMu.RUnlock()
+		return kpr.cert, nil
+	}
+}
+
 func GetTLSConfig(tlsConfig *TLSConfig, logger *log.Logger) (TLSConfig *tls.Config, err error) {
 	if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" {
 		kpr, err := NewKeypairReloader(tlsConfig.CertificatePath, tlsConfig.CertificateKeyPath, logger)
@@ -118,6 +126,7 @@ func GetTLSConfig(tlsConfig *TLSConfig, logger *log.Logger) (TLSConfig *tls.Conf
 			PreferServerCipherSuites: true,
 			MinVersion:               tls.VersionTLS12,
 			GetCertificate:           kpr.GetCertificateFunc(),
+			GetClientCertificate:     kpr.GetClientCertificateFunc(),
 		}
 		if tlsConfig.CACertPath != "" {
 			b, err := ioutil.ReadFile(tlsConfig.CACertPath)

test/pilosa.go

@@ -69,8 +69,10 @@ func newCommand(opts ...server.CommandOption) *Command {
 	m := &Command{commandOptions: opts}
 	m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, opts...)
 	m.Config.DataDir = path
-	m.Config.Bind = "http://localhost:0"
-	m.Config.Cluster.Disabled = true
+	defaultConf := server.NewConfig()
+	if m.Config.Bind == defaultConf.Bind {
+		m.Config.Bind = "http://localhost:0"
+	}
 	m.Config.Translation.MapSize = 140000
 	m.Config.WorkerPoolSize = 2
 
@@ -94,6 +96,7 @@ func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
 func MustRunCommand() *Command {
 	m := newCommand()
 	m.Config.Metric.Diagnostics = false // Disable diagnostics.
+	m.Config.Gossip.Port = "0"
 	if err := m.Start(); err != nil {
 		panic(err)
 	}
@@ -172,11 +175,7 @@ func (m *Command) URL() string { return m.API.Node().URI.String() }
 
 // Client returns a client to connect to the program.
 func (m *Command) Client() *http.InternalClient {
-	client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil))
-	if err != nil {
-		panic(err)
-	}
-	return client
+	return m.Server.InternalClient().(*http.InternalClient)
 }
 
 // Query executes a query against the program through the HTTP API.