clusters with https + self signed certificates work (using --tls.skip-verify)

client.go

@@ -47,7 +47,6 @@ type Client struct {
 	options *ClientOptions
 
 	// The client to use for HTTP communication.
-	// Defaults to the http.DefaultClient.
 	HTTPClient *http.Client
 }
 

client_test.go

@@ -54,7 +54,7 @@ func TestClient_MultiNode(t *testing.T) {
 	}
 
 	s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
-		e := pilosa.NewExecutor()
+		e := pilosa.NewExecutor(nil)
 		e.Holder = hldr[0].Holder
 		e.Scheme = cluster.Nodes[0].Scheme
 		e.Host = cluster.Nodes[0].Host
@@ -62,7 +62,7 @@ func TestClient_MultiNode(t *testing.T) {
 		return e.Execute(ctx, index, query, slices, opt)
 	}
 	s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
-		e := pilosa.NewExecutor()
+		e := pilosa.NewExecutor(nil)
 		e.Holder = hldr[1].Holder
 		e.Scheme = cluster.Nodes[1].Scheme
 		e.Host = cluster.Nodes[1].Host
@@ -70,7 +70,7 @@ func TestClient_MultiNode(t *testing.T) {
 		return e.Execute(ctx, index, query, slices, opt)
 	}
 	s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
-		e := pilosa.NewExecutor()
+		e := pilosa.NewExecutor(nil)
 		e.Holder = hldr[2].Holder
 		e.Scheme = cluster.Nodes[2].Scheme
 		e.Host = cluster.Nodes[2].Host

config.go

@@ -54,6 +54,8 @@ type TLSConfig struct {
 	CertificatePath string `toml:"certificate-path"`
 	// CertificateKeyPath contains the path to the certificate key (.key file)
 	CertificateKeyPath string `toml:"certificate-key-path"`
+	// SkipVerify disables verification for self-signed certificates
+	SkipVerify bool `toml:"skip-verify"`
 }
 
 // Config represents the configuration for the command.

ctl/server.go

@@ -44,4 +44,5 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.")
 	flags.StringVarP(&srv.Config.TLS.CertificatePath, "tls.certificate", "", "", "TLS certificate path (usually has the .crt or .pem extension")
 	flags.StringVarP(&srv.Config.TLS.CertificateKeyPath, "tls.key", "", "", "TLS certificate key path (usually has the .key extension")
+	flags.BoolVarP(&srv.Config.TLS.SkipVerify, "tls.skip-verify", "", false, "Skip TLS certificate verification (not secure)")
 }

executor.go

@@ -55,9 +55,17 @@ type Executor struct {
 }
 
 // NewExecutor returns a new instance of Executor.
-func NewExecutor() *Executor {
+func NewExecutor(clientOptions *ClientOptions) *Executor {
+	if clientOptions == nil {
+		clientOptions = &ClientOptions{}
+	}
+	transport := &http.Transport{}
+	if clientOptions.TLS != nil {
+		transport.TLSClientConfig = clientOptions.TLS
+	}
+	client := &http.Client{Transport: transport}
 	return &Executor{
-		HTTPClient: http.DefaultClient,
+		HTTPClient: client,
 	}
 }
 

holder_test.go

@@ -314,7 +314,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 	defer s.Close()
 	s.Handler.Holder = hldr1.Holder
 	s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
-		e := pilosa.NewExecutor()
+		e := pilosa.NewExecutor(nil)
 		e.Holder = hldr1.Holder
 		e.Scheme = cluster.Nodes[1].Scheme
 		e.Host = cluster.Nodes[1].Host

server.go

@@ -75,6 +75,8 @@ type Server struct {
 	MaxWritesPerRequest int
 
 	LogOutput io.Writer
+
+	defaultClient *http.Client
 }
 
 // NewServer returns a new instance of Server.
@@ -158,8 +160,11 @@ func (s *Server) Open() error {
 		return fmt.Errorf("opening NodeSet: %v", err)
 	}
 
+	// Create default HTTP client
+	s.createDefaultClient()
+
 	// Create executor for executing queries.
-	e := NewExecutor()
+	e := NewExecutor(&ClientOptions{TLS: s.TLS})
 	e.Holder = s.Holder
 	e.Scheme = s.Host.Scheme()
 	e.Host = s.Host.HostPort()
@@ -279,7 +284,7 @@ func (s *Server) monitorMaxSlices() {
 		oldmaxslices := s.Holder.MaxSlices()
 		for _, node := range s.Cluster.Nodes {
 			if s.Host.HostPort() != node.Host {
-				maxSlices, _ := checkMaxSlices(node.Scheme, node.Host)
+				maxSlices, _ := s.checkMaxSlices(node.Scheme, node.Host)
 				for index, newmax := range maxSlices {
 					// if we don't know about an index locally, log an error because
 					// indexes should be created and synced prior to slice creation
@@ -458,7 +463,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
 	return nil
 }
 
-func checkMaxSlices(scheme string, hostPort string) (map[string]uint64, error) {
+func (s *Server) checkMaxSlices(scheme string, hostPort string) (map[string]uint64, error) {
 	// Create HTTP request.
 	req, err := http.NewRequest("GET", (&url.URL{
 		Scheme: scheme,
@@ -475,8 +480,7 @@ func checkMaxSlices(scheme string, hostPort string) (map[string]uint64, error) {
 	req.Header.Set("Content-Type", "application/x-protobuf")
 	req.Header.Set("User-Agent", "pilosa/"+Version)
 
-	// Send request to remote node.
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := s.defaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -546,6 +550,14 @@ func (s *Server) monitorRuntime() {
 	}
 }
 
+func (s *Server) createDefaultClient() {
+	transport := &http.Transport{}
+	if s.TLS != nil {
+		transport.TLSClientConfig = s.TLS
+	}
+	s.defaultClient = &http.Client{Transport: transport}
+}
+
 // CountOpenFiles on opperating systems that support lsof
 func CountOpenFiles() int {
 	count := 0

server/server.go

@@ -164,7 +164,10 @@ func (m *Command) SetupServer() error {
 		if err != nil {
 			return err
 		}
-		m.Server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}}
+		m.Server.TLS = &tls.Config{
+			Certificates:       []tls.Certificate{cert},
+			InsecureSkipVerify: m.Config.TLS.SkipVerify,
+		}
 		m.Server.Handler.ClientOptions = &pilosa.ClientOptions{TLS: m.Server.TLS}
 	}
 

test/executor.go

@@ -15,7 +15,7 @@ type Executor struct {
 // NewExecutor returns a new instance of Executor.
 // The executor always matches the hostname of the first cluster node.
 func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor {
-	e := &Executor{Executor: pilosa.NewExecutor()}
+	e := &Executor{Executor: pilosa.NewExecutor(nil)}
 	e.Holder = holder
 	e.Cluster = cluster
 	e.Scheme = cluster.Nodes[0].Scheme

uri_test.go

@@ -0,0 +1,230 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its
+// contributors may be used to endorse or promote products derived
+// from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+// CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+
+package pilosa
+
+import "testing"
+
+func TestDefaultURI(t *testing.T) {
+	uri := DefaultURI()
+	compare(t, uri, "http", "localhost", 10101)
+}
+
+func TestURIWithHostPort(t *testing.T) {
+	uri, err := NewURIFromHostPort("index1.pilosa.com", 3333)
+	if err != nil {
+		t.Fatal(err)
+	}
+	compare(t, uri, "http", "index1.pilosa.com", 3333)
+}
+
+func TestURIWithInvalidHostPort(t *testing.T) {
+	_, err := NewURIFromHostPort("index?.pilosa.com", 3333)
+	if err == nil {
+		t.Fatalf("should have failed")
+	}
+}
+
+func TestNewURIFromAddress(t *testing.T) {
+	for _, item := range validFixture() {
+		uri, err := NewURIFromAddress(item.address)
+		if err != nil {
+			t.Fatalf("Can't parse address: %s, %s", item.address, err)
+		}
+		if uri.Error() != nil {
+			t.Fatalf("Valid addresses shouldn't have attached errors")
+		}
+		if !uri.Valid() {
+			t.Fatalf("Valid() should return true for valid addresses")
+		}
+		compare(t, uri, item.scheme, item.host, item.port)
+	}
+}
+
+func TestURIFromAddress(t *testing.T) {
+	for _, item := range validFixture() {
+		uri := URIFromAddress(item.address)
+		if uri.Error() != nil {
+			t.Fatalf("Can't parse address: %s, %s", item.address, uri.Error())
+		}
+		if !uri.Valid() {
+			t.Fatalf("Valid() should return true for valid addresses")
+		}
+		compare(t, uri, item.scheme, item.host, item.port)
+	}
+}
+
+func TestNewURIFromAddressInvalidAddress(t *testing.T) {
+	for _, addr := range invalidFixture() {
+		uri, err := NewURIFromAddress(addr)
+		if err == nil {
+			t.Fatalf("Invalid address should return an error: %s", addr)
+		}
+		if uri.Error() == nil {
+			t.Fatalf("Invalid addreseses should have attached errors")
+		}
+		if uri.Valid() {
+			t.Fatalf("Valid() should return false for invalid addresses")
+		}
+	}
+}
+
+func TestURIFromAddressInvalidAddress(t *testing.T) {
+	for _, addr := range invalidFixture() {
+		uri := URIFromAddress(addr)
+		if uri.Error() == nil {
+			t.Fatalf("Invalid address should return an error: %s", addr)
+		}
+		if uri.Valid() {
+			t.Fatalf("Valid() should return false for invalid addresses")
+		}
+	}
+}
+
+func TestNormalizedAddress(t *testing.T) {
+	uri, err := NewURIFromAddress("http+protobuf://big-data.pilosa.com:6888")
+	if err != nil {
+		t.Fatalf("Can't parse address")
+	}
+	if uri.Normalize() != "http://big-data.pilosa.com:6888" {
+		t.Fatalf("Normalized address is not normal")
+	}
+}
+
+func TestEquals(t *testing.T) {
+	uri1 := DefaultURI()
+	if uri1.Equals(nil) {
+		t.Fatalf("URI should not be equal to nil")
+	}
+	if !uri1.Equals(DefaultURI()) {
+		t.Fatalf("URI should be equal to another URI with the same scheme, host and port")
+	}
+}
+
+func TestSetScheme(t *testing.T) {
+	uri := DefaultURI()
+	target := "fun"
+	err := uri.SetScheme(target)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if uri.Scheme() != target {
+		t.Fatalf("%s != %s", uri.Scheme(), target)
+	}
+}
+
+func TestSetHost(t *testing.T) {
+	uri := DefaultURI()
+	target := "10.20.30.40"
+	err := uri.SetHost(target)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if uri.Host() != target {
+		t.Fatalf("%s != %s", uri.host, target)
+	}
+}
+
+func TestSetPort(t *testing.T) {
+	uri := DefaultURI()
+	target := uint16(9999)
+	uri.SetPort(target)
+	if uri.Port() != target {
+		t.Fatalf("%d != %d", uri.port, target)
+	}
+}
+
+func TestSetInvalidScheme(t *testing.T) {
+	uri := DefaultURI()
+	err := uri.SetScheme("?invalid")
+	if err == nil {
+		t.Fatalf("Should have failed")
+	}
+}
+
+func TestSetInvalidHost(t *testing.T) {
+	uri := DefaultURI()
+	err := uri.SetHost("index?.pilosa.com")
+	if err == nil {
+		t.Fatalf("Should have failed")
+	}
+}
+
+func TestHostPort(t *testing.T) {
+	uri, err := NewURIFromHostPort("i.pilosa.com", 15001)
+	if err != nil {
+		t.Fatal(err)
+	}
+	target := "i.pilosa.com:15001"
+	if uri.HostPort() != target {
+		t.Fatalf("%s != %s", uri.HostPort(), target)
+	}
+}
+
+func compare(t *testing.T, uri *URI, scheme string, host string, port uint16) {
+	if uri.Scheme() != scheme {
+		t.Fatalf("Scheme does not match: %s != %s", uri.scheme, scheme)
+	}
+	if uri.Host() != host {
+		t.Fatalf("Host does not match: %s != %s", uri.host, host)
+	}
+	if uri.Port() != port {
+		t.Fatalf("Port does not match: %d != %d", uri.port, port)
+	}
+}
+
+type uriItem struct {
+	address string
+	scheme  string
+	host    string
+	port    uint16
+}
+
+func validFixture() []uriItem {
+	var test = []uriItem{
+		{"http+protobuf://index1.pilosa.com:3333", "http+protobuf", "index1.pilosa.com", 3333},
+		{"index1.pilosa.com:3333", "http", "index1.pilosa.com", 3333},
+		{"https://index1.pilosa.com", "https", "index1.pilosa.com", 10101},
+		{"index1.pilosa.com", "http", "index1.pilosa.com", 10101},
+		{"https://:3333", "https", "localhost", 3333},
+		{":3333", "http", "localhost", 3333},
+		{"[::1]", "http", "[::1]", 10101},
+		{"[::1]:3333", "http", "[::1]", 3333},
+		{"[fd42:4201:f86b:7e09:216:3eff:fefa:ed80]:3333", "http", "[fd42:4201:f86b:7e09:216:3eff:fefa:ed80]", 3333},
+		{"https://[fd42:4201:f86b:7e09:216:3eff:fefa:ed80]:3333", "https", "[fd42:4201:f86b:7e09:216:3eff:fefa:ed80]", 3333},
+	}
+	return test
+}
+
+func invalidFixture() []string {
+	return []string{"foo:bar", "http://foo:", "foo:", ":bar", "http://pilosa.com:129999999999999999999999993", "fd42:4201:f86b:7e09:216:3eff:fefa:ed80"}
+}