From 65ad94c376d553a0e56141963e487dba5669ea9c Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 10 Oct 2017 04:48:15 +0300 Subject: [PATCH] clusters with https + self signed certificates work (using --tls.skip-verify) --- client.go | 1 - client_test.go | 6 +- config.go | 2 + ctl/server.go | 1 + executor.go | 12 ++- holder_test.go | 2 +- server.go | 22 +++-- server/server.go | 5 +- test/executor.go | 2 +- uri_test.go | 230 +++++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 269 insertions(+), 14 deletions(-) create mode 100644 uri_test.go diff --git a/client.go b/client.go index bd1940f87..fbd0d3717 100644 --- a/client.go +++ b/client.go @@ -47,7 +47,6 @@ type Client struct { options *ClientOptions // The client to use for HTTP communication. - // Defaults to the http.DefaultClient. HTTPClient *http.Client } diff --git a/client_test.go b/client_test.go index e4f834184..636873242 100644 --- a/client_test.go +++ b/client_test.go @@ -54,7 +54,7 @@ func TestClient_MultiNode(t *testing.T) { } s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor() + e := pilosa.NewExecutor(nil) e.Holder = hldr[0].Holder e.Scheme = cluster.Nodes[0].Scheme e.Host = cluster.Nodes[0].Host @@ -62,7 +62,7 @@ func TestClient_MultiNode(t *testing.T) { return e.Execute(ctx, index, query, slices, opt) } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor() + e := pilosa.NewExecutor(nil) e.Holder = hldr[1].Holder e.Scheme = cluster.Nodes[1].Scheme e.Host = cluster.Nodes[1].Host @@ -70,7 +70,7 @@ func TestClient_MultiNode(t *testing.T) { return e.Execute(ctx, index, query, slices, opt) } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor() + e := pilosa.NewExecutor(nil) e.Holder = hldr[2].Holder e.Scheme = cluster.Nodes[2].Scheme e.Host = cluster.Nodes[2].Host diff --git a/config.go b/config.go index e7863db6f..fe93be19e 100644 --- a/config.go +++ b/config.go @@ -54,6 +54,8 @@ type TLSConfig struct { CertificatePath string `toml:"certificate-path"` // CertificateKeyPath contains the path to the certificate key (.key file) CertificateKeyPath string `toml:"certificate-key-path"` + // SkipVerify disables verification for self-signed certificates + SkipVerify bool `toml:"skip-verify"` } // Config represents the configuration for the command. diff --git a/ctl/server.go b/ctl/server.go index 81f89b4ec..1c1711391 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -44,4 +44,5 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.") flags.StringVarP(&srv.Config.TLS.CertificatePath, "tls.certificate", "", "", "TLS certificate path (usually has the .crt or .pem extension") flags.StringVarP(&srv.Config.TLS.CertificateKeyPath, "tls.key", "", "", "TLS certificate key path (usually has the .key extension") + flags.BoolVarP(&srv.Config.TLS.SkipVerify, "tls.skip-verify", "", false, "Skip TLS certificate verification (not secure)") } diff --git a/executor.go b/executor.go index de1385101..030242b3e 100644 --- a/executor.go +++ b/executor.go @@ -55,9 +55,17 @@ type Executor struct { } // NewExecutor returns a new instance of Executor. -func NewExecutor() *Executor { +func NewExecutor(clientOptions *ClientOptions) *Executor { + if clientOptions == nil { + clientOptions = &ClientOptions{} + } + transport := &http.Transport{} + if clientOptions.TLS != nil { + transport.TLSClientConfig = clientOptions.TLS + } + client := &http.Client{Transport: transport} return &Executor{ - HTTPClient: http.DefaultClient, + HTTPClient: client, } } diff --git a/holder_test.go b/holder_test.go index f1a14df3d..8158de43f 100644 --- a/holder_test.go +++ b/holder_test.go @@ -314,7 +314,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { defer s.Close() s.Handler.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor() + e := pilosa.NewExecutor(nil) e.Holder = hldr1.Holder e.Scheme = cluster.Nodes[1].Scheme e.Host = cluster.Nodes[1].Host diff --git a/server.go b/server.go index 3c882aa90..bdc602a1d 100644 --- a/server.go +++ b/server.go @@ -75,6 +75,8 @@ type Server struct { MaxWritesPerRequest int LogOutput io.Writer + + defaultClient *http.Client } // NewServer returns a new instance of Server. @@ -158,8 +160,11 @@ func (s *Server) Open() error { return fmt.Errorf("opening NodeSet: %v", err) } + // Create default HTTP client + s.createDefaultClient() + // Create executor for executing queries. - e := NewExecutor() + e := NewExecutor(&ClientOptions{TLS: s.TLS}) e.Holder = s.Holder e.Scheme = s.Host.Scheme() e.Host = s.Host.HostPort() @@ -279,7 +284,7 @@ func (s *Server) monitorMaxSlices() { oldmaxslices := s.Holder.MaxSlices() for _, node := range s.Cluster.Nodes { if s.Host.HostPort() != node.Host { - maxSlices, _ := checkMaxSlices(node.Scheme, node.Host) + maxSlices, _ := s.checkMaxSlices(node.Scheme, node.Host) for index, newmax := range maxSlices { // if we don't know about an index locally, log an error because // indexes should be created and synced prior to slice creation @@ -458,7 +463,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { return nil } -func checkMaxSlices(scheme string, hostPort string) (map[string]uint64, error) { +func (s *Server) checkMaxSlices(scheme string, hostPort string) (map[string]uint64, error) { // Create HTTP request. req, err := http.NewRequest("GET", (&url.URL{ Scheme: scheme, @@ -475,8 +480,7 @@ func checkMaxSlices(scheme string, hostPort string) (map[string]uint64, error) { req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("User-Agent", "pilosa/"+Version) - // Send request to remote node. - resp, err := http.DefaultClient.Do(req) + resp, err := s.defaultClient.Do(req) if err != nil { return nil, err } @@ -546,6 +550,14 @@ func (s *Server) monitorRuntime() { } } +func (s *Server) createDefaultClient() { + transport := &http.Transport{} + if s.TLS != nil { + transport.TLSClientConfig = s.TLS + } + s.defaultClient = &http.Client{Transport: transport} +} + // CountOpenFiles on opperating systems that support lsof func CountOpenFiles() int { count := 0 diff --git a/server/server.go b/server/server.go index 2d528410c..5fde9443f 100644 --- a/server/server.go +++ b/server/server.go @@ -164,7 +164,10 @@ func (m *Command) SetupServer() error { if err != nil { return err } - m.Server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + m.Server.TLS = &tls.Config{ + Certificates: []tls.Certificate{cert}, + InsecureSkipVerify: m.Config.TLS.SkipVerify, + } m.Server.Handler.ClientOptions = &pilosa.ClientOptions{TLS: m.Server.TLS} } diff --git a/test/executor.go b/test/executor.go index de051075e..73445a1cd 100644 --- a/test/executor.go +++ b/test/executor.go @@ -15,7 +15,7 @@ type Executor struct { // NewExecutor returns a new instance of Executor. // The executor always matches the hostname of the first cluster node. func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { - e := &Executor{Executor: pilosa.NewExecutor()} + e := &Executor{Executor: pilosa.NewExecutor(nil)} e.Holder = holder e.Cluster = cluster e.Scheme = cluster.Nodes[0].Scheme diff --git a/uri_test.go b/uri_test.go new file mode 100644 index 000000000..ca664fe15 --- /dev/null +++ b/uri_test.go @@ -0,0 +1,230 @@ +// Copyright 2017 Pilosa Corp. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +// CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +// DAMAGE. + +package pilosa + +import "testing" + +func TestDefaultURI(t *testing.T) { + uri := DefaultURI() + compare(t, uri, "http", "localhost", 10101) +} + +func TestURIWithHostPort(t *testing.T) { + uri, err := NewURIFromHostPort("index1.pilosa.com", 3333) + if err != nil { + t.Fatal(err) + } + compare(t, uri, "http", "index1.pilosa.com", 3333) +} + +func TestURIWithInvalidHostPort(t *testing.T) { + _, err := NewURIFromHostPort("index?.pilosa.com", 3333) + if err == nil { + t.Fatalf("should have failed") + } +} + +func TestNewURIFromAddress(t *testing.T) { + for _, item := range validFixture() { + uri, err := NewURIFromAddress(item.address) + if err != nil { + t.Fatalf("Can't parse address: %s, %s", item.address, err) + } + if uri.Error() != nil { + t.Fatalf("Valid addresses shouldn't have attached errors") + } + if !uri.Valid() { + t.Fatalf("Valid() should return true for valid addresses") + } + compare(t, uri, item.scheme, item.host, item.port) + } +} + +func TestURIFromAddress(t *testing.T) { + for _, item := range validFixture() { + uri := URIFromAddress(item.address) + if uri.Error() != nil { + t.Fatalf("Can't parse address: %s, %s", item.address, uri.Error()) + } + if !uri.Valid() { + t.Fatalf("Valid() should return true for valid addresses") + } + compare(t, uri, item.scheme, item.host, item.port) + } +} + +func TestNewURIFromAddressInvalidAddress(t *testing.T) { + for _, addr := range invalidFixture() { + uri, err := NewURIFromAddress(addr) + if err == nil { + t.Fatalf("Invalid address should return an error: %s", addr) + } + if uri.Error() == nil { + t.Fatalf("Invalid addreseses should have attached errors") + } + if uri.Valid() { + t.Fatalf("Valid() should return false for invalid addresses") + } + } +} + +func TestURIFromAddressInvalidAddress(t *testing.T) { + for _, addr := range invalidFixture() { + uri := URIFromAddress(addr) + if uri.Error() == nil { + t.Fatalf("Invalid address should return an error: %s", addr) + } + if uri.Valid() { + t.Fatalf("Valid() should return false for invalid addresses") + } + } +} + +func TestNormalizedAddress(t *testing.T) { + uri, err := NewURIFromAddress("http+protobuf://big-data.pilosa.com:6888") + if err != nil { + t.Fatalf("Can't parse address") + } + if uri.Normalize() != "http://big-data.pilosa.com:6888" { + t.Fatalf("Normalized address is not normal") + } +} + +func TestEquals(t *testing.T) { + uri1 := DefaultURI() + if uri1.Equals(nil) { + t.Fatalf("URI should not be equal to nil") + } + if !uri1.Equals(DefaultURI()) { + t.Fatalf("URI should be equal to another URI with the same scheme, host and port") + } +} + +func TestSetScheme(t *testing.T) { + uri := DefaultURI() + target := "fun" + err := uri.SetScheme(target) + if err != nil { + t.Fatal(err) + } + if uri.Scheme() != target { + t.Fatalf("%s != %s", uri.Scheme(), target) + } +} + +func TestSetHost(t *testing.T) { + uri := DefaultURI() + target := "10.20.30.40" + err := uri.SetHost(target) + if err != nil { + t.Fatal(err) + } + if uri.Host() != target { + t.Fatalf("%s != %s", uri.host, target) + } +} + +func TestSetPort(t *testing.T) { + uri := DefaultURI() + target := uint16(9999) + uri.SetPort(target) + if uri.Port() != target { + t.Fatalf("%d != %d", uri.port, target) + } +} + +func TestSetInvalidScheme(t *testing.T) { + uri := DefaultURI() + err := uri.SetScheme("?invalid") + if err == nil { + t.Fatalf("Should have failed") + } +} + +func TestSetInvalidHost(t *testing.T) { + uri := DefaultURI() + err := uri.SetHost("index?.pilosa.com") + if err == nil { + t.Fatalf("Should have failed") + } +} + +func TestHostPort(t *testing.T) { + uri, err := NewURIFromHostPort("i.pilosa.com", 15001) + if err != nil { + t.Fatal(err) + } + target := "i.pilosa.com:15001" + if uri.HostPort() != target { + t.Fatalf("%s != %s", uri.HostPort(), target) + } +} + +func compare(t *testing.T, uri *URI, scheme string, host string, port uint16) { + if uri.Scheme() != scheme { + t.Fatalf("Scheme does not match: %s != %s", uri.scheme, scheme) + } + if uri.Host() != host { + t.Fatalf("Host does not match: %s != %s", uri.host, host) + } + if uri.Port() != port { + t.Fatalf("Port does not match: %d != %d", uri.port, port) + } +} + +type uriItem struct { + address string + scheme string + host string + port uint16 +} + +func validFixture() []uriItem { + var test = []uriItem{ + {"http+protobuf://index1.pilosa.com:3333", "http+protobuf", "index1.pilosa.com", 3333}, + {"index1.pilosa.com:3333", "http", "index1.pilosa.com", 3333}, + {"https://index1.pilosa.com", "https", "index1.pilosa.com", 10101}, + {"index1.pilosa.com", "http", "index1.pilosa.com", 10101}, + {"https://:3333", "https", "localhost", 3333}, + {":3333", "http", "localhost", 3333}, + {"[::1]", "http", "[::1]", 10101}, + {"[::1]:3333", "http", "[::1]", 3333}, + {"[fd42:4201:f86b:7e09:216:3eff:fefa:ed80]:3333", "http", "[fd42:4201:f86b:7e09:216:3eff:fefa:ed80]", 3333}, + {"https://[fd42:4201:f86b:7e09:216:3eff:fefa:ed80]:3333", "https", "[fd42:4201:f86b:7e09:216:3eff:fefa:ed80]", 3333}, + } + return test +} + +func invalidFixture() []string { + return []string{"foo:bar", "http://foo:", "foo:", ":bar", "http://pilosa.com:129999999999999999999999993", "fd42:4201:f86b:7e09:216:3eff:fefa:ed80"} +}