From deed9adfe498e2b6475c7a897cf2ed9bbd4c6e4f Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 25 Oct 2017 09:05:43 -0500 Subject: [PATCH] Fix existing tests --- client.go | 2 +- cluster.go | 6 +- cluster_test.go | 145 ++++++++++++++++++++++-------------------------- handler_test.go | 7 +-- test/handler.go | 33 +++-------- uri.go | 30 ++++++++++ 6 files changed, 110 insertions(+), 113 deletions(-) diff --git a/client.go b/client.go index 5edd45468..a579e527f 100644 --- a/client.go +++ b/client.go @@ -1226,7 +1226,7 @@ func uriPathToURL(uri *URI, path string) url.URL { func nodePathToURL(node *Node, path string) url.URL { return url.URL{ Scheme: node.URI.Scheme(), - Host: node.URI.Host(), + Host: node.URI.HostPort(), Path: path, } } diff --git a/cluster.go b/cluster.go index c3b02ecd2..4f389fb4b 100644 --- a/cluster.go +++ b/cluster.go @@ -55,7 +55,7 @@ const ( type Node struct { //Scheme string `json:"scheme"` //Host string `json:"host"` // HostPort - URI URI // TODO: add json tags: `json:"uri"` + URI URI `json:"uri"` status *internal.NodeStatus `json:"status"` } @@ -220,7 +220,6 @@ func (c *Cluster) AddHost(uri URI) error { // add to cluster _, added := c.AddNode(uri) if !added { - fmt.Println("NOT added") return nil } @@ -229,12 +228,10 @@ func (c *Cluster) AddHost(uri URI) error { return fmt.Errorf("Cluster.Topology is nil") } if !c.Topology.AddURI(uri) { - fmt.Println("call top.AddHost()") return nil } // save topology - fmt.Println("calll c.saveTop()") return c.saveTopology() } @@ -974,7 +971,6 @@ func (c *Cluster) loadTopology() error { // saveTopology writes the current topology to disk. func (c *Cluster) saveTopology() error { - fmt.Println("saveTopology", filepath.Join(c.Path, ".topology")) if buf, err := proto.Marshal(encodeTopology(c.Topology)); err != nil { return err } else if err := ioutil.WriteFile(filepath.Join(c.Path, ".topology"), buf, 0666); err != nil { diff --git a/cluster_test.go b/cluster_test.go index 0be636181..674be6919 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -22,6 +22,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/test" ) @@ -91,38 +92,6 @@ func TestHasher(t *testing.T) { } } -/* TODO travis: fix test -// Ensure cluster can compare its Nodes and Members -func TestCluster_NodeStates(t *testing.T) { - c := pilosa.Cluster{ - Nodes: []*pilosa.Node{ - {Host: "serverA:1000"}, - {Host: "serverB:1000"}, - {Host: "serverC:1000"}, - }, - NodeSet: &pilosa.StaticNodeSet{}, - } - - err := c.NodeSet.(*pilosa.StaticNodeSet).Join([]*pilosa.Node{ - &pilosa.Node{Host: "serverA:1000"}, - &pilosa.Node{Host: "serverC:1000"}, - &pilosa.Node{Host: "serverD:1000"}, - }) - if err != nil { - t.Fatalf("unexpected gossiper nodes: %s", err) - } - - // Verify a DOWN node is reported, and extraneous nodes are ignored - if a := c.NodeStates(); !reflect.DeepEqual(a, map[string]string{ - "serverA:1000": pilosa.NodeStateUp, - "serverB:1000": pilosa.NodeStateDown, - "serverC:1000": pilosa.NodeStateUp, - }) { - t.Fatalf("unexpected node state: %s", spew.Sdump(a)) - } -} -*/ - // Ensure OwnsSlices can find the actual slice list for node and index. func TestCluster_OwnsSlices(t *testing.T) { c := test.NewCluster(5) @@ -133,35 +102,37 @@ func TestCluster_OwnsSlices(t *testing.T) { } } -// TODO travis: fix these tests -/* func TestCluster_Nodes(t *testing.T) { + uri0 := test.NewURIFromHostPort("node0", 0) + uri1 := test.NewURIFromHostPort("node1", 0) + uri2 := test.NewURIFromHostPort("node2", 0) + uri3 := test.NewURIFromHostPort("node3", 0) nodes := []*pilosa.Node{ - {URI: test.NewURIFromHostPort("node0", 0)}, - {URI: test.NewURIFromHostPort("node1", 0)}, - {URI: test.NewURIFromHostPort("node2", 0)}, + {URI: uri0}, + {URI: uri1}, + {URI: uri2}, } t.Run("URISet", func(t *testing.T) { actual := pilosa.Nodes(nodes).URIs() - expected := []string{"node0", "node1", "node2"} + expected := []pilosa.URI{uri0, uri1, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } }) t.Run("Filter", func(t *testing.T) { - actual := pilosa.Nodes(pilosa.Nodes(nodes).Filter(nodes[1])).Hosts() - expected := []string{"node0", "node2"} + actual := pilosa.Nodes(pilosa.Nodes(nodes).Filter(nodes[1])).URIs() + expected := []pilosa.URI{uri0, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } }) - t.Run("FilterHost", func(t *testing.T) { - actual := pilosa.Nodes(pilosa.Nodes(nodes).FilterHost("node1")).Hosts() - expected := []string{"node0", "node2"} + t.Run("FilterURI", func(t *testing.T) { + actual := pilosa.Nodes(pilosa.Nodes(nodes).FilterURI(uri1)).URIs() + expected := []pilosa.URI{uri0, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } @@ -178,9 +149,9 @@ func TestCluster_Nodes(t *testing.T) { } }) - t.Run("ContainsHost", func(t *testing.T) { - actualTrue := pilosa.Nodes(nodes).ContainsHost("node1") - actualFalse := pilosa.Nodes(nodes).ContainsHost("nodeX") + t.Run("ContainsURI", func(t *testing.T) { + actualTrue := pilosa.Nodes(nodes).ContainsURI(uri1) + actualFalse := pilosa.Nodes(nodes).ContainsURI(uri3) if !reflect.DeepEqual(actualTrue, true) { t.Errorf("expected: %v, but got: %v", true, actualTrue) } @@ -191,8 +162,8 @@ func TestCluster_Nodes(t *testing.T) { t.Run("Clone", func(t *testing.T) { clone := pilosa.Nodes(nodes).Clone() - actual := pilosa.Nodes(clone).Hosts() - expected := []string{"node0", "node1", "node2"} + actual := pilosa.Nodes(clone).URIs() + expected := []pilosa.URI{uri0, uri1, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } @@ -200,19 +171,21 @@ func TestCluster_Nodes(t *testing.T) { } func TestCluster_Coordinator(t *testing.T) { + uri1 := test.NewURIFromHostPort("node1", 0) + uri2 := test.NewURIFromHostPort("node2", 0) c1 := *pilosa.NewCluster() - c1.Host = "host0:port0" - c1.Coordinator = "host0:port0" + c1.URI = uri1 + c1.Coordinator = uri1 c2 := *pilosa.NewCluster() - c2.Host = "host1:port1" - c2.Coordinator = "host0:port0" + c2.URI = uri2 + c2.Coordinator = uri1 t.Run("IsCoordinator", func(t *testing.T) { if !c1.IsCoordinator() { - t.Errorf("!IsCoordinator error: %v", c1.Host) + t.Errorf("!IsCoordinator error: %v", c1.URI) } else if c2.IsCoordinator() { - t.Errorf("IsCoordinator error: %v", c2.Host) + t.Errorf("IsCoordinator error: %v", c2.URI) } }) } @@ -220,34 +193,39 @@ func TestCluster_Coordinator(t *testing.T) { func TestCluster_Topology(t *testing.T) { c1 := test.NewCluster(1) + uri1 := test.NewURIFromHostPort("node1", 0) + uri2 := test.NewURIFromHostPort("node2", 0) + base := test.NewURIFromHostPort("host0", 0) + invalid := test.NewURIFromHostPort("invalid", 0) + t.Run("AddHost", func(t *testing.T) { - err := c1.AddHost("abc") + err := c1.AddHost(uri1) if err != nil { t.Fatal(err) } // add the same host. - err = c1.AddHost("abc") + err = c1.AddHost(uri1) if err != nil { t.Fatal(err) } - err = c1.AddHost("xyz") + err = c1.AddHost(uri2) if err != nil { t.Fatal(err) } - actual := pilosa.Nodes(c1.Nodes).Hosts() - expected := []string{"abc", "host0", "xyz"} + actual := pilosa.Nodes(c1.Nodes).URIs() + expected := []pilosa.URI{base, uri1, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } }) - t.Run("ContainsHost", func(t *testing.T) { - if !c1.Topology.ContainsHost("abc") { - t.Errorf("!ContainsHost error: %v", "abc") - } else if c1.Topology.ContainsHost("invalidHost") { - t.Errorf("ContainsHost error: %v", "invalidHost") + t.Run("ContainsURI", func(t *testing.T) { + if !c1.Topology.ContainsURI(uri1) { + t.Errorf("!ContainsHost error: %v", uri1) + } else if c1.Topology.ContainsURI(invalid) { + t.Errorf("ContainsHost error: %v", invalid) } }) } @@ -290,23 +268,33 @@ func TestCluster_Resize(t *testing.T) { c2 := test.NewCluster(4) c2.ReplicaN = 2 - expected := map[string][]*internal.ResizeSource{ - "host0": []*internal.ResizeSource{ - {Host: "host1", Index: "i", Frame: "f", View: "inverse", Slice: 5}, + u0 := test.NewURIFromHostPort("host0", 0) + u1 := test.NewURIFromHostPort("host1", 0) + u2 := test.NewURIFromHostPort("host2", 0) + u3 := test.NewURIFromHostPort("host3", 0) + + uri0 := u0.Encode() + uri1 := u1.Encode() + uri2 := u2.Encode() + //uri3 := u3.Encode() + + expected := map[pilosa.URI][]*internal.ResizeSource{ + u0: []*internal.ResizeSource{ + {URI: uri1, Index: "i", Frame: "f", View: "inverse", Slice: 5}, }, - "host1": []*internal.ResizeSource{ - {Host: "host2", Index: "i", Frame: "f", View: "v", Slice: 0}, - {Host: "host2", Index: "i", Frame: "f", View: "inverse", Slice: 0}, + u1: []*internal.ResizeSource{ + {URI: uri2, Index: "i", Frame: "f", View: "v", Slice: 0}, + {URI: uri2, Index: "i", Frame: "f", View: "inverse", Slice: 0}, }, - "host2": []*internal.ResizeSource{ - {Host: "host0", Index: "i", Frame: "f", View: "inverse", Slice: 3}, + u2: []*internal.ResizeSource{ + {URI: uri0, Index: "i", Frame: "f", View: "inverse", Slice: 3}, }, - "host3": []*internal.ResizeSource{ - {Host: "host0", Index: "i", Frame: "f", View: "v", Slice: 1}, - {Host: "host1", Index: "i", Frame: "f", View: "v", Slice: 2}, - {Host: "host0", Index: "i", Frame: "f", View: "inverse", Slice: 1}, - {Host: "host1", Index: "i", Frame: "f", View: "inverse", Slice: 2}, - {Host: "host1", Index: "i", Frame: "f", View: "inverse", Slice: 5}, + u3: []*internal.ResizeSource{ + {URI: uri0, Index: "i", Frame: "f", View: "v", Slice: 1}, + {URI: uri1, Index: "i", Frame: "f", View: "v", Slice: 2}, + {URI: uri0, Index: "i", Frame: "f", View: "inverse", Slice: 1}, + {URI: uri1, Index: "i", Frame: "f", View: "inverse", Slice: 2}, + {URI: uri1, Index: "i", Frame: "f", View: "inverse", Slice: 5}, }, } @@ -316,4 +304,3 @@ func TestCluster_Resize(t *testing.T) { } }) } -*/ diff --git a/handler_test.go b/handler_test.go index 42048a224..0a540d059 100644 --- a/handler_test.go +++ b/handler_test.go @@ -74,7 +74,6 @@ func TestHandler_NotFound(t *testing.T) { } } -/* TODO travis: fix test // Ensure the handler can return the schema. func TestHandler_Schema(t *testing.T) { hldr := test.MustOpenHolder() @@ -107,6 +106,7 @@ func TestHandler_Schema(t *testing.T) { if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"rowLabel":"rowID","inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -147,11 +147,10 @@ func TestHandler_Status(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"status":{"State":"UP","Indexes":[{"Name":"i0","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}},{"Name":"f1","Meta":{"RowLabel":"rowID","InverseEnabled":true,"CacheType":"ranked","CacheSize":50000}}]},{"Name":"i1","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}}]}]}}`+"\n" { + } else if body := w.Body.String(); body != `{"status":{"State":"NORMAL","URISet":[{"Scheme":"http","Host":"localhost","Port":10101}]}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } -*/ // Ensure the handler can return the maxslice map. func TestHandler_MaxSlices(t *testing.T) { @@ -1185,7 +1184,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `[{"scheme":"http","host":"host2"},{"scheme":"http","host":"host0"}]`+"\n" { + } else if w.Body.String() != `[{"uri":{"scheme":"http","host":"host2"}},{"uri":{"scheme":"http","host":"host0"}}]`+"\n" { t.Fatalf("unexpected body: %q", w.Body.String()) } } diff --git a/test/handler.go b/test/handler.go index 44ca6d6e0..6d08bc6c4 100644 --- a/test/handler.go +++ b/test/handler.go @@ -11,6 +11,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) @@ -75,35 +76,19 @@ func NewServer() *Server { return s } -/* TODO travis: fix this test -// LocalStatus returns the state of the local node as well as the -// holder (indexes/frames) according to the local node. +// LocalStatus exists so that test.Server implements StatusHandler. func (s *Server) LocalStatus() (proto.Message, error) { - if s.Handler.Holder == nil { - return nil, errors.New("Server.Holder is nil") - } - - ns := internal.NodeStatus{ - Host: s.Handler.Handler.URI.HostPort(), - State: pilosa.NodeStateUp, - Indexes: pilosa.EncodeIndexes(s.Handler.Holder.Indexes()), - } - - // Append Slice list per this Node's indexes - for _, index := range ns.Indexes { - index.Slices = s.Handler.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.URI.HostPort()) - } - - return &ns, nil + return nil, nil } -// ClusterStatus returns the NodeState for all nodes in the cluster. +// ClusterStatus exists so that test.Server implements StatusHandler. func (s *Server) ClusterStatus() (proto.Message, error) { - // Assuming we are only testing this with one Node - // So just return its status - return s.LocalStatus() + uri := pilosa.DefaultURI() + return &internal.ClusterStatus{ + State: pilosa.NodeStateNormal, + URISet: []*internal.URI{uri.Encode()}, + }, nil } -*/ // HandleRemoteStatus just need to implement a nop to complete the Interface func (s *Server) HandleRemoteStatus(pb proto.Message) error { return nil } diff --git a/uri.go b/uri.go index 24d902f70..23aba4dfc 100644 --- a/uri.go +++ b/uri.go @@ -15,6 +15,7 @@ package pilosa import ( + "encoding/json" "errors" "fmt" "regexp" @@ -235,3 +236,32 @@ func decodeURIs(a []*internal.URI) []URI { } return other } + +// MarshalJSON marshals URI into a JSON-encoded byte slice. +func (u *URI) MarshalJSON() ([]byte, error) { + var output struct { + Scheme string `json:"scheme,omitempty"` + Host string `json:"host,omitempty"` + Port uint16 `json:"port,omitempty"` + } + output.Scheme = u.scheme + output.Host = u.host + output.Port = u.port + + return json.Marshal(output) +} + +func (u *URI) UnmarshalJSON(b []byte) error { + var input struct { + Scheme string `json:"scheme,omitempty"` + Host string `json:"host,omitempty"` + Port uint16 `json:"port,omitempty"` + } + if err := json.Unmarshal(b, &input); err != nil { + return err + } + u.scheme = input.Scheme + u.host = input.Host + u.port = input.Port + return nil +}