mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 02:44:59 +00:00
Merge branch 'master' into 'cluster-resize'
This commit is contained in:
Travis Turner
commit
e641aa1d8a
32 changed files with 768 additions and 256 deletions
8
Gopkg.lock
generated
8
Gopkg.lock
generated
|
|
@ -162,6 +162,12 @@
|
|||
packages = ["."]
|
||||
revision = "e2103e2c35297fb7e17febb81e49b312087a2372"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/sony/gobreaker"
|
||||
packages = ["."]
|
||||
revision = "e9556a45379ef1da12e54847edb2fb3d7d566f36"
|
||||
version = "0.3.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/spf13/afero"
|
||||
|
|
@ -231,6 +237,6 @@
|
|||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
inputs-digest = "72a71ef2a911e41396bea6c2639c4d2e6faa69ca74c3acb47fb249a054036685"
|
||||
inputs-digest = "4dd559e1f44fd2ad3032f1076f627d86736f47eac961527cddbef21b0c7c0006"
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
|
|
|
|||
226
client.go
226
client.go
|
|
@ -37,22 +37,22 @@ import (
|
|||
"github.com/pilosa/pilosa/internal"
|
||||
)
|
||||
|
||||
// ClientOptions represents the configuration for a Client
|
||||
// ClientOptions represents the configuration for a InternalHTTPClient
|
||||
type ClientOptions struct {
|
||||
TLS *tls.Config
|
||||
}
|
||||
|
||||
// Client represents a client to the Pilosa cluster.
|
||||
type Client struct {
|
||||
host *URI
|
||||
options *ClientOptions
|
||||
// InternalHTTPClient represents a client to the Pilosa cluster.
|
||||
type InternalHTTPClient struct {
|
||||
defaultURI *URI
|
||||
options *ClientOptions
|
||||
|
||||
// The client to use for HTTP communication.
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// NewClient returns a new instance of Client to connect to host.
|
||||
func NewClient(host string, options *ClientOptions) (*Client, error) {
|
||||
// NewInternalHTTPClient returns a new instance of InternalHTTPClient to connect to host.
|
||||
func NewInternalHTTPClient(host string, options *ClientOptions) (*InternalHTTPClient, error) {
|
||||
if host == "" {
|
||||
return nil, ErrHostRequired
|
||||
}
|
||||
|
|
@ -62,13 +62,11 @@ func NewClient(host string, options *ClientOptions) (*Client, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
return NewClientFromURI(uri, options)
|
||||
client := NewInternalHTTPClientFromURI(uri, options)
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func NewClientFromURI(uri *URI, options *ClientOptions) (*Client, error) {
|
||||
if uri == nil {
|
||||
return nil, ErrHostRequired
|
||||
}
|
||||
func NewInternalHTTPClientFromURI(defaultURI *URI, options *ClientOptions) *InternalHTTPClient {
|
||||
if options == nil {
|
||||
options = &ClientOptions{}
|
||||
}
|
||||
|
|
@ -77,29 +75,29 @@ func NewClientFromURI(uri *URI, options *ClientOptions) (*Client, error) {
|
|||
transport.TLSClientConfig = options.TLS
|
||||
}
|
||||
client := &http.Client{Transport: transport}
|
||||
return &Client{
|
||||
host: uri,
|
||||
return &InternalHTTPClient{
|
||||
defaultURI: defaultURI,
|
||||
HTTPClient: client,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Host returns the host the client was initialized with.
|
||||
func (c *Client) Host() *URI { return c.host }
|
||||
func (c *InternalHTTPClient) Host() *URI { return c.defaultURI }
|
||||
|
||||
// MaxSliceByIndex returns the number of slices on a server by index.
|
||||
func (c *Client) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) {
|
||||
func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) {
|
||||
return c.maxSliceByIndex(ctx, false)
|
||||
}
|
||||
|
||||
// MaxInverseSliceByIndex returns the number of inverse slices on a server by index.
|
||||
func (c *Client) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) {
|
||||
func (c *InternalHTTPClient) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) {
|
||||
return c.maxSliceByIndex(ctx, true)
|
||||
}
|
||||
|
||||
// maxSliceByIndex returns the number of slices on a server by index.
|
||||
func (c *Client) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) {
|
||||
func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) {
|
||||
// Execute request against the host.
|
||||
u := uriPathToURL(c.host, "/slices/max")
|
||||
u := uriPathToURL(c.clientURI(ctx), "/slices/max")
|
||||
|
||||
// Build request.
|
||||
req, err := http.NewRequest("GET", u.String(), nil)
|
||||
|
|
@ -130,12 +128,12 @@ func (c *Client) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]
|
|||
}
|
||||
|
||||
// Schema returns all index and frame schema information.
|
||||
func (c *Client) Schema(ctx context.Context) ([]*IndexInfo, error) {
|
||||
func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) {
|
||||
// Execute request against the host.
|
||||
u := uriPathToURL(c.host, "/schema")
|
||||
u := c.defaultURI.Path("/schema")
|
||||
|
||||
// Build request.
|
||||
req, err := http.NewRequest("GET", u.String(), nil)
|
||||
req, err := http.NewRequest("GET", u, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -159,7 +157,7 @@ func (c *Client) Schema(ctx context.Context) ([]*IndexInfo, error) {
|
|||
}
|
||||
|
||||
// CreateIndex creates a new index on the server.
|
||||
func (c *Client) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
|
||||
func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
|
||||
// Encode query request.
|
||||
buf, err := json.Marshal(&postIndexRequest{
|
||||
Options: opt,
|
||||
|
|
@ -169,7 +167,7 @@ func (c *Client) CreateIndex(ctx context.Context, index string, opt IndexOptions
|
|||
}
|
||||
|
||||
// Create URL & HTTP request.
|
||||
u := uriPathToURL(c.host, fmt.Sprintf("/index/%s", index))
|
||||
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index))
|
||||
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -204,9 +202,9 @@ func (c *Client) CreateIndex(ctx context.Context, index string, opt IndexOptions
|
|||
}
|
||||
|
||||
// FragmentNodes returns a list of nodes that own a slice.
|
||||
func (c *Client) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) {
|
||||
func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) {
|
||||
// Execute request against the host.
|
||||
u := uriPathToURL(c.host, "/fragment/nodes")
|
||||
u := uriPathToURL(c.defaultURI, "/fragment/nodes")
|
||||
u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode()
|
||||
|
||||
// Build request.
|
||||
|
|
@ -235,28 +233,26 @@ func (c *Client) FragmentNodes(ctx context.Context, index string, slice uint64)
|
|||
}
|
||||
|
||||
// ExecuteQuery executes query against index on the server.
|
||||
func (c *Client) ExecuteQuery(ctx context.Context, index, query string, allowRedirect bool) (result interface{}, err error) {
|
||||
func (c *InternalHTTPClient) ExecuteQuery(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
|
||||
if index == "" {
|
||||
return nil, ErrIndexRequired
|
||||
} else if query == "" {
|
||||
} else if queryRequest.Query == "" {
|
||||
return nil, ErrQueryRequired
|
||||
}
|
||||
|
||||
// Encode query request.
|
||||
buf, err := proto.Marshal(&internal.QueryRequest{
|
||||
Query: query,
|
||||
Remote: !allowRedirect,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal: %s", err)
|
||||
}
|
||||
|
||||
// Create URL & HTTP request.
|
||||
u := uriPathToURL(c.host, fmt.Sprintf("/index/%s/query", index))
|
||||
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
|
||||
// Encode request object.
|
||||
buf, err := proto.Marshal(queryRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create HTTP request.
|
||||
u := c.clientURI(ctx).Path(fmt.Sprintf("/index/%s/query", index))
|
||||
req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
|
||||
req.Header.Set("Content-Type", "application/x-protobuf")
|
||||
req.Header.Set("Accept", "application/x-protobuf")
|
||||
|
|
@ -277,8 +273,8 @@ func (c *Client) ExecuteQuery(ctx context.Context, index, query string, allowRed
|
|||
return nil, errors.New(string(body))
|
||||
}
|
||||
|
||||
var qresp internal.QueryResponse
|
||||
if err := proto.Unmarshal(body, &qresp); err != nil {
|
||||
qresp := &internal.QueryResponse{}
|
||||
if err := proto.Unmarshal(body, qresp); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal response: %s", err)
|
||||
} else if s := qresp.Err; s != "" {
|
||||
return nil, errors.New(s)
|
||||
|
|
@ -287,43 +283,15 @@ func (c *Client) ExecuteQuery(ctx context.Context, index, query string, allowRed
|
|||
return qresp, nil
|
||||
}
|
||||
|
||||
// ExecutePQL executes query string against index on the server.
|
||||
func (c *Client) ExecutePQL(ctx context.Context, index, query string) (interface{}, error) {
|
||||
u := uriPathToURL(c.host, "/query")
|
||||
u.RawQuery = url.Values{"index": {index}}.Encode()
|
||||
|
||||
req, err := http.NewRequest("POST", u.String(), bytes.NewReader([]byte(query)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "pilosa/"+Version)
|
||||
|
||||
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if resp.StatusCode != http.StatusOK {
|
||||
return nil, errors.New(string(body))
|
||||
}
|
||||
return string(body), nil
|
||||
|
||||
}
|
||||
|
||||
// Import bulk imports bits for a single slice to a host.
|
||||
func (c *Client) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error {
|
||||
func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error {
|
||||
if index == "" {
|
||||
return ErrIndexRequired
|
||||
} else if frame == "" {
|
||||
return ErrFrameRequired
|
||||
}
|
||||
|
||||
buf, err := MarshalImportPayload(index, frame, slice, bits)
|
||||
buf, err := marshalImportPayload(index, frame, slice, bits)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error Creating Payload: %s", err)
|
||||
}
|
||||
|
|
@ -344,7 +312,7 @@ func (c *Client) Import(ctx context.Context, index, frame string, slice uint64,
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
|
||||
func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
|
||||
err := c.CreateIndex(ctx, name, options)
|
||||
if err == nil || err == ErrIndexExists {
|
||||
return nil
|
||||
|
|
@ -352,7 +320,7 @@ func (c *Client) EnsureIndex(ctx context.Context, name string, options IndexOpti
|
|||
return err
|
||||
}
|
||||
|
||||
func (c *Client) EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error {
|
||||
func (c *InternalHTTPClient) EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error {
|
||||
err := c.CreateFrame(ctx, indexName, frameName, options)
|
||||
if err == nil || err == ErrFrameExists {
|
||||
return nil
|
||||
|
|
@ -360,8 +328,8 @@ func (c *Client) EnsureFrame(ctx context.Context, indexName string, frameName st
|
|||
return err
|
||||
}
|
||||
|
||||
// MarshalImportPayload marshalls the import parameters into a protobuf byte slice.
|
||||
func MarshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) {
|
||||
// marshalImportPayload marshalls the import parameters into a protobuf byte slice.
|
||||
func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) {
|
||||
// Separate row and column IDs to reduce allocations.
|
||||
rowIDs := Bits(bits).RowIDs()
|
||||
columnIDs := Bits(bits).ColumnIDs()
|
||||
|
|
@ -383,7 +351,7 @@ func MarshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte
|
|||
}
|
||||
|
||||
// importNode sends a pre-marshaled import request to a node.
|
||||
func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error {
|
||||
func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []byte) error {
|
||||
// Create URL & HTTP request.
|
||||
u := nodePathToURL(node, "/import")
|
||||
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
|
||||
|
|
@ -421,14 +389,14 @@ func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error {
|
|||
}
|
||||
|
||||
// ImportValue bulk imports field values for a single slice to a host.
|
||||
func (c *Client) ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error {
|
||||
func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error {
|
||||
if index == "" {
|
||||
return ErrIndexRequired
|
||||
} else if frame == "" {
|
||||
return ErrFrameRequired
|
||||
}
|
||||
|
||||
buf, err := MarshalImportValuePayload(index, frame, field, slice, vals)
|
||||
buf, err := marshalImportValuePayload(index, frame, field, slice, vals)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error Creating Payload: %s", err)
|
||||
}
|
||||
|
|
@ -449,8 +417,8 @@ func (c *Client) ImportValue(ctx context.Context, index, frame, field string, sl
|
|||
return nil
|
||||
}
|
||||
|
||||
// MarshalImportValuePayload marshalls the import parameters into a protobuf byte slice.
|
||||
func MarshalImportValuePayload(index, frame, field string, slice uint64, vals []FieldValue) ([]byte, error) {
|
||||
// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice.
|
||||
func marshalImportValuePayload(index, frame, field string, slice uint64, vals []FieldValue) ([]byte, error) {
|
||||
// Separate row and column IDs to reduce allocations.
|
||||
columnIDs := FieldValues(vals).ColumnIDs()
|
||||
values := FieldValues(vals).Values()
|
||||
|
|
@ -471,7 +439,7 @@ func MarshalImportValuePayload(index, frame, field string, slice uint64, vals []
|
|||
}
|
||||
|
||||
// importValueNode sends a pre-marshaled import request to a node.
|
||||
func (c *Client) importValueNode(ctx context.Context, node *Node, buf []byte) error {
|
||||
func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, buf []byte) error {
|
||||
// Create URL & HTTP request.
|
||||
u := nodePathToURL(node, "/import-value")
|
||||
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
|
||||
|
|
@ -509,7 +477,7 @@ func (c *Client) importValueNode(ctx context.Context, node *Node, buf []byte) er
|
|||
}
|
||||
|
||||
// ExportCSV bulk exports data for a single slice from a host to CSV format.
|
||||
func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error {
|
||||
func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error {
|
||||
if index == "" {
|
||||
return ErrIndexRequired
|
||||
} else if frame == "" {
|
||||
|
|
@ -541,7 +509,7 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice
|
|||
}
|
||||
|
||||
// exportNode copies a CSV export from a node to w.
|
||||
func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame, view string, slice uint64, w io.Writer) error {
|
||||
func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, frame, view string, slice uint64, w io.Writer) error {
|
||||
// Create URL.
|
||||
u := nodePathToURL(node, "/export")
|
||||
u.RawQuery = url.Values{
|
||||
|
|
@ -580,7 +548,7 @@ func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame, vi
|
|||
}
|
||||
|
||||
// BackupTo backs up an entire frame from a cluster to w.
|
||||
func (c *Client) BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error {
|
||||
func (c *InternalHTTPClient) BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error {
|
||||
if index == "" {
|
||||
return ErrIndexRequired
|
||||
} else if frame == "" {
|
||||
|
|
@ -621,7 +589,7 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, index, frame, view s
|
|||
}
|
||||
|
||||
// backupSliceTo backs up a single slice to tw.
|
||||
func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame, view string, slice uint64) error {
|
||||
func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame, view string, slice uint64) error {
|
||||
// Return error if unable to backup from any slice.
|
||||
r, err := c.BackupSlice(ctx, index, frame, view, slice)
|
||||
if err != nil {
|
||||
|
|
@ -659,7 +627,7 @@ func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame
|
|||
|
||||
// BackupSlice retrieves a streaming backup from a single slice.
|
||||
// This function tries slice owners until one succeeds.
|
||||
func (c *Client) BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) {
|
||||
func (c *InternalHTTPClient) BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) {
|
||||
// Retrieve a list of nodes that own the slice.
|
||||
nodes, err := c.FragmentNodes(ctx, index, slice)
|
||||
if err != nil {
|
||||
|
|
@ -682,14 +650,14 @@ func (c *Client) BackupSlice(ctx context.Context, index, frame, view string, sli
|
|||
return nil, fmt.Errorf("unable to connect to any owner")
|
||||
}
|
||||
|
||||
func (c *Client) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) {
|
||||
func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) {
|
||||
node := &Node{
|
||||
URI: uri,
|
||||
}
|
||||
return c.backupSliceNode(ctx, index, frame, view, slice, node)
|
||||
}
|
||||
|
||||
func (c *Client) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) {
|
||||
func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) {
|
||||
u := nodePathToURL(node, "/fragment/data")
|
||||
u.RawQuery = url.Values{
|
||||
"index": {index},
|
||||
|
|
@ -725,7 +693,7 @@ func (c *Client) backupSliceNode(ctx context.Context, index, frame, view string,
|
|||
}
|
||||
|
||||
// RestoreFrom restores a frame from a backup file to an entire cluster.
|
||||
func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error {
|
||||
func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error {
|
||||
if index == "" {
|
||||
return ErrIndexRequired
|
||||
} else if frame == "" {
|
||||
|
|
@ -764,7 +732,7 @@ func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, index, frame, vie
|
|||
}
|
||||
|
||||
// restoreSliceFrom restores a single slice to all owning nodes.
|
||||
func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, view string, slice uint64) error {
|
||||
func (c *InternalHTTPClient) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, view string, slice uint64) error {
|
||||
// Retrieve a list of nodes that own the slice.
|
||||
nodes, err := c.FragmentNodes(ctx, index, slice)
|
||||
if err != nil {
|
||||
|
|
@ -805,7 +773,7 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, index, frame,
|
|||
}
|
||||
|
||||
// CreateFrame creates a new frame on the server.
|
||||
func (c *Client) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error {
|
||||
func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error {
|
||||
if index == "" {
|
||||
return ErrIndexRequired
|
||||
}
|
||||
|
|
@ -819,7 +787,7 @@ func (c *Client) CreateFrame(ctx context.Context, index, frame string, opt Frame
|
|||
}
|
||||
|
||||
// Create URL & HTTP request.
|
||||
u := uriPathToURL(c.host, fmt.Sprintf("/index/%s/frame/%s", index, frame))
|
||||
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s", index, frame))
|
||||
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -854,8 +822,8 @@ func (c *Client) CreateFrame(ctx context.Context, index, frame string, opt Frame
|
|||
}
|
||||
|
||||
// RestoreFrame restores an entire frame from a host in another cluster.
|
||||
func (c *Client) RestoreFrame(ctx context.Context, host, index, frame string) error {
|
||||
u := uriPathToURL(c.host, fmt.Sprintf("/index/%s/frame/%s/restore", index, frame))
|
||||
func (c *InternalHTTPClient) RestoreFrame(ctx context.Context, host, index, frame string) error {
|
||||
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/restore", index, frame))
|
||||
u.RawQuery = url.Values{
|
||||
"host": {host},
|
||||
}.Encode()
|
||||
|
|
@ -884,9 +852,9 @@ func (c *Client) RestoreFrame(ctx context.Context, host, index, frame string) er
|
|||
}
|
||||
|
||||
// FrameViews returns a list of view names for a frame.
|
||||
func (c *Client) FrameViews(ctx context.Context, index, frame string) ([]string, error) {
|
||||
func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string) ([]string, error) {
|
||||
// Create URL & HTTP request.
|
||||
u := uriPathToURL(c.host, fmt.Sprintf("/index/%s/frame/%s/views", index, frame))
|
||||
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/views", index, frame))
|
||||
req, err := http.NewRequest("GET", u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -921,8 +889,8 @@ func (c *Client) FrameViews(ctx context.Context, index, frame string) ([]string,
|
|||
|
||||
// FragmentBlocks returns a list of block checksums for a fragment on a host.
|
||||
// Only returns blocks which contain data.
|
||||
func (c *Client) FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error) {
|
||||
u := uriPathToURL(c.host, "/fragment/blocks")
|
||||
func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error) {
|
||||
u := uriPathToURL(c.defaultURI, "/fragment/blocks")
|
||||
u.RawQuery = url.Values{
|
||||
"index": {index},
|
||||
"frame": {frame},
|
||||
|
|
@ -963,7 +931,7 @@ func (c *Client) FragmentBlocks(ctx context.Context, index, frame, view string,
|
|||
}
|
||||
|
||||
// BlockData returns row/column id pairs for a block.
|
||||
func (c *Client) BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) {
|
||||
func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) {
|
||||
buf, err := proto.Marshal(&internal.BlockDataRequest{
|
||||
Index: index,
|
||||
Frame: frame,
|
||||
|
|
@ -975,7 +943,7 @@ func (c *Client) BlockData(ctx context.Context, index, frame, view string, slice
|
|||
return nil, nil, err
|
||||
}
|
||||
|
||||
u := uriPathToURL(c.host, "/fragment/block/data")
|
||||
u := uriPathToURL(c.defaultURI, "/fragment/block/data")
|
||||
req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
|
|
@ -1011,8 +979,8 @@ func (c *Client) BlockData(ctx context.Context, index, frame, view string, slice
|
|||
}
|
||||
|
||||
// ColumnAttrDiff returns data from differing blocks on a remote host.
|
||||
func (c *Client) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
|
||||
u := uriPathToURL(c.host, fmt.Sprintf("/index/%s/attr/diff", index))
|
||||
func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
|
||||
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/attr/diff", index))
|
||||
|
||||
// Encode request.
|
||||
buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks})
|
||||
|
|
@ -1051,8 +1019,8 @@ func (c *Client) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBl
|
|||
}
|
||||
|
||||
// RowAttrDiff returns data from differing blocks on a remote host.
|
||||
func (c *Client) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
|
||||
u := uriPathToURL(c.host, fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, frame))
|
||||
func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
|
||||
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, frame))
|
||||
|
||||
// Encode request.
|
||||
buf, err := json.Marshal(postFrameAttrDiffRequest{Blocks: blks})
|
||||
|
|
@ -1092,6 +1060,16 @@ func (c *Client) RowAttrDiff(ctx context.Context, index, frame string, blks []At
|
|||
return rsp.Attrs, nil
|
||||
}
|
||||
|
||||
func (c *InternalHTTPClient) clientURI(ctx context.Context) *URI {
|
||||
clientURI := c.defaultURI
|
||||
if contextURI, ok := ctx.Value("uri").(*URI); ok {
|
||||
clientURI = contextURI
|
||||
} else if contextURI, ok := ctx.Value("uri").(URI); ok {
|
||||
clientURI = &contextURI
|
||||
}
|
||||
return clientURI
|
||||
}
|
||||
|
||||
// Bit represents the location of a single bit.
|
||||
type Bit struct {
|
||||
RowID uint64
|
||||
|
|
@ -1162,7 +1140,7 @@ func (p Bits) GroupBySlice() map[uint64][]Bit {
|
|||
// range-encoded frame.
|
||||
type FieldValue struct {
|
||||
ColumnID uint64
|
||||
Value uint64
|
||||
Value int64
|
||||
}
|
||||
|
||||
// FieldValues represents a slice of field values.
|
||||
|
|
@ -1185,8 +1163,8 @@ func (p FieldValues) ColumnIDs() []uint64 {
|
|||
}
|
||||
|
||||
// Values returns a slice of all the values.
|
||||
func (p FieldValues) Values() []uint64 {
|
||||
other := make([]uint64, len(p))
|
||||
func (p FieldValues) Values() []int64 {
|
||||
other := make([]int64, len(p))
|
||||
for i := range p {
|
||||
other[i] = p[i].Value
|
||||
}
|
||||
|
|
@ -1237,3 +1215,33 @@ func nodePathToURL(node *Node, path string) url.URL {
|
|||
Path: path,
|
||||
}
|
||||
}
|
||||
|
||||
// InternalClient should be implemented by any struct that enables any transport between nodes
|
||||
// TODO: Refactor
|
||||
// Note from Travis: Typically an interface containing more than two or three methods is an indication that
|
||||
// something hasn't been architected correctly.
|
||||
// While I understand that putting the entire Client behind an interface might require this many methods,
|
||||
// I don't want to let it go unquestioned.
|
||||
type InternalClient interface {
|
||||
MaxSliceByIndex(ctx context.Context) (map[string]uint64, error)
|
||||
MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error)
|
||||
Schema(ctx context.Context) ([]*IndexInfo, error)
|
||||
CreateIndex(ctx context.Context, index string, opt IndexOptions) error
|
||||
FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error)
|
||||
ExecuteQuery(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
|
||||
Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error
|
||||
EnsureIndex(ctx context.Context, name string, options IndexOptions) error
|
||||
EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error
|
||||
ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error
|
||||
ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error
|
||||
BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error
|
||||
BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error)
|
||||
RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error
|
||||
CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error
|
||||
RestoreFrame(ctx context.Context, host, index, frame string) error
|
||||
FrameViews(ctx context.Context, index, frame string) ([]string, error)
|
||||
FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error)
|
||||
BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error)
|
||||
ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
|
||||
RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -137,15 +137,17 @@ func TestClient_MultiNode(t *testing.T) {
|
|||
client[2] = test.MustNewClient(s[2].Host())
|
||||
|
||||
topN := 4
|
||||
q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN)
|
||||
|
||||
result, err := client[0].ExecuteQuery(context.Background(), "i", q, true)
|
||||
queryRequest := &internal.QueryRequest{
|
||||
Query: fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN),
|
||||
Remote: false,
|
||||
}
|
||||
result, err := client[0].ExecuteQuery(context.Background(), "i", queryRequest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Check the results before every node has the correct max slice value.
|
||||
pairs := result.(internal.QueryResponse).Results[0].Pairs
|
||||
pairs := result.Results[0].Pairs
|
||||
for _, pair := range pairs {
|
||||
if pair.Key == 22 && pair.Count != 3 {
|
||||
t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair)
|
||||
|
|
@ -157,13 +159,13 @@ func TestClient_MultiNode(t *testing.T) {
|
|||
hldr[1].Index("i").SetRemoteMaxSlice(maxSlice)
|
||||
hldr[2].Index("i").SetRemoteMaxSlice(maxSlice)
|
||||
|
||||
result, err = client[0].ExecuteQuery(context.Background(), "i", q, true)
|
||||
result, err = client[0].ExecuteQuery(context.Background(), "i", queryRequest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Test must return exactly N results.
|
||||
if len(result.(internal.QueryResponse).Results[0].Pairs) != topN {
|
||||
if len(result.Results[0].Pairs) != topN {
|
||||
t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result))
|
||||
}
|
||||
p := []*internal.Pair{
|
||||
|
|
@ -173,15 +175,15 @@ func TestClient_MultiNode(t *testing.T) {
|
|||
{Key: 99, Count: 7}}
|
||||
|
||||
// Valdidate the Top 4 result counts.
|
||||
if !reflect.DeepEqual(result.(internal.QueryResponse).Results[0].Pairs, p) {
|
||||
if !reflect.DeepEqual(result.Results[0].Pairs, p) {
|
||||
t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
|
||||
}
|
||||
|
||||
result1, err := client[1].ExecuteQuery(context.Background(), "i", q, true)
|
||||
result1, err := client[1].ExecuteQuery(context.Background(), "i", queryRequest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result2, err := client[2].ExecuteQuery(context.Background(), "i", q, true)
|
||||
result2, err := client[2].ExecuteQuery(context.Background(), "i", queryRequest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -294,7 +296,7 @@ func TestClient_ImportValue(t *testing.T) {
|
|||
fld := pilosa.Field{
|
||||
Name: "fld",
|
||||
Type: pilosa.FieldTypeInt,
|
||||
Min: 0,
|
||||
Min: -100,
|
||||
Max: 100,
|
||||
}
|
||||
|
||||
|
|
@ -315,7 +317,7 @@ func TestClient_ImportValue(t *testing.T) {
|
|||
// Send import request.
|
||||
c := test.MustNewClient(s.Host())
|
||||
if err := c.ImportValue(context.Background(), "i", "f", fld.Name, 0, []pilosa.FieldValue{
|
||||
{ColumnID: 1, Value: 10},
|
||||
{ColumnID: 1, Value: -10},
|
||||
{ColumnID: 2, Value: 20},
|
||||
{ColumnID: 3, Value: 40},
|
||||
}); err != nil {
|
||||
|
|
@ -328,7 +330,7 @@ func TestClient_ImportValue(t *testing.T) {
|
|||
}
|
||||
|
||||
// Verify data.
|
||||
if sum != 70 || cnt != 3 {
|
||||
if sum != 50 || cnt != 3 {
|
||||
t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=70, cnt=3", sum, cnt)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
13
cluster.go
13
cluster.go
|
|
@ -746,10 +746,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
|
|||
}
|
||||
|
||||
// Create a client for calling remote nodes.
|
||||
client, err := NewClientFromURI(&c.URI, nil) // TODO: ClientOptions
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client := NewInternalHTTPClientFromURI(&c.URI, nil) // TODO: ClientOptions
|
||||
|
||||
// Request each source file in ResizeSources.
|
||||
for _, src := range instr.Sources {
|
||||
|
|
@ -966,6 +963,14 @@ func (u NodeSet) ToHostPortStrings() []string {
|
|||
return other
|
||||
}
|
||||
|
||||
func (u NodeSet) ToStrings() []string {
|
||||
other := make([]string, 0, len(u))
|
||||
for _, uri := range u {
|
||||
other = append(other, uri.String())
|
||||
}
|
||||
return other
|
||||
}
|
||||
|
||||
// Topology represents the list of hosts in the cluster.
|
||||
type Topology struct {
|
||||
mu sync.RWMutex
|
||||
|
|
|
|||
|
|
@ -99,6 +99,7 @@ type Config struct {
|
|||
Service string `toml:"service"`
|
||||
Host string `toml:"host"`
|
||||
PollInterval Duration `toml:"poll-interval"`
|
||||
Diagnostics bool `toml:"diagnostics"`
|
||||
} `toml:"metric"`
|
||||
|
||||
TLS TLSConfig
|
||||
|
|
@ -115,6 +116,7 @@ func NewConfig() *Config {
|
|||
c.Cluster.Hosts = []string{}
|
||||
c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval)
|
||||
c.Metric.Service = DefaultMetrics
|
||||
c.Metric.Diagnostics = true
|
||||
c.TLS = TLSConfig{}
|
||||
return c
|
||||
}
|
||||
|
|
|
|||
11
ctl/bench.go
11
ctl/bench.go
|
|
@ -23,6 +23,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/pilosa/pilosa"
|
||||
"github.com/pilosa/pilosa/internal"
|
||||
)
|
||||
|
||||
// BenchCommand represents a command for benchmarking index operations.
|
||||
|
|
@ -70,7 +71,7 @@ func (cmd *BenchCommand) Run(ctx context.Context) error {
|
|||
}
|
||||
|
||||
// runSetBit executes a benchmark of random SetBit() operations.
|
||||
func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) error {
|
||||
func (cmd *BenchCommand) runSetBit(ctx context.Context, client pilosa.InternalClient) error {
|
||||
if cmd.N == 0 {
|
||||
return errors.New("operation count required")
|
||||
} else if cmd.Index == "" {
|
||||
|
|
@ -89,9 +90,11 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) e
|
|||
rowID := rand.Intn(maxRowID)
|
||||
columnID := rand.Intn(maxColumnID)
|
||||
|
||||
q := fmt.Sprintf(`SetBit(id=%d, frame="%s", columnID=%d)`, rowID, cmd.Frame, columnID)
|
||||
|
||||
if _, err := client.ExecuteQuery(ctx, cmd.Index, q, true); err != nil {
|
||||
queryRequest := &internal.QueryRequest{
|
||||
Query: fmt.Sprintf(`SetBit(id=%d, frame="%s", columnID=%d)`, rowID, cmd.Frame, columnID),
|
||||
Remote: false,
|
||||
}
|
||||
if _, err := client.ExecuteQuery(ctx, cmd.Index, queryRequest); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP
|
|||
flags.BoolVarP(skipVerify, "tls.skip-verify", "", false, "Skip TLS certificate verification (not secure)")
|
||||
}
|
||||
|
||||
// CommandClient returns a pilosa.Client for the command
|
||||
func CommandClient(cmd CommandWithTLSSupport) (*pilosa.Client, error) {
|
||||
// CommandClient returns a pilosa.InternalHTTPClient for the command
|
||||
func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error) {
|
||||
tlsConfig := cmd.TLSConfiguration()
|
||||
var clientOptions *pilosa.ClientOptions
|
||||
if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" {
|
||||
|
|
@ -34,7 +34,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.Client, error) {
|
|||
}
|
||||
clientOptions = &pilosa.ClientOptions{TLS: TLSConfig}
|
||||
}
|
||||
client, err := pilosa.NewClient(cmd.TLSHost(), clientOptions)
|
||||
client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), clientOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ type ImportCommand struct {
|
|||
Sort bool `json:"sort"`
|
||||
|
||||
// Reusable client.
|
||||
Client *pilosa.Client `json:"-"`
|
||||
Client pilosa.InternalClient `json:"-"`
|
||||
|
||||
// Standard input/output
|
||||
*pilosa.CmdIO
|
||||
|
|
@ -293,7 +293,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
|
|||
val.ColumnID = columnID
|
||||
|
||||
// Parse field value.
|
||||
value, err := strconv.ParseUint(record[1], 10, 64)
|
||||
value, err := strconv.ParseInt(record[1], 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid value on row %d: %q", rnum, record[1])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
|
|||
flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "gossip", "Determine how the cluster handles membership and state sharing. Choose from [static, gossip]")
|
||||
flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.")
|
||||
flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.")
|
||||
flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", true, "Enabled diagnostics reporting.")
|
||||
flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.")
|
||||
SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify)
|
||||
}
|
||||
|
|
|
|||
216
diagnostics/diagnostics.go
Normal file
216
diagnostics/diagnostics.go
Normal file
|
|
@ -0,0 +1,216 @@
|
|||
package diagnostics
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/sony/gobreaker"
|
||||
)
|
||||
|
||||
// TODO: unique Cluster ID
|
||||
|
||||
// Default version check URL.
|
||||
const (
|
||||
DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version"
|
||||
)
|
||||
|
||||
type versionResponse struct {
|
||||
Version string `json:"version"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// Diagnostics represents a client to the Pilosa cluster.
|
||||
type Diagnostics struct {
|
||||
mu sync.Mutex
|
||||
wg sync.WaitGroup
|
||||
closing chan struct{}
|
||||
host string
|
||||
VersionURL string
|
||||
version string
|
||||
lastVersion string
|
||||
startTime int64
|
||||
start time.Time
|
||||
|
||||
metrics map[string]interface{}
|
||||
|
||||
client *http.Client
|
||||
interval time.Duration
|
||||
|
||||
cb *gobreaker.CircuitBreaker
|
||||
logOutput io.Writer
|
||||
}
|
||||
|
||||
// New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port".
|
||||
func New(host string) *Diagnostics {
|
||||
|
||||
return &Diagnostics{
|
||||
closing: make(chan struct{}),
|
||||
host: host,
|
||||
VersionURL: DefaultVersionCheckURL,
|
||||
startTime: time.Now().Unix(),
|
||||
start: time.Now(),
|
||||
client: http.DefaultClient,
|
||||
metrics: make(map[string]interface{}),
|
||||
logOutput: ioutil.Discard,
|
||||
}
|
||||
}
|
||||
|
||||
// SetVersion of locally running Pilosa Cluster to check against master.
|
||||
func (d *Diagnostics) SetVersion(v string) {
|
||||
d.version = v
|
||||
d.Set("Version", v)
|
||||
}
|
||||
|
||||
// SetInterval of the diagnostic go routine and match with the circuit breaker timeout.
|
||||
func (d *Diagnostics) SetInterval(i time.Duration) {
|
||||
d.interval = i
|
||||
}
|
||||
|
||||
// schedule start the diagnostics service ticker.
|
||||
func (d *Diagnostics) schedule() {
|
||||
ticker := time.NewTicker(d.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-d.closing:
|
||||
return
|
||||
case <-ticker.C:
|
||||
d.CheckVersion()
|
||||
d.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush sends the current metrics.
|
||||
func (d *Diagnostics) Flush() error {
|
||||
d.mu.Lock()
|
||||
d.metrics["uptime"] = (time.Now().Unix() - d.startTime)
|
||||
buf, _ := d.Encode()
|
||||
d.mu.Unlock()
|
||||
|
||||
_, err := d.cb.Execute(func() (interface{}, error) {
|
||||
req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := d.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// TODO verify response
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return body, nil
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Open configures the circuit breaker used by the HTTP client.
|
||||
func (d *Diagnostics) Open() {
|
||||
var st gobreaker.Settings
|
||||
if d.interval > 0 {
|
||||
st.Timeout = d.interval * 2
|
||||
}
|
||||
d.cb = gobreaker.NewCircuitBreaker(st)
|
||||
|
||||
d.logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every hour. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics")
|
||||
}
|
||||
|
||||
// Close notify goroutine to stop.
|
||||
func (d *Diagnostics) Close() error {
|
||||
close(d.closing)
|
||||
d.wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// CheckVersion of the local build against Pilosa master.
|
||||
func (d *Diagnostics) CheckVersion() error {
|
||||
var rsp versionResponse
|
||||
req, err := http.NewRequest("GET", d.VersionURL, nil)
|
||||
resp, err := d.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("http: status=%d", resp.StatusCode)
|
||||
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
|
||||
return fmt.Errorf("json decode: %s", err)
|
||||
}
|
||||
|
||||
// Same a version as last test
|
||||
if rsp.Version == d.lastVersion {
|
||||
return nil
|
||||
}
|
||||
|
||||
d.lastVersion = rsp.Version
|
||||
if err := d.CompareVersion(rsp.Version); err != nil {
|
||||
d.logger().Printf("%s\n", err.Error())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CompareVersion check version strings.
|
||||
func (d *Diagnostics) CompareVersion(value string) error {
|
||||
currentVersion := VersionSegments(value)
|
||||
localVersion := VersionSegments(d.version)
|
||||
|
||||
if localVersion[0] < currentVersion[0] { //Major
|
||||
return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value)
|
||||
} else if localVersion[1] < currentVersion[1] { // Minor
|
||||
return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value)
|
||||
} else if localVersion[2] < currentVersion[2] { // Patch
|
||||
return fmt.Errorf("There is a new patch release of Pilosa availbale: %s: https://github.com/pilosa/pilosa/releases", value)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Encode metrics maps into the json message format.
|
||||
func (d *Diagnostics) Encode() ([]byte, error) {
|
||||
return json.Marshal(d.metrics)
|
||||
}
|
||||
|
||||
// Set adds a key value metric.
|
||||
func (d *Diagnostics) Set(name string, value interface{}) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.metrics[name] = value
|
||||
}
|
||||
|
||||
// SetLogger Set the logger output type.
|
||||
func (d *Diagnostics) SetLogger(logger io.Writer) {
|
||||
d.logOutput = logger
|
||||
}
|
||||
|
||||
// logger returns a logger that writes to LogOutput.
|
||||
func (d *Diagnostics) logger() *log.Logger {
|
||||
return log.New(d.logOutput, "", log.LstdFlags)
|
||||
}
|
||||
|
||||
// VersionSegments returns the numeric segments of the version as a slice of ints.
|
||||
func VersionSegments(segments string) []int {
|
||||
segments = strings.Trim(segments, "v")
|
||||
segments = strings.Split(segments, "-")[0]
|
||||
s := strings.Split(segments, ".")
|
||||
segmentSlice := make([]int, len(s))
|
||||
for i, v := range s {
|
||||
segmentSlice[i], _ = strconv.Atoi(v)
|
||||
}
|
||||
return segmentSlice
|
||||
}
|
||||
158
diagnostics/diagnostics_test.go
Normal file
158
diagnostics/diagnostics_test.go
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
package diagnostics_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/pilosa/pilosa/diagnostics"
|
||||
)
|
||||
|
||||
func TestDiagnosticsClient(t *testing.T) {
|
||||
// Mock server.
|
||||
server := httptest.NewServer(nil)
|
||||
defer server.Close()
|
||||
|
||||
// Create a new client.
|
||||
d := diagnostics.New(server.URL)
|
||||
d.SetLogger(ioutil.Discard)
|
||||
d.Open()
|
||||
defer d.Close()
|
||||
|
||||
d.Set("gg", 10)
|
||||
d.Set("ss", "ss")
|
||||
|
||||
data, err := d.Encode()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Test the recorded metrics, note that some types are skipped.
|
||||
var eq bool
|
||||
output1 := []byte(`{"gg":10,"ss":"ss"}`)
|
||||
if eq, err = compareJSON(data, output1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !eq {
|
||||
t.Fatalf("unexpected diagnostics: %+v", string(data))
|
||||
}
|
||||
|
||||
// Test the metrics after a flush.
|
||||
d.Flush()
|
||||
data, err = d.Encode()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
output2 := []byte(`{"gg":10,"ss":"ss","uptime":0}`)
|
||||
if eq, err = compareJSON(data, output2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !eq {
|
||||
t.Fatalf("unexpected diagnostics after flush: %+v", string(data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsVersion_Parse(t *testing.T) {
|
||||
version := "0.1.1"
|
||||
vs := diagnostics.VersionSegments(version)
|
||||
|
||||
output := []int{0, 1, 1}
|
||||
if !reflect.DeepEqual(vs, output) {
|
||||
t.Fatalf("unexpected version: %+v", vs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsVersion_Compare(t *testing.T) {
|
||||
d := diagnostics.New("localhost:10101")
|
||||
d.Open()
|
||||
defer d.Close()
|
||||
|
||||
version := "v0.1.1"
|
||||
d.SetVersion(version)
|
||||
|
||||
err := d.CompareVersion("v1.7.0")
|
||||
if !strings.Contains(err.Error(), "A newer version") {
|
||||
t.Fatalf("Expected a newer version is available, actual error: %s", err)
|
||||
}
|
||||
err = d.CompareVersion("1.7.0")
|
||||
if !strings.Contains(err.Error(), "A newer version") {
|
||||
t.Fatalf("Expected a newer version is available, actual error: %s", err)
|
||||
}
|
||||
err = d.CompareVersion("0.7.0")
|
||||
if !strings.Contains(err.Error(), "The latest Minor release is") {
|
||||
t.Fatalf("Expected Minor Version Missmatch, actual error: %s", err)
|
||||
}
|
||||
err = d.CompareVersion("0.1.2")
|
||||
if !strings.Contains(err.Error(), "There is a new patch release of Pilosa") {
|
||||
t.Fatalf("Expected Patch Version Missmatch, actual error: %s", err)
|
||||
}
|
||||
err = d.CompareVersion("0.1.1")
|
||||
if err != nil {
|
||||
t.Fatalf("Versions should match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsVersion_Check(t *testing.T) {
|
||||
// Mock server.
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
json.NewEncoder(w).Encode(versionResponse{
|
||||
Version: "1.1.1",
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
// Create a new client.
|
||||
d := diagnostics.New("localhost:10101")
|
||||
defer d.Close()
|
||||
|
||||
version := "0.1.1"
|
||||
d.SetVersion(version)
|
||||
d.VersionURL = server.URL
|
||||
|
||||
d.CheckVersion()
|
||||
}
|
||||
|
||||
type versionResponse struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
func compareJSON(a, b []byte) (bool, error) {
|
||||
var j1, j2 interface{}
|
||||
if err := json.Unmarshal(a, &j1); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := json.Unmarshal(b, &j2); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return reflect.DeepEqual(j1, j2), nil
|
||||
}
|
||||
|
||||
func BenchmarkDiagnostics(b *testing.B) {
|
||||
// Mock server.
|
||||
server := httptest.NewServer(nil)
|
||||
defer server.Close()
|
||||
|
||||
// Create a new client.
|
||||
d := diagnostics.New(server.URL)
|
||||
d.SetLogger(ioutil.Discard)
|
||||
defer d.Close()
|
||||
|
||||
prev := runtime.GOMAXPROCS(4)
|
||||
defer runtime.GOMAXPROCS(prev)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
d.Set("cc", 1)
|
||||
d.Set("gg", "test")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -112,6 +112,27 @@ Note: This will only work when the replication factor is >= 2
|
|||
- Restart the cluster
|
||||
- Wait for the 1st sync (10 minutes) to validate Index connections
|
||||
|
||||
#### Diagnostics
|
||||
|
||||
Each Pilosa cluster is configured by default to share anonymous usage details with Pilosa Corp. These metrics allow us to understand how Pilosa is used by the community and improve the technology to suit your needs. Diagnostics are sent to Pilosa every hour. Each of the metrics are detailed below as well as opt-out instructions.
|
||||
|
||||
<strong id="version">Version:</strong> Version string of the build.
|
||||
<strong id="host">Host:</strong> Host URI.
|
||||
<strong id="cluster">Cluster:</strong> List of nodes in the Cluster.
|
||||
<strong id="num_nodes">NumNodes:</strong> Number of nodes in the Cluster.
|
||||
<strong id="num_cpu">NumCPU:</strong> Number of Cores per Node
|
||||
<strong id="bsa_enabled">BSIEnabled:</strong> Bit Slice Index Frames in use.
|
||||
<strong id="time_quantum_enabled">TimeQuantumEnabled:</strong> Time Quantum Frames in use.
|
||||
<strong id="inverse_enabled">InverseEnabled:</strong> Inverse Frames in use.
|
||||
<strong id="num_indexes">NumIndexes:</strong> Number of Indexes in the Cluster.
|
||||
<strong id="num_frames">NumFrames:</strong> Number of Frames in the Cluster.
|
||||
<strong id="num_slices">NumSlices:</strong> Number of Slices in the Cluster.
|
||||
<strong id="num_views">NumViews:</strong> Number of Views in the Cluster.
|
||||
<strong id="open_files">OpenFiles:</strong> Open file handle count.
|
||||
<strong id="go_routines">GoRoutines:</strong> Go routine count.
|
||||
|
||||
You can opt-out of the Pilosa diagnostics reporting by setting either the command line configuration option `--metric.diagnostics=false`, use the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option.
|
||||
|
||||
#### Metrics
|
||||
|
||||
Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default.
|
||||
|
|
|
|||
|
|
@ -206,6 +206,19 @@ Any flag that has a value that is a comma separated list on the command line bec
|
|||
poll-interval = "0m15s"
|
||||
```
|
||||
|
||||
##### Metric Diagnostics
|
||||
|
||||
* Description: Enable diagnostic reporting. To disable diagnostics set to false.
|
||||
* Flag: `metric.diagnostics`
|
||||
* Env: `PILOSA_METRIC_DIAGNOSTICS`
|
||||
* Config:
|
||||
|
||||
```toml
|
||||
[metric]
|
||||
diagnostics = true
|
||||
```
|
||||
|
||||
|
||||
##### TLS Certificate
|
||||
|
||||
* Description: Path to the TLS certificate to use for serving HTTPS. Usually has one of`.crt` or `.pem` extensions.
|
||||
|
|
|
|||
55
executor.go
55
executor.go
|
|
@ -15,16 +15,12 @@
|
|||
package pilosa
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/pilosa/pilosa/internal"
|
||||
"github.com/pilosa/pilosa/pql"
|
||||
)
|
||||
|
|
@ -46,8 +42,8 @@ type Executor struct {
|
|||
URI URI
|
||||
Cluster *Cluster
|
||||
|
||||
// Client used for remote HTTP requests.
|
||||
HTTPClient *http.Client
|
||||
// Client used for remote requests.
|
||||
client InternalClient
|
||||
|
||||
// Maximum number of SetBit() or ClearBit() commands per request.
|
||||
MaxWritesPerRequest int
|
||||
|
|
@ -58,13 +54,8 @@ func NewExecutor(clientOptions *ClientOptions) *Executor {
|
|||
if clientOptions == nil {
|
||||
clientOptions = &ClientOptions{}
|
||||
}
|
||||
transport := &http.Transport{}
|
||||
if clientOptions.TLS != nil {
|
||||
transport.TLSClientConfig = clientOptions.TLS
|
||||
}
|
||||
client := &http.Client{Transport: transport}
|
||||
return &Executor{
|
||||
HTTPClient: client,
|
||||
client: NewInternalHTTPClientFromURI(nil, clientOptions),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1376,47 +1367,13 @@ func (e *Executor) exec(ctx context.Context, node *Node, index string, q *pql.Qu
|
|||
Slices: slices,
|
||||
Remote: true,
|
||||
}
|
||||
buf, err := proto.Marshal(pbreq)
|
||||
|
||||
ctx = context.WithValue(ctx, "uri", node.URI)
|
||||
pb, err := e.client.ExecuteQuery(ctx, index, pbreq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create HTTP request.
|
||||
u := nodePathToURL(node, fmt.Sprintf("/index/%s/query", index))
|
||||
req, err := http.NewRequest("POST", (&u).String(), bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Require protobuf encoding.
|
||||
req.Header.Set("Accept", "application/x-protobuf")
|
||||
req.Header.Set("Content-Type", "application/x-protobuf")
|
||||
req.Header.Set("User-Agent", "pilosa/"+Version)
|
||||
|
||||
// Send request to remote node.
|
||||
resp, err := e.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read response into buffer.
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check status code.
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("invalid status Executor.exec: code=%d, err=%s, req: %v", resp.StatusCode, body, req)
|
||||
}
|
||||
|
||||
// Decode response object.
|
||||
var pb internal.QueryResponse
|
||||
if err := proto.Unmarshal(body, &pb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Return an error, if specified on response.
|
||||
if err := decodeError(pb.Err); err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
20
fragment.go
20
fragment.go
|
|
@ -1714,10 +1714,8 @@ func (s *FragmentSyncer) SyncFragment() error {
|
|||
}
|
||||
|
||||
// Retrieve remote blocks.
|
||||
client, err := NewClientFromURI(&node.URI, s.ClientOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client := NewInternalHTTPClientFromURI(&node.URI, s.ClientOptions)
|
||||
|
||||
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice())
|
||||
if err != nil && err != ErrFragmentNotFound {
|
||||
return err
|
||||
|
|
@ -1782,7 +1780,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
|
|||
|
||||
// Read pairs from each remote block.
|
||||
var pairSets []PairSet
|
||||
var clients []*Client
|
||||
var clients []InternalClient
|
||||
for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) {
|
||||
if s.URI == node.URI {
|
||||
continue
|
||||
|
|
@ -1793,10 +1791,8 @@ func (s *FragmentSyncer) syncBlock(id int) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
client, err := NewClientFromURI(&node.URI, s.ClientOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client := NewInternalHTTPClientFromURI(&node.URI, s.ClientOptions)
|
||||
|
||||
clients = append(clients, client)
|
||||
|
||||
// Only sync the standard block.
|
||||
|
|
@ -1848,7 +1844,11 @@ func (s *FragmentSyncer) syncBlock(id int) error {
|
|||
}
|
||||
|
||||
// Execute query.
|
||||
_, err := clients[i].ExecuteQuery(context.Background(), f.Index(), buf.String(), false)
|
||||
queryRequest := &internal.QueryRequest{
|
||||
Query: buf.String(),
|
||||
Remote: true,
|
||||
}
|
||||
_, err := clients[i].ExecuteQuery(context.Background(), f.Index(), queryRequest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
9
frame.go
9
frame.go
|
|
@ -901,7 +901,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
|
|||
}
|
||||
|
||||
// ImportValue bulk imports range-encoded value data.
|
||||
func (f *Frame) ImportValue(fieldName string, columnIDs, values []uint64) error {
|
||||
func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error {
|
||||
// Verify that this frame is range-encoded.
|
||||
if !f.RangeEnabled() {
|
||||
return fmt.Errorf("Frame not RangeEnabled: %s", f.name)
|
||||
|
|
@ -949,7 +949,12 @@ func (f *Frame) ImportValue(fieldName string, columnIDs, values []uint64) error
|
|||
return err
|
||||
}
|
||||
|
||||
if err := frag.ImportValue(data.ColumnIDs, data.Values, field.BitDepth()); err != nil {
|
||||
baseValues := make([]uint64, len(data.Values))
|
||||
for i, value := range data.Values {
|
||||
baseValues[i] = uint64(value - field.Min)
|
||||
}
|
||||
|
||||
if err := frag.ImportValue(data.ColumnIDs, baseValues, field.BitDepth()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1538,11 +1538,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
|
|||
}
|
||||
|
||||
// Create a client for the remote cluster.
|
||||
client, err := NewClientFromURI(host, h.ClientOptions)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
client := NewInternalHTTPClientFromURI(host, h.ClientOptions)
|
||||
|
||||
// Determine the maximum number of slices.
|
||||
maxSlices, err := client.MaxSliceByIndex(r.Context())
|
||||
|
|
|
|||
13
holder.go
13
holder.go
|
|
@ -125,11 +125,14 @@ func (h *Holder) Open() error {
|
|||
h.wg.Add(1)
|
||||
go func() { defer h.wg.Done(); h.monitorCacheFlush() }()
|
||||
|
||||
h.Stats.Open()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes all open fragments.
|
||||
func (h *Holder) Close() error {
|
||||
h.Stats.Close()
|
||||
|
||||
// Notify goroutines of closing and wait for completion.
|
||||
close(h.closing)
|
||||
h.wg.Wait()
|
||||
|
|
@ -568,10 +571,7 @@ func (s *HolderSyncer) syncIndex(index string) error {
|
|||
|
||||
// Sync with every other host.
|
||||
for _, node := range Nodes(s.Cluster.Nodes).FilterURI(s.URI) {
|
||||
client, err := NewClientFromURI(&node.URI, s.ClientOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client := NewInternalHTTPClientFromURI(&node.URI, s.ClientOptions)
|
||||
|
||||
// Retrieve attributes from differing blocks.
|
||||
// Skip update and recomputation if no attributes have changed.
|
||||
|
|
@ -613,10 +613,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
|
|||
|
||||
// Sync with every other host.
|
||||
for _, node := range Nodes(s.Cluster.Nodes).FilterURI(s.URI) {
|
||||
client, err := NewClientFromURI(&node.URI, s.ClientOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client := NewInternalHTTPClientFromURI(&node.URI, s.ClientOptions)
|
||||
|
||||
// Retrieve attributes from differing blocks.
|
||||
// Skip update and recomputation if no attributes have changed.
|
||||
|
|
|
|||
2
index.go
2
index.go
|
|
@ -674,7 +674,7 @@ type importData struct {
|
|||
|
||||
type importValueData struct {
|
||||
ColumnIDs []uint64
|
||||
Values []uint64
|
||||
Values []int64
|
||||
}
|
||||
|
||||
// CreateInputDefinition creates a new input definition.
|
||||
|
|
|
|||
|
|
@ -440,7 +440,7 @@ type ImportValueRequest struct {
|
|||
Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
|
||||
Field string `protobuf:"bytes,4,opt,name=Field,proto3" json:"Field,omitempty"`
|
||||
ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"`
|
||||
Values []uint64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"`
|
||||
Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"`
|
||||
}
|
||||
|
||||
func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} }
|
||||
|
|
@ -483,7 +483,7 @@ func (m *ImportValueRequest) GetColumnIDs() []uint64 {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (m *ImportValueRequest) GetValues() []uint64 {
|
||||
func (m *ImportValueRequest) GetValues() []int64 {
|
||||
if m != nil {
|
||||
return m.Values
|
||||
}
|
||||
|
|
@ -1100,7 +1100,8 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) {
|
|||
if len(m.Values) > 0 {
|
||||
dAtA16 := make([]byte, len(m.Values)*10)
|
||||
var j15 int
|
||||
for _, num := range m.Values {
|
||||
for _, num1 := range m.Values {
|
||||
num := uint64(num1)
|
||||
for num >= 1<<7 {
|
||||
dAtA16[j15] = uint8(uint64(num)&0x7f | 0x80)
|
||||
num >>= 7
|
||||
|
|
@ -3244,7 +3245,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
|
|||
}
|
||||
case 6:
|
||||
if wireType == 0 {
|
||||
var v uint64
|
||||
var v int64
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowPublic
|
||||
|
|
@ -3254,7 +3255,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
|
|||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
v |= (uint64(b) & 0x7F) << shift
|
||||
v |= (int64(b) & 0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
|
|
@ -3284,7 +3285,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
|
|||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
for iNdEx < postIndex {
|
||||
var v uint64
|
||||
var v int64
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowPublic
|
||||
|
|
@ -3294,7 +3295,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
|
|||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
v |= (uint64(b) & 0x7F) << shift
|
||||
v |= (int64(b) & 0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
|
|
@ -3433,7 +3434,7 @@ var (
|
|||
func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
|
||||
|
||||
var fileDescriptorPublic = []byte{
|
||||
// 653 bytes of a gzipped FileDescriptorProto
|
||||
// 651 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xcb, 0x6e, 0xd3, 0x40,
|
||||
0x14, 0x65, 0x62, 0xe7, 0x75, 0x93, 0x56, 0xd5, 0x08, 0x8a, 0x85, 0x50, 0x14, 0x59, 0x2c, 0xbc,
|
||||
0x4a, 0xa5, 0xf0, 0x01, 0x08, 0xb7, 0xa9, 0x64, 0x21, 0x2a, 0x98, 0x14, 0xf6, 0x6e, 0x3b, 0x2a,
|
||||
|
|
@ -3472,7 +3473,7 @@ var fileDescriptorPublic = []byte{
|
|||
0x67, 0x5c, 0x79, 0xae, 0x51, 0x8e, 0x41, 0xa8, 0x73, 0x7b, 0xc5, 0x95, 0xd7, 0x27, 0x53, 0x4b,
|
||||
0xa0, 0xce, 0xb7, 0x67, 0x8c, 0xda, 0x70, 0x02, 0x47, 0x74, 0x18, 0xff, 0x23, 0x03, 0x6e, 0x2a,
|
||||
0x25, 0xdd, 0xff, 0xbf, 0x72, 0xd1, 0x37, 0x91, 0xa9, 0x19, 0x25, 0xfa, 0x22, 0xf8, 0x4b, 0xb1,
|
||||
0x87, 0x30, 0xa0, 0x2a, 0x4c, 0xa1, 0xae, 0x68, 0x50, 0x78, 0xf0, 0x6d, 0x33, 0x63, 0xdf, 0x37,
|
||||
0x33, 0xf6, 0x63, 0x33, 0x63, 0x1f, 0x7e, 0xce, 0xee, 0x5c, 0x0c, 0xe8, 0x07, 0xfd, 0xf8, 0x57,
|
||||
0x00, 0x00, 0x00, 0xff, 0xff, 0xf6, 0x73, 0x96, 0xb9, 0xb0, 0x05, 0x00, 0x00,
|
||||
0x87, 0x30, 0xa0, 0x2a, 0x6c, 0xa1, 0x0d, 0x0a, 0x0f, 0xbe, 0x6d, 0x66, 0xec, 0xfb, 0x66, 0xc6,
|
||||
0x7e, 0x6c, 0x66, 0xec, 0xc3, 0xcf, 0xd9, 0x9d, 0x8b, 0x01, 0xfd, 0xa0, 0x1f, 0xff, 0x0a, 0x00,
|
||||
0x00, 0xff, 0xff, 0x4d, 0x1e, 0xdf, 0xba, 0xb0, 0x05, 0x00, 0x00,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,5 +79,5 @@ message ImportValueRequest {
|
|||
uint64 Slice = 3;
|
||||
string Field = 4;
|
||||
repeated uint64 ColumnIDs = 5;
|
||||
repeated uint64 Values = 6;
|
||||
repeated int64 Values = 6;
|
||||
}
|
||||
|
|
|
|||
105
server.go
105
server.go
|
|
@ -32,12 +32,15 @@ import (
|
|||
|
||||
"github.com/CAFxX/gcnotifier"
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/pilosa/pilosa/diagnostics"
|
||||
"github.com/pilosa/pilosa/internal"
|
||||
)
|
||||
|
||||
// Default server settings.
|
||||
const (
|
||||
DefaultAntiEntropyInterval = 10 * time.Minute
|
||||
DefaultPollingInterval = 60 * time.Second
|
||||
DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics"
|
||||
)
|
||||
|
||||
// Server represents a holder wrapped by a running HTTP server.
|
||||
|
|
@ -56,13 +59,15 @@ type Server struct {
|
|||
|
||||
// Cluster configuration.
|
||||
// Host is replaced with actual host after opening if port is ":0".
|
||||
Network string
|
||||
URI URI
|
||||
Cluster *Cluster
|
||||
Network string
|
||||
URI URI
|
||||
Cluster *Cluster
|
||||
diagnostics *diagnostics.Diagnostics
|
||||
|
||||
// Background monitoring intervals.
|
||||
AntiEntropyInterval time.Duration
|
||||
MetricInterval time.Duration
|
||||
DiagnosticInterval time.Duration
|
||||
|
||||
// TLS configuration
|
||||
TLS *tls.Config
|
||||
|
|
@ -72,7 +77,7 @@ type Server struct {
|
|||
|
||||
LogOutput io.Writer
|
||||
|
||||
defaultClient *http.Client
|
||||
defaultClient InternalClient
|
||||
}
|
||||
|
||||
// NewServer returns a new instance of Server.
|
||||
|
|
@ -84,11 +89,13 @@ func NewServer() *Server {
|
|||
Handler: NewHandler(),
|
||||
Broadcaster: NopBroadcaster,
|
||||
BroadcastReceiver: NopBroadcastReceiver,
|
||||
diagnostics: diagnostics.New(DefaultDiagnosticServer),
|
||||
|
||||
Network: "tcp",
|
||||
|
||||
AntiEntropyInterval: DefaultAntiEntropyInterval,
|
||||
MetricInterval: 0,
|
||||
DiagnosticInterval: 0,
|
||||
|
||||
LogOutput: os.Stderr,
|
||||
}
|
||||
|
|
@ -183,12 +190,11 @@ func (s *Server) Open() error {
|
|||
}
|
||||
}()
|
||||
|
||||
/*
|
||||
// Start background monitoring.
|
||||
s.wg.Add(2)
|
||||
go func() { defer s.wg.Done(); s.monitorAntiEntropy() }()
|
||||
go func() { defer s.wg.Done(); s.monitorRuntime() }()
|
||||
*/
|
||||
// Start background monitoring.
|
||||
s.wg.Add(3)
|
||||
go func() { defer s.wg.Done(); s.monitorAntiEntropy() }()
|
||||
go func() { defer s.wg.Done(); s.monitorRuntime() }()
|
||||
go func() { defer s.wg.Done(); s.monitorDiagnostics() }()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -220,10 +226,10 @@ func (s *Server) Addr() net.Addr {
|
|||
return s.ln.Addr()
|
||||
}
|
||||
|
||||
// Logger returns a logger that writes to LogOutput
|
||||
func (s *Server) Logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) }
|
||||
|
||||
func (s *Server) monitorAntiEntropy() {
|
||||
t := time.Now()
|
||||
ticker := time.NewTicker(s.AntiEntropyInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
|
|
@ -237,7 +243,7 @@ func (s *Server) monitorAntiEntropy() {
|
|||
case <-ticker.C:
|
||||
s.Holder.Stats.Count("AntiEntropy", 1, 1.0)
|
||||
}
|
||||
|
||||
t := time.Now()
|
||||
s.Logger().Printf("holder sync beginning")
|
||||
|
||||
// Initialize syncer with local holder and remote client.
|
||||
|
|
@ -256,9 +262,9 @@ func (s *Server) monitorAntiEntropy() {
|
|||
|
||||
// Record successful sync in log.
|
||||
s.Logger().Printf("holder sync complete")
|
||||
dif := time.Since(t)
|
||||
s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0)
|
||||
}
|
||||
dif := time.Since(t)
|
||||
s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0)
|
||||
}
|
||||
|
||||
// ReceiveMessage represents an implementation of BroadcastHandler.
|
||||
|
|
@ -431,9 +437,66 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// monitorDiagnostics periodically polls the the Pilosa Indexes for cluster info.
|
||||
func (s *Server) monitorDiagnostics() {
|
||||
if s.DiagnosticInterval <= 0 {
|
||||
s.Logger().Printf("diagnostics disabled")
|
||||
return
|
||||
}
|
||||
|
||||
s.diagnostics.SetLogger(s.LogOutput)
|
||||
s.diagnostics.SetVersion(Version)
|
||||
s.diagnostics.SetInterval(s.DiagnosticInterval)
|
||||
s.diagnostics.Open()
|
||||
s.diagnostics.Set("Host", s.URI.host)
|
||||
s.diagnostics.Set("Cluster", strings.Join(NodeSet(s.Cluster.NodeSet()).ToStrings(), ","))
|
||||
s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes))
|
||||
s.diagnostics.Set("NumCPU", runtime.NumCPU())
|
||||
// TODO: unique cluster ID
|
||||
|
||||
// Flush the diagnostics metrics at startup, then on each tick interval
|
||||
flush := func() {
|
||||
numFrames := 0
|
||||
numSlices := uint64(0)
|
||||
for _, index := range s.Holder.Indexes() {
|
||||
numSlices += index.MaxSlice() + 1
|
||||
for _, f := range index.Frames() {
|
||||
numFrames++
|
||||
if f.rangeEnabled {
|
||||
s.diagnostics.Set("BSIEnabled", true)
|
||||
}
|
||||
if f.timeQuantum != "" {
|
||||
s.diagnostics.Set("TimeQuantumEnabled", true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
s.diagnostics.Set("NumIndexes", len(s.Holder.Indexes()))
|
||||
s.diagnostics.Set("NumFrames", numFrames)
|
||||
s.diagnostics.Set("NumSlices", numSlices)
|
||||
s.diagnostics.Set("OpenFiles", CountOpenFiles())
|
||||
s.diagnostics.Set("GoRoutines", runtime.NumGoroutine())
|
||||
s.diagnostics.CheckVersion()
|
||||
s.diagnostics.Flush()
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(s.DiagnosticInterval)
|
||||
defer ticker.Stop()
|
||||
flush()
|
||||
for {
|
||||
// Wait for tick or a close.
|
||||
select {
|
||||
case <-s.closing:
|
||||
return
|
||||
case <-ticker.C:
|
||||
flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// monitorRuntime periodically polls the Go runtime metrics.
|
||||
func (s *Server) monitorRuntime() {
|
||||
// Disable metrics when poll interval is zero
|
||||
// Disable metrics when poll interval is zero.
|
||||
if s.MetricInterval <= 0 {
|
||||
return
|
||||
}
|
||||
|
|
@ -453,18 +516,18 @@ func (s *Server) monitorRuntime() {
|
|||
case <-s.closing:
|
||||
return
|
||||
case <-gcn.AfterGC():
|
||||
// GC just ran
|
||||
// GC just ran.
|
||||
s.Holder.Stats.Count("garbage_collection", 1, 1.0)
|
||||
case <-ticker.C:
|
||||
}
|
||||
|
||||
// Record the number of go routines
|
||||
// Record the number of go routines.
|
||||
s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0)
|
||||
|
||||
// Open File handles
|
||||
// Open File handles.
|
||||
s.Holder.Stats.Gauge("OpenFiles", float64(CountOpenFiles()), 1.0)
|
||||
|
||||
// Runtime memory metrics
|
||||
// Runtime memory metrics.
|
||||
runtime.ReadMemStats(&m)
|
||||
s.Holder.Stats.Gauge("HeapAlloc", float64(m.HeapAlloc), 1.0)
|
||||
s.Holder.Stats.Gauge("HeapInuse", float64(m.HeapInuse), 1.0)
|
||||
|
|
@ -479,10 +542,10 @@ func (s *Server) createDefaultClient() {
|
|||
if s.TLS != nil {
|
||||
transport.TLSClientConfig = s.TLS
|
||||
}
|
||||
s.defaultClient = &http.Client{Transport: transport}
|
||||
s.defaultClient = NewInternalHTTPClientFromURI(nil, &ClientOptions{TLS: s.TLS})
|
||||
}
|
||||
|
||||
// CountOpenFiles on opperating systems that support lsof
|
||||
// CountOpenFiles on operating systems that support lsof.
|
||||
func CountOpenFiles() int {
|
||||
count := 0
|
||||
|
||||
|
|
|
|||
|
|
@ -44,6 +44,9 @@ func init() {
|
|||
const (
|
||||
// DefaultDataDir is the default data directory.
|
||||
DefaultDataDir = "~/.pilosa"
|
||||
|
||||
// DefaultDiagnosticsInterval is the default sync frequency diagnostic metrics.
|
||||
DefaultDiagnosticsInterval = 1 * time.Hour
|
||||
)
|
||||
|
||||
// Command represents the state of the pilosa server command.
|
||||
|
|
@ -139,6 +142,9 @@ func (m *Command) SetupServer() error {
|
|||
m.Server.Logger().Printf("Using data from: %s\n", m.Config.DataDir)
|
||||
m.Server.Holder.Path = m.Config.DataDir
|
||||
m.Server.MetricInterval = time.Duration(m.Config.Metric.PollInterval)
|
||||
if m.Config.Metric.Diagnostics {
|
||||
m.Server.DiagnosticInterval = time.Duration(DefaultDiagnosticsInterval)
|
||||
}
|
||||
m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ func TestMain_Set_Quick(t *testing.T) {
|
|||
defer m.Close()
|
||||
|
||||
// Create client.
|
||||
client, err := pilosa.NewClient(m.Server.URI.HostPort(), nil)
|
||||
client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -326,7 +326,7 @@ func TestMain_FrameRestore(t *testing.T) {
|
|||
defer m2.Close()
|
||||
|
||||
// Import from first cluster.
|
||||
client, err := pilosa.NewClient(m2.Server.URI.HostPort(), nil)
|
||||
client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
|
|
@ -705,8 +705,8 @@ func (m *Main) Reopen() error {
|
|||
func (m *Main) URL() string { return "http://" + m.Server.Addr().String() }
|
||||
|
||||
// Client returns a client to connect to the program.
|
||||
func (m *Main) Client() *pilosa.Client {
|
||||
client, err := pilosa.NewClient(m.Server.URI.HostPort(), nil)
|
||||
func (m *Main) Client() *pilosa.InternalHTTPClient {
|
||||
client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
|
|
|||
36
stats.go
36
stats.go
|
|
@ -58,6 +58,12 @@ type StatsClient interface {
|
|||
|
||||
// SetLogger Set the logger output type
|
||||
SetLogger(logger io.Writer)
|
||||
|
||||
// Starts the service
|
||||
Open()
|
||||
|
||||
// Closes the client
|
||||
Close() error
|
||||
}
|
||||
|
||||
// NopStatsClient represents a client that doesn't do anything.
|
||||
|
|
@ -74,6 +80,8 @@ func (c *nopStatsClient) Histogram(name string, value float64, rate float64)
|
|||
func (c *nopStatsClient) Set(name string, value string, rate float64) {}
|
||||
func (c *nopStatsClient) Timing(name string, value time.Duration, rate float64) {}
|
||||
func (c *nopStatsClient) SetLogger(logger io.Writer) {}
|
||||
func (c *nopStatsClient) Open() {}
|
||||
func (c *nopStatsClient) Close() error { return nil }
|
||||
|
||||
// ExpvarStatsClient writes stats out to expvars.
|
||||
type ExpvarStatsClient struct {
|
||||
|
|
@ -145,10 +153,16 @@ func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float6
|
|||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetLogger has no logger
|
||||
// SetLogger has no logger.
|
||||
func (c *ExpvarStatsClient) SetLogger(logger io.Writer) {
|
||||
}
|
||||
|
||||
// Open no-op.
|
||||
func (c *ExpvarStatsClient) Open() {}
|
||||
|
||||
// Close no-op.
|
||||
func (c *ExpvarStatsClient) Close() error { return nil }
|
||||
|
||||
// MultiStatsClient joins multiple stats clients together.
|
||||
type MultiStatsClient []StatsClient
|
||||
|
||||
|
|
@ -211,13 +225,31 @@ func (a MultiStatsClient) Timing(name string, value time.Duration, rate float64)
|
|||
}
|
||||
}
|
||||
|
||||
// SetLogger Sets the StatsD logger output type
|
||||
// SetLogger Sets the StatsD logger output type.
|
||||
func (a MultiStatsClient) SetLogger(logger io.Writer) {
|
||||
for _, c := range a {
|
||||
c.SetLogger(logger)
|
||||
}
|
||||
}
|
||||
|
||||
// Open starts the stat service.
|
||||
func (a MultiStatsClient) Open() {
|
||||
for _, c := range a {
|
||||
c.Open()
|
||||
}
|
||||
}
|
||||
|
||||
// Close shuts down the stats clients.
|
||||
func (a MultiStatsClient) Close() error {
|
||||
for _, c := range a {
|
||||
err := c.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UnionStringSlice returns a sorted set of tags which combine a & b.
|
||||
func UnionStringSlice(a, b []string) []string {
|
||||
// Sort both sets first.
|
||||
|
|
|
|||
|
|
@ -344,3 +344,5 @@ func (c *MockStats) Histogram(name string, value float64, rate float64) {}
|
|||
func (c *MockStats) Set(name string, value string, rate float64) {}
|
||||
func (c *MockStats) Timing(name string, value time.Duration, rate float64) {}
|
||||
func (c *MockStats) SetLogger(logger io.Writer) {}
|
||||
func (c *MockStats) Open() {}
|
||||
func (c *MockStats) Close() error { return nil }
|
||||
|
|
|
|||
|
|
@ -58,6 +58,9 @@ func NewStatsClient(host string) (*StatsClient, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
// Open no-op
|
||||
func (c *StatsClient) Open() {}
|
||||
|
||||
// Close closes the connection to the agent.
|
||||
func (c *StatsClient) Close() error {
|
||||
return c.client.Close()
|
||||
|
|
|
|||
|
|
@ -6,14 +6,14 @@ import (
|
|||
|
||||
// Client represents a test wrapper for pilosa.Client.
|
||||
type Client struct {
|
||||
*pilosa.Client
|
||||
*pilosa.InternalHTTPClient
|
||||
}
|
||||
|
||||
// MustNewClient returns a new instance of Client. Panic on error.
|
||||
func MustNewClient(host string) *Client {
|
||||
c, err := pilosa.NewClient(host, nil)
|
||||
c, err := pilosa.NewInternalHTTPClient(host, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &Client{Client: c}
|
||||
return &Client{InternalHTTPClient: c}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,8 @@ type Executor struct {
|
|||
// NewExecutor returns a new instance of Executor.
|
||||
// The executor always matches the uri of the first cluster node.
|
||||
func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor {
|
||||
e := &Executor{Executor: pilosa.NewExecutor(nil)}
|
||||
executor := pilosa.NewExecutor(nil)
|
||||
e := &Executor{Executor: executor}
|
||||
e.Holder = holder
|
||||
e.Cluster = cluster
|
||||
e.URI = cluster.Nodes[0].URI
|
||||
|
|
|
|||
5
uri.go
5
uri.go
|
|
@ -146,6 +146,11 @@ func (u URI) Equals(other *URI) bool {
|
|||
return u == *other
|
||||
}
|
||||
|
||||
// Path returns URI with path
|
||||
func (u *URI) Path(path string) string {
|
||||
return fmt.Sprintf("%s%s", u.Normalize(), path)
|
||||
}
|
||||
|
||||
// The following methods are required to implement pflag Value interface.
|
||||
|
||||
// Set sets the time quantum value.
|
||||
|
|
|
|||
11
uri_test.go
11
uri_test.go
|
|
@ -83,6 +83,17 @@ func TestNormalizedAddress(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestURIPath(t *testing.T) {
|
||||
uri, err := NewURIFromAddress("http+protobuf://big-data.pilosa.com:6888")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := "http://big-data.pilosa.com:6888/index/foo"
|
||||
if uri.Path("/index/foo") != target {
|
||||
t.Fatalf("%s != %s", uri.Path("/index/foo"), target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEquals(t *testing.T) {
|
||||
uri1 := DefaultURI()
|
||||
if uri1.Equals(nil) {
|
||||
|
|
|
|||
|
|
@ -344,12 +344,11 @@ function render_status(status) {
|
|||
tbody = document.createElement("tbody")
|
||||
table.appendChild(tbody)
|
||||
var caption = document.createElement("caption")
|
||||
caption.innerHTML = indexes[n]["Name"] + " (Column Label: " + indexes[n]["Meta"]["ColumnLabel"] + ")"
|
||||
caption.innerHTML = indexes[n]["Name"]
|
||||
table.appendChild(caption)
|
||||
|
||||
var header = document.createElement('tr')
|
||||
markup = `<th>Name</th>
|
||||
<th>Row Label</th>
|
||||
<th>Cache Type</th>
|
||||
<th>Cache Size</th>`
|
||||
header.innerHTML = markup
|
||||
|
|
@ -360,7 +359,6 @@ function render_status(status) {
|
|||
for(var m=0; m<frames.length; m++) {
|
||||
var row = document.createElement("tr")
|
||||
row.innerHTML = `<td>${frames[m]["Name"]}</td>
|
||||
<td>${frames[m]["Meta"]["RowLabel"]}</td>
|
||||
<td>${frames[m]["Meta"]["CacheType"]}</td>
|
||||
<td>${frames[m]["Meta"]["CacheSize"]}</td>`
|
||||
tbody.appendChild(row)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue