From f2c104dfef75d30854c235fa9dc5c78905024651 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 12 Jun 2018 13:22:40 -0500 Subject: [PATCH] Migrate HTTP handler and client into http subpackage. --- cache.go | 2 +- client.go | 1110 ++------------- cluster.go | 9 +- ctl/common.go | 6 +- ctl/import.go | 9 +- executor.go | 31 +- fragment.go | 15 +- handler.go | 1187 +--------------- holder.go | 8 +- holder_test.go | 27 +- http/client.go | 1034 ++++++++++++++ client_test.go => http/client_test.go | 27 +- http/handler.go | 1231 +++++++++++++++++ .../handler_internal_test.go | 10 +- handler_test.go => http/handler_test.go | 101 +- pilosa.go | 10 +- row.go | 8 +- server.go | 45 +- server/server.go | 39 +- server/server_test.go | 3 +- test/client.go | 10 +- test/executor.go | 11 +- test/handler.go | 15 +- test/pilosa.go | 16 +- 24 files changed, 2577 insertions(+), 2387 deletions(-) create mode 100644 http/client.go rename client_test.go => http/client_test.go (93%) create mode 100644 http/handler.go rename handler_internal_test.go => http/handler_internal_test.go (94%) rename handler_test.go => http/handler_test.go (93%) diff --git a/cache.go b/cache.go index 06046220a..ec9ade91b 100644 --- a/cache.go +++ b/cache.go @@ -409,7 +409,7 @@ func (p Pairs) String() string { return buf.String() } -func encodePairs(a Pairs) []*internal.Pair { +func EncodePairs(a Pairs) []*internal.Pair { other := make([]*internal.Pair, len(a)) for i := range a { other[i] = encodePair(a[i]) diff --git a/client.go b/client.go index 81f750a39..56cba08e1 100644 --- a/client.go +++ b/client.go @@ -1,878 +1,13 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package pilosa import ( - "bytes" "context" - "encoding/json" - "fmt" "io" - "io/ioutil" - "math/rand" - "net/http" - "net/url" - "sort" - "strconv" - - "crypto/tls" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" - "github.com/pkg/errors" ) -// ClientOptions represents the configuration for a InternalHTTPClient -type ClientOptions struct { - TLS *tls.Config -} - -// InternalHTTPClient represents a client to the Pilosa cluster. -type InternalHTTPClient struct { - defaultURI *URI - - // The client to use for HTTP communication. - HTTPClient *http.Client -} - -// NewInternalHTTPClient returns a new instance of InternalHTTPClient to connect to host. -func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTTPClient, error) { - if host == "" { - return nil, ErrHostRequired - } - - uri, err := NewURIFromAddress(host) - if err != nil { - return nil, errors.Wrap(err, "getting URI") - } - - client := NewInternalHTTPClientFromURI(uri, remoteClient) - return client, nil -} - -func NewInternalHTTPClientFromURI(defaultURI *URI, remoteClient *http.Client) *InternalHTTPClient { - return &InternalHTTPClient{ - defaultURI: defaultURI, - HTTPClient: remoteClient, - } -} - -// Host returns the host the client was initialized with. -func (c *InternalHTTPClient) Host() *URI { return c.defaultURI } - -// MaxSliceByIndex returns the number of slices on a server by index. -func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { - return c.maxSliceByIndex(ctx) -} - -// maxSliceByIndex returns the number of slices on a server by index. -func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) { - // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/slices/max") - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - var rsp getSlicesMaxResponse - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, fmt.Errorf("json decode: %s", err) - } - - return rsp.Standard, nil -} - -// Schema returns all index and field schema information. -func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) { - // Execute request against the host. - u := c.defaultURI.Path("/schema") - - // Build request. - req, err := http.NewRequest("GET", u, nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - var rsp getSchemaResponse - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, fmt.Errorf("json decode: %s", err) - } - return rsp.Indexes, nil -} - -// CreateIndex creates a new index on the server. -func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { - // Encode query request. - buf, err := json.Marshal(&postIndexRequest{ - Options: opt, - }) - if err != nil { - return errors.Wrap(err, "encoding request") - } - - // Create URL & HTTP request. - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index)) - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } - - // Handle response based on status code. - switch resp.StatusCode { - case http.StatusOK: - return nil // ok - case http.StatusConflict: - return ErrIndexExists - default: - return errors.New(string(body)) - } -} - -// FragmentNodes returns a list of nodes that own a slice. -func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) { - // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/fragment/nodes") - u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode() - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - var a []*Node - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&a); err != nil { - return nil, fmt.Errorf("json decode: %s", err) - } - - return a, nil -} - -// Query executes query against the index. -func (c *InternalHTTPClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { - return c.QueryNode(ctx, c.defaultURI, index, queryRequest) -} - -// QueryNode executes query against the index, sending the request to the node specified. -func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { - if index == "" { - return nil, ErrIndexRequired - } else if queryRequest.Query == "" { - return nil, ErrQueryRequired - } - - // Encode request object. - buf, err := proto.Marshal(queryRequest) - if err != nil { - return nil, errors.Wrap(err, "marshaling") - } - - // Create HTTP request. - u := uri.Path(fmt.Sprintf("/index/%s/query", index)) - req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrap(err, "reading") - } else if resp.StatusCode != http.StatusOK { - return nil, errors.New(string(body)) - } - - qresp := &internal.QueryResponse{} - if err := proto.Unmarshal(body, qresp); err != nil { - return nil, fmt.Errorf("unmarshal response: %s", err) - } else if s := qresp.Err; s != "" { - return nil, errors.New(s) - } - - return qresp, nil -} - -// Import bulk imports bits for a single slice to a host. -func (c *InternalHTTPClient) Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error { - if index == "" { - return ErrIndexRequired - } else if field == "" { - return ErrFieldRequired - } - - buf, err := marshalImportPayload(index, field, slice, bits) - if err != nil { - return fmt.Errorf("Error Creating Payload: %s", err) - } - - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Import to each node. - for _, node := range nodes { - if err := c.importNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) - } - } - - return nil -} - -// ImportK bulk imports bits specified by string keys to a host. -func (c *InternalHTTPClient) ImportK(ctx context.Context, index, field string, columns []Bit) error { - if index == "" { - return ErrIndexRequired - } else if field == "" { - return ErrFieldRequired - } - - buf, err := marshalImportPayloadK(index, field, columns) - if err != nil { - return fmt.Errorf("Error Creating Payload: %s", err) - } - - node := &Node{ - URI: *c.defaultURI, - } - - // Import to node. - if err := c.importNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) - } - - return nil -} - -func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { - err := c.CreateIndex(ctx, name, options) - if err == nil || err == ErrIndexExists { - return nil - } - return err -} - -func (c *InternalHTTPClient) EnsureField(ctx context.Context, indexName string, fieldName string, options FieldOptions) error { - err := c.CreateField(ctx, indexName, fieldName, options) - if err == nil || err == ErrFieldExists { - return nil - } - return err -} - -// marshalImportPayload marshalls the import parameters into a protobuf byte slice. -func marshalImportPayload(index, field string, slice uint64, bits []Bit) ([]byte, error) { - // Separate row and column IDs to reduce allocations. - rowIDs := Bits(bits).RowIDs() - columnIDs := Bits(bits).ColumnIDs() - timestamps := Bits(bits).Timestamps() - - // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ - Index: index, - Field: field, - Slice: slice, - RowIDs: rowIDs, - ColumnIDs: columnIDs, - Timestamps: timestamps, - }) - if err != nil { - return nil, fmt.Errorf("marshal import request: %s", err) - } - return buf, nil -} - -// marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. -func marshalImportPayloadK(index, field string, bits []Bit) ([]byte, error) { - // Separate row and column IDs to reduce allocations. - rowKeys := Bits(bits).RowKeys() - columnKeys := Bits(bits).ColumnKeys() - timestamps := Bits(bits).Timestamps() - - // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ - Index: index, - Field: field, - RowKeys: rowKeys, - ColumnKeys: columnKeys, - Timestamps: timestamps, - }) - if err != nil { - return nil, fmt.Errorf("marshal import request: %s", err) - } - return buf, nil -} - -// importNode sends a pre-marshaled import request to a node. -func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []byte) error { - // Create URL & HTTP request. - u := nodePathToURL(node, "/import") - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } else if resp.StatusCode != http.StatusOK { - return errors.New(string(body)) - } - - var isresp internal.ImportResponse - if err := proto.Unmarshal(body, &isresp); err != nil { - return fmt.Errorf("unmarshal import response: %s", err) - } else if s := isresp.Err; s != "" { - return errors.New(s) - } - - return nil -} - -// ImportValue bulk imports field values for a single slice to a host. -func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error { - if index == "" { - return ErrIndexRequired - } else if field == "" { - return ErrFieldRequired - } - - buf, err := marshalImportValuePayload(index, field, slice, vals) - if err != nil { - return fmt.Errorf("Error Creating Payload: %s", err) - } - - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Import to each node. - for _, node := range nodes { - if err := c.importValueNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) - } - } - - return nil -} - -// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. -func marshalImportValuePayload(index, field string, slice uint64, vals []FieldValue) ([]byte, error) { - // Separate row and column IDs to reduce allocations. - columnIDs := FieldValues(vals).ColumnIDs() - values := FieldValues(vals).Values() - - // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportValueRequest{ - Index: index, - Field: field, - Slice: slice, - ColumnIDs: columnIDs, - Values: values, - }) - if err != nil { - return nil, fmt.Errorf("marshal import request: %s", err) - } - return buf, nil -} - -// importValueNode sends a pre-marshaled import request to a node. -func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, buf []byte) error { - // Create URL & HTTP request. - u := nodePathToURL(node, "/import-value") - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } else if resp.StatusCode != http.StatusOK { - return errors.New(string(body)) - } - - var isresp internal.ImportResponse - if err := proto.Unmarshal(body, &isresp); err != nil { - return fmt.Errorf("unmarshal import response: %s", err) - } else if s := isresp.Err; s != "" { - return errors.New(s) - } - - return nil -} - -// ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { - if index == "" { - return ErrIndexRequired - } else if field == "" { - return ErrFieldRequired - } - - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Attempt nodes in random order. - var e error - for _, i := range rand.Perm(len(nodes)) { - node := nodes[i] - - if err := c.exportNodeCSV(ctx, node, index, field, slice, w); err != nil { - e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) - continue - } else { - return nil - } - } - - return e -} - -// exportNode copies a CSV export from a node to w. -func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, field string, slice uint64, w io.Writer) error { - // Create URL. - u := nodePathToURL(node, "/export") - u.RawQuery = url.Values{ - "index": {index}, - "field": {field}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Generate HTTP request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Accept", "text/csv") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Validate status code. - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("invalid status: %d", resp.StatusCode) - } - - // Copy body to writer. - if _, err := io.Copy(w, resp.Body); err != nil { - return errors.Wrap(err, "copying") - } - - return nil -} - -func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) { - node := &Node{ - URI: uri, - } - return c.backupSliceNode(ctx, index, field, slice, node) -} - -func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, field string, slice uint64, node *Node) (io.ReadCloser, error) { - u := nodePathToURL(node, "/fragment/data") - u.RawQuery = url.Values{ - "index": {index}, - "field": {field}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - - // Return error if status is not OK. - if resp.StatusCode == http.StatusNotFound { - resp.Body.Close() - return nil, ErrFragmentNotFound - } else if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode) - } - - return resp.Body, nil -} - -// CreateField creates a new field on the server. -func (c *InternalHTTPClient) CreateField(ctx context.Context, index, field string, opt FieldOptions) error { - if index == "" { - return ErrIndexRequired - } - - // Encode query request. - buf, err := json.Marshal(&postFieldRequest{ - Options: opt, - }) - if err != nil { - return errors.Wrap(err, "marshaling") - } - - // Create URL & HTTP request. - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/field/%s", index, field)) - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } - - // Handle response based on status code. - switch resp.StatusCode { - case http.StatusOK: - return nil // ok - case http.StatusConflict: - return ErrFieldExists - default: - return errors.New(string(body)) - } -} - -// FragmentBlocks returns a list of block checksums for a fragment on a host. -// Only returns blocks which contain data. -func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, field string, slice uint64) ([]FragmentBlock, error) { - u := uriPathToURL(c.defaultURI, "/fragment/blocks") - u.RawQuery = url.Values{ - "index": {index}, - "field": {field}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - case http.StatusNotFound: - return nil, ErrFragmentNotFound - default: - return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp getFragmentBlocksResponse - if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, errors.Wrap(err, "decoding") - } - return rsp.Blocks, nil -} - -// BlockData returns row/column id pairs for a block. -func (c *InternalHTTPClient) BlockData(ctx context.Context, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { - buf, err := proto.Marshal(&internal.BlockDataRequest{ - Index: index, - Field: field, - Slice: slice, - Block: uint64(block), - }) - if err != nil { - return nil, nil, errors.Wrap(err, "marshaling") - } - - u := uriPathToURL(c.defaultURI, "/fragment/block/data") - req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) - if err != nil { - return nil, nil, errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/protobuf") - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Accept", "application/protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // fallthrough - case http.StatusNotFound: - return nil, nil, nil - default: - return nil, nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp internal.BlockDataResponse - if body, err := ioutil.ReadAll(resp.Body); err != nil { - return nil, nil, errors.Wrap(err, "reading") - } else if err := proto.Unmarshal(body, &rsp); err != nil { - return nil, nil, errors.Wrap(err, "unmarshalling") - } - return rsp.RowIDs, rsp.ColumnIDs, nil -} - -// ColumnAttrDiff returns data from differing blocks on a remote host. -func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/attr/diff", index)) - - // Encode request. - buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks}) - if err != nil { - return nil, errors.Wrap(err, "marshaling") - } - - // Build request. - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - default: - return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp postIndexAttrDiffResponse - if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, errors.Wrap(err, "decoding") - } - return rsp.Attrs, nil -} - -// RowAttrDiff returns data from differing blocks on a remote host. -func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/field/%s/attr/diff", index, field)) - - // Encode request. - buf, err := json.Marshal(postFieldAttrDiffRequest{Blocks: blks}) - if err != nil { - return nil, errors.Wrap(err, "marshaling") - } - - // Build request. - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - case http.StatusNotFound: - return nil, ErrFieldNotFound - default: - return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp postFieldAttrDiffResponse - if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, errors.Wrap(err, "decoding") - } - return rsp.Attrs, nil -} - -// SendMessage posts a message synchronously. -func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { - msg, err := MarshalMessage(pb) - if err != nil { - return fmt.Errorf("marshaling message: %v", err) - } - - u := uriPathToURL(uri, "/cluster/message") - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) - if err != nil { - return errors.Wrap(err, "making new request") - } - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return fmt.Errorf("executing http request: %v", err) - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response body: %v", err) - } - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - default: - return fmt.Errorf("unexpected response status code: %d: %s", resp.StatusCode, body) - } - - return nil -} - // Bit represents the intersection of a row and a column. It can be specifed by // integer ids or string keys. type Bit struct { @@ -883,83 +18,6 @@ type Bit struct { Timestamp int64 } -// Bits is a slice of Bit. -type Bits []Bit - -func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p Bits) Len() int { return len(p) } - -func (p Bits) Less(i, j int) bool { - if p[i].RowID == p[j].RowID { - if p[i].ColumnID < p[j].ColumnID { - return p[i].Timestamp < p[j].Timestamp - } - return p[i].ColumnID < p[j].ColumnID - } - return p[i].RowID < p[j].RowID -} - -// RowIDs returns a slice of all the row IDs. -func (p Bits) RowIDs() []uint64 { - other := make([]uint64, len(p)) - for i := range p { - other[i] = p[i].RowID - } - return other -} - -// ColumnIDs returns a slice of all the column IDs. -func (p Bits) ColumnIDs() []uint64 { - other := make([]uint64, len(p)) - for i := range p { - other[i] = p[i].ColumnID - } - return other -} - -// RowKeys returns a slice of all the row keys. -func (p Bits) RowKeys() []string { - other := make([]string, len(p)) - for i := range p { - other[i] = p[i].RowKey - } - return other -} - -// ColumnKeys returns a slice of all the column keys. -func (p Bits) ColumnKeys() []string { - other := make([]string, len(p)) - for i := range p { - other[i] = p[i].ColumnKey - } - return other -} - -// Timestamps returns a slice of all the timestamps. -func (p Bits) Timestamps() []int64 { - other := make([]int64, len(p)) - for i := range p { - other[i] = p[i].Timestamp - } - return other -} - -// GroupBySlice returns a map of bits by slice. -func (p Bits) GroupBySlice() map[uint64][]Bit { - m := make(map[uint64][]Bit) - for _, bit := range p { - slice := bit.ColumnID / SliceWidth - m[slice] = append(m[slice], bit) - } - - for slice, bits := range m { - sort.Sort(Bits(bits)) - m[slice] = bits - } - - return m -} - // FieldValues represents the value for a column within a // range-encoded field. type FieldValue struct { @@ -967,79 +25,6 @@ type FieldValue struct { Value int64 } -// FieldValues represents a slice of field values. -type FieldValues []FieldValue - -func (p FieldValues) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p FieldValues) Len() int { return len(p) } - -func (p FieldValues) Less(i, j int) bool { - return p[i].ColumnID < p[j].ColumnID -} - -// ColumnIDs returns a slice of all the column IDs. -func (p FieldValues) ColumnIDs() []uint64 { - other := make([]uint64, len(p)) - for i := range p { - other[i] = p[i].ColumnID - } - return other -} - -// Values returns a slice of all the values. -func (p FieldValues) Values() []int64 { - other := make([]int64, len(p)) - for i := range p { - other[i] = p[i].Value - } - return other -} - -// GroupBySlice returns a map of field values by slice. -func (p FieldValues) GroupBySlice() map[uint64][]FieldValue { - m := make(map[uint64][]FieldValue) - for _, val := range p { - slice := val.ColumnID / SliceWidth - m[slice] = append(m[slice], val) - } - - for slice, vals := range m { - sort.Sort(FieldValues(vals)) - m[slice] = vals - } - - return m -} - -// BitsByPos is a slice of bits sorted row then column. -type BitsByPos []Bit - -func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p BitsByPos) Len() int { return len(p) } -func (p BitsByPos) Less(i, j int) bool { - p0, p1 := Pos(p[i].RowID, p[i].ColumnID), Pos(p[j].RowID, p[j].ColumnID) - if p0 == p1 { - return p[i].Timestamp < p[j].Timestamp - } - return p0 < p1 -} - -func uriPathToURL(uri *URI, path string) url.URL { - return url.URL{ - Scheme: uri.Scheme(), - Host: uri.HostPort(), - Path: path, - } -} - -func nodePathToURL(node *Node, path string) url.URL { - return url.URL{ - Scheme: node.URI.Scheme(), - Host: node.URI.HostPort(), - Path: path, - } -} - // InternalClient should be implemented by any struct that enables any transport between nodes // TODO: Refactor // Note from Travis: Typically an interface containing more than two or three methods is an indication that @@ -1060,9 +45,96 @@ type InternalClient interface { ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error CreateField(ctx context.Context, index, field string, opt FieldOptions) error - FragmentBlocks(ctx context.Context, index, field string, slice uint64) ([]FragmentBlock, error) - BlockData(ctx context.Context, index, field string, slice uint64, block int) ([]uint64, []uint64, error) - ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - RowAttrDiff(ctx context.Context, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) + FragmentBlocks(ctx context.Context, uri *URI, index, field string, slice uint64) ([]FragmentBlock, error) + BlockData(ctx context.Context, uri *URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) + ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) + RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error + RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) +} + +//=============== + +type InternalQueryClient interface { + QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) +} + +type NopInternalQueryClient struct{} + +func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return nil, nil +} + +func NewNopInternalQueryClient() *NopInternalQueryClient { + return &NopInternalQueryClient{} +} + +var _ InternalQueryClient = NewNopInternalQueryClient() + +//=============== + +type NopInternalClient struct{} + +func NewNopInternalClient() *NopInternalClient { + return &NopInternalClient{} +} + +var _ InternalClient = NewNopInternalClient() + +func (n *NopInternalClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { + return nil, nil +} +func (n *NopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { + return nil, nil +} +func (n *NopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { + return nil +} +func (n *NopInternalClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) { + return nil, nil +} +func (n *NopInternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return nil, nil +} +func (n *NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return nil, nil +} +func (n *NopInternalClient) Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error { + return nil +} +func (n *NopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { + return nil +} +func (n *NopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { + return nil +} +func (n *NopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string, options FieldOptions) error { + return nil +} +func (n *NopInternalClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error { + return nil +} +func (n *NopInternalClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { + return nil +} +func (n *NopInternalClient) CreateField(ctx context.Context, index, field string, opt FieldOptions) error { + return nil +} +func (n *NopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, slice uint64) ([]FragmentBlock, error) { + return nil, nil +} +func (n *NopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { + return nil, nil, nil +} +func (n *NopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { + return nil, nil +} +func (n *NopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { + return nil, nil +} +func (n *NopInternalClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { + return nil +} +func (n *NopInternalClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) { + return nil, nil } diff --git a/cluster.go b/cluster.go index 9723e5bce..81ac03424 100644 --- a/cluster.go +++ b/cluster.go @@ -266,6 +266,8 @@ type Cluster struct { // RemoteClient *http.Client + + InternalClient InternalClient } // NewCluster returns a new instance of Cluster with defaults. @@ -281,6 +283,8 @@ func NewCluster() *Cluster { closing: make(chan struct{}), joining: make(chan struct{}), + InternalClient: NewNopInternalClient(), + Logger: NopLogger, } } @@ -1230,9 +1234,6 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err return errors.Wrap(err, "applying schema") } - // Create a client for calling remote nodes. - client := NewInternalHTTPClientFromURI(&c.Node.URI, c.RemoteClient) // TODO: ClientOptions - // Request each source file in ResizeSources. for _, src := range instr.Sources { c.Logger.Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) @@ -1259,7 +1260,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err // Stream slice from remote node. c.Logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) - rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.Slice, srcURI) + rd, err := c.InternalClient.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.Slice, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found // on the remote node. This occurs when a slice has been skipped and diff --git a/ctl/common.go b/ctl/common.go index 11042a704..fe0c46f7b 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -17,7 +17,7 @@ package ctl import ( "crypto/tls" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pkg/errors" "github.com/spf13/pflag" @@ -37,7 +37,7 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP } // CommandClient returns a pilosa.InternalHTTPClient for the command -func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error) { +func CommandClient(cmd CommandWithTLSSupport) (*http.InternalHTTPClient, error) { tlsConfig := cmd.TLSConfiguration() var TLSConfig *tls.Config if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" { @@ -50,7 +50,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error InsecureSkipVerify: tlsConfig.SkipVerify, } } - client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig)) + client, err := http.NewInternalHTTPClient(cmd.TLSHost(), http.GetHTTPClient(TLSConfig)) if err != nil { return nil, errors.Wrap(err, "getting internal client") } diff --git a/ctl/import.go b/ctl/import.go index 9383c300f..c4d65f232 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -26,6 +26,7 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pkg/errors" ) @@ -245,12 +246,12 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err // Group bits by slice. logger.Printf("grouping %d bits", len(bits)) - bitsBySlice := pilosa.Bits(bits).GroupBySlice() + bitsBySlice := http.Bits(bits).GroupBySlice() // Parse path into bits. for slice, chunk := range bitsBySlice { if cmd.Sort { - sort.Sort(pilosa.BitsByPos(chunk)) + sort.Sort(http.BitsByPos(chunk)) } logger.Printf("importing slice: %d, n=%d", slice, len(chunk)) @@ -439,12 +440,12 @@ func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldV // Group vals by slice. logger.Printf("grouping %d vals", len(vals)) - valsBySlice := pilosa.FieldValues(vals).GroupBySlice() + valsBySlice := http.FieldValues(vals).GroupBySlice() // Parse path into FieldValues. for slice, vals := range valsBySlice { if cmd.Sort { - sort.Sort(pilosa.FieldValues(vals)) + sort.Sort(http.FieldValues(vals)) } logger.Printf("importing slice: %d, n=%d", slice, len(vals)) diff --git a/executor.go b/executor.go index 411039d40..4e2ffe9bf 100644 --- a/executor.go +++ b/executor.go @@ -17,7 +17,6 @@ package pilosa import ( "context" "fmt" - "net/http" "sort" "time" @@ -47,19 +46,35 @@ type Executor struct { Cluster *Cluster // Client used for remote requests. - client InternalClient + client InternalQueryClient // Maximum number of SetBit() or ClearBit() commands per request. MaxWritesPerRequest int } -// NewExecutor returns a new instance of Executor. -func NewExecutor(remoteClient *http.Client) *Executor { - return &Executor{ - client: NewInternalHTTPClientFromURI(nil, remoteClient), +type ExecutorOpt func(e *Executor) error + +func ExecutorOptInternalQueryClient(c InternalQueryClient) ExecutorOpt { + return func(e *Executor) error { + e.client = c + return nil } } +// NewExecutor returns a new instance of Executor. +func NewExecutor(opts ...ExecutorOpt) *Executor { + e := &Executor{ + client: NewNopInternalQueryClient(), + } + for _, opt := range opts { + err := opt(e) + if err != nil { + panic(err) + } + } + return e +} + // Execute executes a PQL query. func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { // Verify that an index is set. @@ -1380,7 +1395,7 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q * case "SetRowAttrs": case "SetColumnAttrs": default: - v, err = decodeRow(pb.Results[i].GetRow()), nil + v, err = DecodeRow(pb.Results[i].GetRow()), nil } if err != nil { return nil, err @@ -1619,7 +1634,7 @@ func (vc *ValCount) Add(other ValCount) ValCount { } } -func encodeValCount(vc ValCount) *internal.ValCount { +func EncodeValCount(vc ValCount) *internal.ValCount { return &internal.ValCount{ Val: vc.Val, Count: vc.Count, diff --git a/fragment.go b/fragment.go index 1dfa9d09f..afa80f5f1 100644 --- a/fragment.go +++ b/fragment.go @@ -1782,8 +1782,7 @@ func (s *FragmentSyncer) SyncFragment() error { } // Retrieve remote blocks. - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Field(), s.Fragment.Slice()) + blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.Index(), s.Fragment.Field(), s.Fragment.Slice()) if err != nil && err != ErrFragmentNotFound { return errors.Wrap(err, "getting blocks") } @@ -1847,7 +1846,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Read pairs from each remote block. var pairSets []PairSet - var clients []InternalClient + var uris []*URI for _, node := range s.Cluster.SliceNodes(f.Index(), f.Slice()) { if s.Node.ID == node.ID { continue @@ -1858,11 +1857,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { return nil } - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - clients = append(clients, client) + uri := &node.URI + uris = append(uris, uri) // Only sync the standard block. - rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Field(), f.Slice(), id) + rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.Index(), f.Field(), f.Slice(), id) if err != nil { return errors.Wrap(err, "getting block") } @@ -1885,7 +1884,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { } // Write updates to remote blocks. - for i := 0; i < len(clients); i++ { + for i := 0; i < len(uris); i++ { set, clear := sets[i], clears[i] count := 0 @@ -1924,7 +1923,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { Query: buffers[k].String(), Remote: true, } - _, err := clients[i].Query(context.Background(), f.Index(), queryRequest) + _, err := s.Cluster.InternalClient.QueryNode(context.Background(), uris[i], f.Index(), queryRequest) if err != nil { return errors.Wrap(err, "executing") } diff --git a/handler.go b/handler.go index c4bb364ea..e70368c49 100644 --- a/handler.go +++ b/handler.go @@ -1,947 +1,8 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package pilosa import ( "encoding/json" - "expvar" - "fmt" - "io" - "io/ioutil" "net/http" - "net/url" - // Imported for its side-effect of registering pprof endpoints with the server. - _ "net/http/pprof" - "reflect" - "runtime/debug" - "strconv" - "strings" - "time" - - "github.com/gogo/protobuf/proto" - "github.com/gorilla/handlers" - "github.com/gorilla/mux" - "github.com/pilosa/pilosa/internal" - - "github.com/pkg/errors" -) - -// Handler represents an HTTP handler. -type Handler struct { - Handler http.Handler - - Logger Logger - - // Keeps the query argument validators for each handler - validators map[string]*queryValidationSpec - - API *API - - AllowedOrigins []string -} - -// externalPrefixFlag denotes endpoints that are intended to be exposed to clients. -// This is used for stats tagging. -var externalPrefixFlag = map[string]bool{ - "schema": true, - "query": true, - "import": true, - "export": true, - "index": true, - "field": true, - "nodes": true, - "version": true, -} - -type errorResponse struct { - Error string `json:"error"` -} - -// HandlerOption is a functional option type for pilosa.Handler -type HandlerOption func(s *Handler) error - -func OptHandlerAllowedOrigins(origins []string) HandlerOption { - return func(h *Handler) error { - h.Handler = handlers.CORS( - handlers.AllowedOrigins(origins), - handlers.AllowedHeaders([]string{"Content-Type"}), - )(h.Handler) - return nil - } -} - -// NewHandler returns a new instance of Handler with a default logger. -func NewHandler(opts ...HandlerOption) (*Handler, error) { - handler := &Handler{ - Logger: NopLogger, - } - handler.Handler = NewRouter(handler) - handler.populateValidators() - - for _, opt := range opts { - err := opt(handler) - if err != nil { - return nil, errors.Wrap(err, "applying option") - } - } - - return handler, nil -} - -func (h *Handler) populateValidators() { - h.validators = map[string]*queryValidationSpec{} - h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") - h.validators["GetSliceMax"] = queryValidationSpecRequired() - h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") - h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "slice") - h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "slice") - h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "slice") - h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "slice") -} - -func (h *Handler) queryArgValidator(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - key := mux.CurrentRoute(r).GetName() - if validator, ok := h.validators[key]; ok { - if err := validator.validate(r.URL.Query()); err != nil { - // TODO: Return the response depending on the Accept header - response := errorResponse{Error: err.Error()} - body, err := json.Marshal(response) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - http.Error(w, string(body), http.StatusBadRequest) - return - } - } - next.ServeHTTP(w, r) - }) -} - -// NewRouter creates a new mux http router. -func NewRouter(handler *Handler) *mux.Router { - router := mux.NewRouter() - router.HandleFunc("/", handler.handleHome).Methods("GET") - router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") - router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") - router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.Handle("/debug/vars", expvar.Handler()).Methods("GET") - router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") - router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client - router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") - router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") - router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") - - router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") - - router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") - router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.Handle("/debug/vars", expvar.Handler()).Methods("GET") - router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") - router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") - router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") - router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") - router.HandleFunc("/import", handler.handlePostImport).Methods("POST") - router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST") - router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") - router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") - router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") - router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") - router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") - //router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented. - router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") - router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") - router.HandleFunc("/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST") - router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") - router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") - - // TODO: Apply MethodNotAllowed statuses to all endpoints. - // Ideally this would be automatic, as described in this (wontfix) ticket: - // https://github.com/gorilla/mux/issues/6 - // For now we just do it for the most commonly used handler, /query - router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") - - router.Use(handler.queryArgValidator) - return router -} - -func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) -} - -// ServeHTTP handles an HTTP request. -func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - defer func() { - if err := recover(); err != nil { - w.WriteHeader(http.StatusInternalServerError) - stack := debug.Stack() - msg := "PANIC: %s\n%s" - h.Logger.Printf(msg, err, stack) - fmt.Fprintf(w, msg, err, stack) - } - }() - - t := time.Now() - h.Handler.ServeHTTP(w, r) - dif := time.Since(t) - - // Calculate per request StatsD metrics when the handler is fully configured. - statsTags := make([]string, 0, 3) - - longQueryTime := h.API.LongQueryTime() - if longQueryTime > 0 && dif > longQueryTime { - h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) - statsTags = append(statsTags, "slow_query") - } - - pathParts := strings.Split(r.URL.Path, "/") - endpointName := strings.Join(pathParts, "_") - - if externalPrefixFlag[pathParts[1]] { - statsTags = append(statsTags, "external") - } - - // useragent tag identifies internal/external endpoints - statsTags = append(statsTags, "useragent:"+r.UserAgent()) - stats := h.API.StatsWithTags(statsTags) - if stats != nil { - stats.Histogram("http."+endpointName, float64(dif), 0.1) - } -} - -func (h *Handler) handleHome(w http.ResponseWriter, r *http.Request) { - http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.", http.StatusNotFound) -} - -// handleGetSchema handles GET /schema requests. -func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { - schema := h.API.Schema(r.Context()) - if err := json.NewEncoder(w).Encode(getSchemaResponse{ - Indexes: schema, - }); err != nil { - h.Logger.Printf("write schema response error: %s", err) - } -} - -// handleGetStatus handles GET /status requests. -func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { - status := getStatusResponse{ - State: h.API.State(), - Nodes: h.API.Hosts(r.Context()), - LocalID: h.API.LocalID(), - } - if err := json.NewEncoder(w).Encode(status); err != nil { - h.Logger.Printf("write status response error: %s", err) - } -} - -func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { - info := h.API.Info() - if err := json.NewEncoder(w).Encode(info); err != nil { - h.Logger.Printf("write info response error: %s", err) - } -} - -type getSchemaResponse struct { - Indexes []*IndexInfo `json:"indexes"` -} - -type getStatusResponse struct { - State string `json:"state"` - Nodes []*Node `json:"nodes"` - LocalID string `json:"localID"` -} - -// handlePostQuery handles /query requests. -func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { - // Parse incoming request. - req, err := h.readQueryRequest(r) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - // TODO: Remove - req.Index = mux.Vars(r)["index"] - - resp, err := h.API.Query(r.Context(), req) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - - // Set appropriate status code, if there is an error. - if resp.Err != nil { - switch resp.Err { - case ErrTooManyWrites: - w.WriteHeader(http.StatusRequestEntityTooLarge) - default: - w.WriteHeader(http.StatusInternalServerError) - } - } - - // Write response back to client. - if err := h.writeQueryResponse(w, r, &resp); err != nil { - h.Logger.Printf("write query response error: %s", err) - } -} - -// handleGetSlicesMax handles GET /schema requests. -func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { - if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ - Standard: h.API.MaxSlices(r.Context()), - }); err != nil { - h.Logger.Printf("write slices-max response error: %s", err) - } -} - -type getSlicesMaxResponse struct { - Standard map[string]uint64 `json:"standard"` -} - -// handleGetIndexes handles GET /index request. -func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { - h.handleGetSchema(w, r) -} - -// handleGetIndex handles GET /index/ requests. -func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - index, err := h.API.Index(r.Context(), indexName) - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - if err := json.NewEncoder(w).Encode(getIndexResponse{ - map[string]string{"name": index.Name()}, - }); err != nil { - h.Logger.Printf("write response error: %s", err) - } -} - -type getIndexResponse struct { - Index map[string]string `json:"index"` -} - -type postIndexRequest struct { - Options IndexOptions `json:"options"` -} - -//_postIndexRequest is necessary to avoid recursion while decoding. -type _postIndexRequest postIndexRequest - -// Custom Unmarshal JSON to validate request body when creating a new index. -func (p *postIndexRequest) UnmarshalJSON(b []byte) error { - - // m is an overflow map used to capture additional, unexpected keys. - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return errors.Wrap(err, "unmarshalling unexpected values") - } - - validIndexOptions := getValidOptions(IndexOptions{}) - err := validateOptions(m, validIndexOptions) - if err != nil { - return err - } - // Unmarshal expected values. - var _p _postIndexRequest - if err := json.Unmarshal(b, &_p); err != nil { - return errors.Wrap(err, "unmarshalling expected values") - } - - p.Options = _p.Options - - return nil -} - -// Raise errors for any unknown key -func validateOptions(data map[string]interface{}, validIndexOptions []string) error { - for k, v := range data { - switch k { - case "options": - options, ok := v.(map[string]interface{}) - if !ok { - return errors.New("options is not map[string]interface{}") - } - for kk, vv := range options { - if !foundItem(validIndexOptions, kk) { - return fmt.Errorf("Unknown key: %v:%v", kk, vv) - } - } - default: - return fmt.Errorf("Unknown key: %v:%v", k, v) - } - } - return nil -} - -func foundItem(items []string, item string) bool { - for _, i := range items { - if item == i { - return true - } - } - return false -} - -type postIndexResponse struct{} - -// handleDeleteIndex handles DELETE /index request. -func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - err := h.API.DeleteIndex(r.Context(), indexName) - if err != nil { - h.Logger.Printf("problem deleting index: %s", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type deleteIndexResponse struct{} - -// handlePostIndex handles POST /index request. -func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - - // Decode request. - var req postIndexRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err == io.EOF { - // If no data was provided (EOF), we still create the index - // with default values. - } else if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - _, err = h.API.CreateIndex(r.Context(), indexName, req.Options) - if errors.Cause(err) == ErrIndexExists { - http.Error(w, err.Error(), http.StatusConflict) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -// handlePostIndexAttrDiff handles POST /index/attr/diff requests. -func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - - // Decode request. - var req postIndexAttrDiffRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - attrs, err := h.API.IndexAttrDiff(r.Context(), indexName, req.Blocks) - if err != nil { - if errors.Cause(err) == ErrIndexNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{ - Attrs: attrs, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type postIndexAttrDiffRequest struct { - Blocks []AttrBlock `json:"blocks"` -} - -type postIndexAttrDiffResponse struct { - Attrs map[uint64]map[string]interface{} `json:"attrs"` -} - -// handlePostField handles POST /field request. -func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - fieldName := mux.Vars(r)["field"] - - // Decode request. - var req postFieldRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err == io.EOF { - // If no data was provided (EOF), we still create the field - // with default values. - } else if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - _, err = h.API.CreateField(r.Context(), indexName, fieldName, req.Options) - if err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrFieldExists: - http.Error(w, err.Error(), http.StatusConflict) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - // Encode response. - if err := json.NewEncoder(w).Encode(postFieldResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type _postFieldRequest postFieldRequest - -// Custom Unmarshal JSON to validate request body when creating a new field. If there's new FieldOptions, -// adding it to validFieldOptions to make sure the new option is validated, otherwise the request will be failed -func (p *postFieldRequest) UnmarshalJSON(b []byte) error { - // m is an overflow map used to capture additional, unexpected keys. - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return errors.Wrap(err, "unmarshaling unexpected keys") - } - - validFieldOptions := getValidOptions(FieldOptions{}) - err := validateOptions(m, validFieldOptions) - if err != nil { - return err - } - - // Unmarshal expected values. - var _p _postFieldRequest - if err := json.Unmarshal(b, &_p); err != nil { - return errors.Wrap(err, "unmarshalling expected keys") - } - - p.Options = _p.Options - return nil - -} - -func getValidOptions(option interface{}) []string { - validOptions := []string{} - val := reflect.ValueOf(option) - for i := 0; i < val.Type().NumField(); i++ { - jsonTag := val.Type().Field(i).Tag.Get("json") - s := strings.Split(jsonTag, ",") - validOptions = append(validOptions, s[0]) - } - return validOptions -} - -type postFieldRequest struct { - Options FieldOptions `json:"options"` -} - -type postFieldResponse struct{} - -// handleDeleteField handles DELETE /field request. -func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - fieldName := mux.Vars(r)["field"] - - err := h.API.DeleteField(r.Context(), indexName, fieldName) - if err != nil { - if errors.Cause(err) == ErrIndexNotFound { - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } - return - } - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(deleteFieldResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type deleteFieldResponse struct{} - -// handlePostFieldAttrDiff handles POST /field/attr/diff requests. -func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - fieldName := mux.Vars(r)["field"] - - // Decode request. - var req postFieldAttrDiffRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - attrs, err := h.API.FieldAttrDiff(r.Context(), indexName, fieldName, req.Blocks) - if err != nil { - switch errors.Cause(err) { - case ErrFragmentNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postFieldAttrDiffResponse{ - Attrs: attrs, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type postFieldAttrDiffRequest struct { - Blocks []AttrBlock `json:"blocks"` -} - -type postFieldAttrDiffResponse struct { - Attrs map[uint64]map[string]interface{} `json:"attrs"` -} - -// readQueryRequest parses an query parameters from r. -func (h *Handler) readQueryRequest(r *http.Request) (*QueryRequest, error) { - switch r.Header.Get("Content-Type") { - case "application/x-protobuf": - return h.readProtobufQueryRequest(r) - default: - return h.readURLQueryRequest(r) - } -} - -// readProtobufQueryRequest parses query parameters in protobuf from r. -func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, error) { - // Slurp the body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, errors.Wrap(err, "reading") - } - - // Unmarshal into object. - var req internal.QueryRequest - if err := proto.Unmarshal(body, &req); err != nil { - return nil, errors.Wrap(err, "unmarshalling") - } - - return decodeQueryRequest(&req), nil -} - -// readURLQueryRequest parses query parameters from URL parameters from r. -func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { - q := r.URL.Query() - - // Parse query string. - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, errors.Wrap(err, "reading") - } - query := string(buf) - - // Parse list of slices. - slices, err := parseUint64Slice(q.Get("slices")) - if err != nil { - return nil, errors.New("invalid slice argument") - } - - return &QueryRequest{ - Query: query, - Slices: slices, - ColumnAttrs: q.Get("columnAttrs") == "true", - ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true", - ExcludeColumns: q.Get("excludeColumns") == "true", - }, nil -} - -// writeQueryResponse writes the response from the executor to w. -func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *QueryResponse) error { - if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { - return h.writeProtobufQueryResponse(w, resp) - } - return h.writeJSONQueryResponse(w, resp) -} - -// writeProtobufQueryResponse writes the response from the executor to w as protobuf. -func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *QueryResponse) error { - if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil { - return errors.Wrap(err, "marshalling") - } else if _, err := w.Write(buf); err != nil { - return errors.Wrap(err, "writing") - } - return nil -} - -// writeJSONQueryResponse writes the response from the executor to w as JSON. -func (h *Handler) writeJSONQueryResponse(w http.ResponseWriter, resp *QueryResponse) error { - return json.NewEncoder(w).Encode(resp) -} - -// handlePostImport handles /import requests. -func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Marshal into request object. - var req internal.ImportRequest - if err := proto.Unmarshal(body, &req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.API.Import(r.Context(), req); err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - fallthrough - case ErrFieldNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrClusterDoesNotOwnSlice: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) - if e != nil { - http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) - return - } - - // Write response. - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - } - w.Write(buf) -} - -// handlePostImportValue handles /import-value requests. -func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Marshal into request object. - var req internal.ImportValueRequest - if err := proto.Unmarshal(body, &req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err = h.API.ImportValue(r.Context(), req); err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - fallthrough - case ErrFieldNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrClusterDoesNotOwnSlice: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) - if e != nil { - http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) - return - } - - // Write response. - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - } - w.Write(buf) -} - -// handleGetExport handles /export requests. -func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { - switch r.Header.Get("Accept") { - case "text/csv": - h.handleGetExportCSV(w, r) - default: - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - } -} - -func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { - // Parse query parameters. - q := r.URL.Query() - index, field := q.Get("index"), q.Get("field") - - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "invalid slice", http.StatusBadRequest) - return - } - - if err = h.API.ExportCSV(r.Context(), index, field, slice, w); err != nil { - switch errors.Cause(err) { - case ErrFragmentNotFound: - break - case ErrClusterDoesNotOwnSlice: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } -} - -// handleGetFragmentNodes handles /fragment/nodes requests. -func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) { - q := r.URL.Query() - index := q.Get("index") - - // Read slice parameter. - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "slice should be an unsigned integer", http.StatusBadRequest) - return - } - - // Retrieve fragment owner nodes. - nodes, err := h.API.SliceNodes(r.Context(), index, slice) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Write to response. - if err := json.NewEncoder(w).Encode(nodes); err != nil { - h.Logger.Printf("json write error: %s", err) - } -} - -// handleGetFragmentBlockData handles GET /fragment/block/data requests. -func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { - buf, err := h.API.FragmentBlockData(r.Context(), r.Body) - if err != nil { - if _, ok := err.(BadRequestError); ok { - http.Error(w, err.Error(), http.StatusBadRequest) - } else if errors.Cause(err) == ErrFragmentNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Write response. - w.Header().Set("Content-Type", "application/protobuf") - w.Header().Set("Content-Length", strconv.Itoa(len(buf))) - w.Write(buf) -} - -// handleGetFragmentBlocks handles GET /fragment/blocks requests. -func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request) { - // Read slice parameter. - q := r.URL.Query() - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "slice required", http.StatusBadRequest) - return - } - - blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), slice) - if err != nil { - if errors.Cause(err) == ErrFragmentNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ - Blocks: blocks, - }); err != nil { - h.Logger.Printf("block response encoding error: %s", err) - } -} - -type getFragmentBlocksResponse struct { - Blocks []FragmentBlock `json:"blocks"` -} - -// handleGetVersion handles /version requests. -func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { - err := json.NewEncoder(w).Encode(struct { - Version string `json:"version"` - }{ - Version: h.API.Version(), - }) - if err != nil { - h.Logger.Printf("write version response error: %s", err) - } -} - -// QueryResult types. -const ( - QueryResultTypeNil uint32 = iota - QueryResultTypeRow - QueryResultTypePairs - QueryResultTypeValCount - QueryResultTypeUint64 - QueryResultTypeBool ) // QueryRequest represent a request to process a query. @@ -970,19 +31,6 @@ type QueryRequest struct { Remote bool } -func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { - req := &QueryRequest{ - Query: pb.Query, - Slices: pb.Slices, - ColumnAttrs: pb.ColumnAttrs, - Remote: pb.Remote, - ExcludeRowAttrs: pb.ExcludeRowAttrs, - ExcludeColumns: pb.ExcludeColumns, - } - - return req -} - // QueryResponse represent a response from a processed query. type QueryResponse struct { // Result for each top-level query call. @@ -1012,234 +60,19 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { return json.Marshal(output) } -func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { - pb := &internal.QueryResponse{ - Results: make([]*internal.QueryResult, len(resp.Results)), - ColumnAttrSets: encodeColumnAttrSets(resp.ColumnAttrSets), - } - - for i := range resp.Results { - pb.Results[i] = &internal.QueryResult{} - - switch result := resp.Results[i].(type) { - case *Row: - pb.Results[i].Type = QueryResultTypeRow - pb.Results[i].Row = encodeRow(result) - case []Pair: - pb.Results[i].Type = QueryResultTypePairs - pb.Results[i].Pairs = encodePairs(result) - case ValCount: - pb.Results[i].Type = QueryResultTypeValCount - pb.Results[i].ValCount = encodeValCount(result) - case uint64: - pb.Results[i].Type = QueryResultTypeUint64 - pb.Results[i].N = result - case bool: - pb.Results[i].Type = QueryResultTypeBool - pb.Results[i].Changed = result - case nil: - pb.Results[i].Type = QueryResultTypeNil - } - } - - if resp.Err != nil { - pb.Err = resp.Err.Error() - } - - return pb +type Handlerer interface { + http.Handler + GetAPI() *API } -// parseUint64Slice returns a slice of uint64s from a comma-delimited string. -func parseUint64Slice(s string) ([]uint64, error) { - var a []uint64 - for _, str := range strings.Split(s, ",") { - // Ignore blanks. - if str == "" { - continue - } +type NopHandler struct{} - // Parse number. - num, err := strconv.ParseUint(str, 10, 64) - if err != nil { - return nil, errors.Wrap(err, "parsing int") - } - a = append(a, num) - } - return a, nil -} +func (n *NopHandler) ServeHTTP(_ http.ResponseWriter, _ *http.Request) {} -// errorString returns the string representation of err. -func errorString(err error) string { - if err == nil { - return "" - } - return err.Error() -} - -func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { - // Decode request. - var req setCoordinatorRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, "decoding request "+err.Error(), http.StatusBadRequest) - return - } - - oldNode, newNode, err := h.API.SetCoordinator(r.Context(), req.ID) - if err != nil { - if errors.Cause(err) == ErrNodeIDNotExists { - http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound) - } else { - http.Error(w, "setting new coordinator: "+err.Error(), http.StatusInternalServerError) - } - return - } - // Encode response. - if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ - Old: oldNode, - New: newNode, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type setCoordinatorRequest struct { - ID string `json:"id"` -} - -type setCoordinatorResponse struct { - Old *Node `json:"old"` - New *Node `json:"new"` -} - -// handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request. -func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *http.Request) { - // Decode request. - var req removeNodeRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - removeNode, err := h.API.RemoveNode(req.ID) - if err != nil { - if errors.Cause(err) == ErrNodeIDNotExists { - http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) - } else { - http.Error(w, "removing node: "+err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(removeNodeResponse{ - Remove: removeNode, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type removeNodeRequest struct { - ID string `json:"id"` -} - -type removeNodeResponse struct { - Remove *Node `json:"remove"` -} - -// handlePostClusterResizeAbort handles POST /cluster/resize/abort request. -func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { - err := h.API.ResizeAbort() - var msg string - if err != nil { - switch errors.Cause(err) { - case ErrNodeNotCoordinator: - http.Error(w, err.Error(), http.StatusBadRequest) - return - case ErrResizeNotRunning: - msg = err.Error() - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - // Encode response. - if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ - Info: msg, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type clusterResizeAbortResponse struct { - Info string `json:"info"` -} - -func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { - err := h.API.RecalculateCaches(r.Context()) - if err != nil { - http.Error(w, "recalculating caches: "+err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusNoContent) -} - -func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } - - err := h.API.ClusterMessage(r.Context(), r.Body) - if err != nil { - // TODO this was the previous behavior, but perhaps not everything is a bad request - http.Error(w, err.Error(), http.StatusBadRequest) - } - - if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type defaultClusterMessageResponse struct{} - -type queryValidationSpec struct { - required []string - args map[string]struct{} -} - -func queryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { - args := map[string]struct{}{} - for _, arg := range requiredArgs { - args[arg] = struct{}{} - } - - return &queryValidationSpec{ - required: requiredArgs, - args: args, - } -} - -func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec { - for _, arg := range args { - s.args[arg] = struct{}{} - } - return s -} - -func (s queryValidationSpec) validate(query url.Values) error { - for _, req := range s.required { - if query.Get(req) == "" { - return errors.Errorf("%s is required", req) - } - } - for k := range query { - if _, ok := s.args[k]; !ok { - return errors.Errorf("%s is not a valid argument", k) - } - } +func (n *NopHandler) GetAPI() *API { return nil } + +func NewNopHandler() Handlerer { + return &NopHandler{} +} diff --git a/holder.go b/holder.go index e8db140ce..2330d59e7 100644 --- a/holder.go +++ b/holder.go @@ -662,11 +662,9 @@ func (s *HolderSyncer) syncIndex(index string) error { // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) { - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.ColumnAttrDiff(context.Background(), index, blks) + m, err := s.Cluster.InternalClient.ColumnAttrDiff(context.Background(), &node.URI, index, blks) if err != nil { return errors.Wrap(err, "getting differing blocks") } else if len(m) == 0 { @@ -708,11 +706,9 @@ func (s *HolderSyncer) syncField(index, name string) error { // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) { - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.RowAttrDiff(context.Background(), index, name, blks) + m, err := s.Cluster.InternalClient.RowAttrDiff(context.Background(), &node.URI, index, name, blks) if err == ErrFieldNotFound { continue // field not created remotely yet, skip } else if err != nil { diff --git a/holder_test.go b/holder_test.go index 5f41cfab7..70fecf245 100644 --- a/holder_test.go +++ b/holder_test.go @@ -24,8 +24,8 @@ import ( "testing" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -360,8 +360,20 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { + s := test.NewServer() + defer s.Close() + + uri, err := pilosa.NewURIFromAddress(s.URL) + if err != nil { + t.Fatal(err) + } + cluster := test.NewCluster(2) - client := server.GetHTTPClient(nil) + client := http.GetHTTPClient(nil) + httpClient := http.NewInternalHTTPClientFromURI(uri, client) + cluster.InternalClient = httpClient + cluster.RemoteClient = client + // Create a local holder. hldr0 := test.MustOpenHolder() defer hldr0.Close() @@ -369,11 +381,9 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Create a remote holder wrapped by an HTTP hldr1 := test.MustOpenHolder() defer hldr1.Close() - s := test.NewServer() - defer s.Close() s.Handler.API.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(client) + e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) e.Holder = hldr1.Holder e.Node = cluster.Nodes[1] e.Cluster = cluster @@ -383,11 +393,6 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Mock 2-node, fully replicated cluster. cluster.ReplicaN = 2 - uri, err := pilosa.NewURIFromAddress(s.URL) - if err != nil { - t.Fatal(err) - } - cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) cluster.Nodes[1].URI = *uri @@ -445,7 +450,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { Holder: hldr0.Holder, Node: cluster.Nodes[0], Cluster: cluster, - RemoteClient: server.GetHTTPClient(nil), + RemoteClient: http.GetHTTPClient(nil), Stats: pilosa.NopStatsClient, } diff --git a/http/client.go b/http/client.go new file mode 100644 index 000000000..b0fee7887 --- /dev/null +++ b/http/client.go @@ -0,0 +1,1034 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + "net/url" + "sort" + "strconv" + + "crypto/tls" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" +) + +// ClientOptions represents the configuration for a InternalHTTPClient +type ClientOptions struct { + TLS *tls.Config +} + +// InternalHTTPClient represents a client to the Pilosa cluster. +type InternalHTTPClient struct { + defaultURI *pilosa.URI + + // The client to use for HTTP communication. + HTTPClient *http.Client +} + +// NewInternalHTTPClient returns a new instance of InternalHTTPClient to connect to host. +func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTTPClient, error) { + if host == "" { + return nil, pilosa.ErrHostRequired + } + + uri, err := pilosa.NewURIFromAddress(host) + if err != nil { + return nil, errors.Wrap(err, "getting URI") + } + + client := NewInternalHTTPClientFromURI(uri, remoteClient) + return client, nil +} + +func NewInternalHTTPClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalHTTPClient { + return &InternalHTTPClient{ + defaultURI: defaultURI, + HTTPClient: remoteClient, + } +} + +// Host returns the host the client was initialized with. +func (c *InternalHTTPClient) Host() *pilosa.URI { return c.defaultURI } + +// MaxSliceByIndex returns the number of slices on a server by index. +func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { + return c.maxSliceByIndex(ctx) +} + +// maxSliceByIndex returns the number of slices on a server by index. +func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) { + // Execute request against the host. + u := uriPathToURL(c.defaultURI, "/slices/max") + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + var rsp getSlicesMaxResponse + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + + return rsp.Standard, nil +} + +// Schema returns all index and field schema information. +func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) { + // Execute request against the host. + u := c.defaultURI.Path("/schema") + + // Build request. + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + var rsp getSchemaResponse + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + return rsp.Indexes, nil +} + +// CreateIndex creates a new index on the server. +func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt pilosa.IndexOptions) error { + // Encode query request. + buf, err := json.Marshal(&postIndexRequest{ + Options: opt, + }) + if err != nil { + return errors.Wrap(err, "encoding request") + } + + // Create URL & HTTP request. + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index)) + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } + + // Handle response based on status code. + switch resp.StatusCode { + case http.StatusOK: + return nil // ok + case http.StatusConflict: + return pilosa.ErrIndexExists + default: + return errors.New(string(body)) + } +} + +// FragmentNodes returns a list of nodes that own a slice. +func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*pilosa.Node, error) { + // Execute request against the host. + u := uriPathToURL(c.defaultURI, "/fragment/nodes") + u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode() + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + var a []*pilosa.Node + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&a); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + + return a, nil +} + +// Query executes query against the index. +func (c *InternalHTTPClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return c.QueryNode(ctx, c.defaultURI, index, queryRequest) +} + +// QueryNode executes query against the index, sending the request to the node specified. +func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + if index == "" { + return nil, pilosa.ErrIndexRequired + } else if queryRequest.Query == "" { + return nil, pilosa.ErrQueryRequired + } + + // Encode request object. + buf, err := proto.Marshal(queryRequest) + if err != nil { + return nil, errors.Wrap(err, "marshaling") + } + + // Create HTTP request. + u := uri.Path(fmt.Sprintf("/index/%s/query", index)) + req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } else if resp.StatusCode != http.StatusOK { + return nil, errors.New(string(body)) + } + + qresp := &internal.QueryResponse{} + if err := proto.Unmarshal(body, qresp); err != nil { + return nil, fmt.Errorf("unmarshal response: %s", err) + } else if s := qresp.Err; s != "" { + return nil, errors.New(s) + } + + return qresp, nil +} + +// Import bulk imports bits for a single slice to a host. +func (c *InternalHTTPClient) Import(ctx context.Context, index, field string, slice uint64, bits []pilosa.Bit) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + buf, err := marshalImportPayload(index, field, slice, bits) + if err != nil { + return fmt.Errorf("Error Creating Payload: %s", err) + } + + // Retrieve a list of nodes that own the slice. + nodes, err := c.FragmentNodes(ctx, index, slice) + if err != nil { + return fmt.Errorf("slice nodes: %s", err) + } + + // Import to each node. + for _, node := range nodes { + if err := c.importNode(ctx, node, buf); err != nil { + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) + } + } + + return nil +} + +// ImportK bulk imports bits specified by string keys to a host. +func (c *InternalHTTPClient) ImportK(ctx context.Context, index, field string, columns []pilosa.Bit) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + buf, err := marshalImportPayloadK(index, field, columns) + if err != nil { + return fmt.Errorf("Error Creating Payload: %s", err) + } + + node := &pilosa.Node{ + URI: *c.defaultURI, + } + + // Import to node. + if err := c.importNode(ctx, node, buf); err != nil { + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) + } + + return nil +} + +func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, options pilosa.IndexOptions) error { + err := c.CreateIndex(ctx, name, options) + if err == nil || err == pilosa.ErrIndexExists { + return nil + } + return err +} + +func (c *InternalHTTPClient) EnsureField(ctx context.Context, indexName string, fieldName string, options pilosa.FieldOptions) error { + err := c.CreateField(ctx, indexName, fieldName, options) + if err == nil || err == pilosa.ErrFieldExists { + return nil + } + return err +} + +// marshalImportPayload marshalls the import parameters into a protobuf byte slice. +func marshalImportPayload(index, field string, slice uint64, bits []pilosa.Bit) ([]byte, error) { + // Separate row and column IDs to reduce allocations. + rowIDs := Bits(bits).RowIDs() + columnIDs := Bits(bits).ColumnIDs() + timestamps := Bits(bits).Timestamps() + + // Marshal data to protobuf. + buf, err := proto.Marshal(&internal.ImportRequest{ + Index: index, + Field: field, + Slice: slice, + RowIDs: rowIDs, + ColumnIDs: columnIDs, + Timestamps: timestamps, + }) + if err != nil { + return nil, fmt.Errorf("marshal import request: %s", err) + } + return buf, nil +} + +// marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. +func marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) { + // Separate row and column IDs to reduce allocations. + rowKeys := Bits(bits).RowKeys() + columnKeys := Bits(bits).ColumnKeys() + timestamps := Bits(bits).Timestamps() + + // Marshal data to protobuf. + buf, err := proto.Marshal(&internal.ImportRequest{ + Index: index, + Field: field, + RowKeys: rowKeys, + ColumnKeys: columnKeys, + Timestamps: timestamps, + }) + if err != nil { + return nil, fmt.Errorf("marshal import request: %s", err) + } + return buf, nil +} + +// importNode sends a pre-marshaled import request to a node. +func (c *InternalHTTPClient) importNode(ctx context.Context, node *pilosa.Node, buf []byte) error { + // Create URL & HTTP request. + u := nodePathToURL(node, "/import") + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } else if resp.StatusCode != http.StatusOK { + return errors.New(string(body)) + } + + var isresp internal.ImportResponse + if err := proto.Unmarshal(body, &isresp); err != nil { + return fmt.Errorf("unmarshal import response: %s", err) + } else if s := isresp.Err; s != "" { + return errors.New(s) + } + + return nil +} + +// ImportValue bulk imports field values for a single slice to a host. +func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []pilosa.FieldValue) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + buf, err := marshalImportValuePayload(index, field, slice, vals) + if err != nil { + return fmt.Errorf("Error Creating Payload: %s", err) + } + + // Retrieve a list of nodes that own the slice. + nodes, err := c.FragmentNodes(ctx, index, slice) + if err != nil { + return fmt.Errorf("slice nodes: %s", err) + } + + // Import to each node. + for _, node := range nodes { + if err := c.importValueNode(ctx, node, buf); err != nil { + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) + } + } + + return nil +} + +// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. +func marshalImportValuePayload(index, field string, slice uint64, vals []pilosa.FieldValue) ([]byte, error) { + // Separate row and column IDs to reduce allocations. + columnIDs := FieldValues(vals).ColumnIDs() + values := FieldValues(vals).Values() + + // Marshal data to protobuf. + buf, err := proto.Marshal(&internal.ImportValueRequest{ + Index: index, + Field: field, + Slice: slice, + ColumnIDs: columnIDs, + Values: values, + }) + if err != nil { + return nil, fmt.Errorf("marshal import request: %s", err) + } + return buf, nil +} + +// importValueNode sends a pre-marshaled import request to a node. +func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *pilosa.Node, buf []byte) error { + // Create URL & HTTP request. + u := nodePathToURL(node, "/import-value") + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } else if resp.StatusCode != http.StatusOK { + return errors.New(string(body)) + } + + var isresp internal.ImportResponse + if err := proto.Unmarshal(body, &isresp); err != nil { + return fmt.Errorf("unmarshal import response: %s", err) + } else if s := isresp.Err; s != "" { + return errors.New(s) + } + + return nil +} + +// ExportCSV bulk exports data for a single slice from a host to CSV format. +func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + // Retrieve a list of nodes that own the slice. + nodes, err := c.FragmentNodes(ctx, index, slice) + if err != nil { + return fmt.Errorf("slice nodes: %s", err) + } + + // Attempt nodes in random order. + var e error + for _, i := range rand.Perm(len(nodes)) { + node := nodes[i] + + if err := c.exportNodeCSV(ctx, node, index, field, slice, w); err != nil { + e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) + continue + } else { + return nil + } + } + + return e +} + +// exportNode copies a CSV export from a node to w. +func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, slice uint64, w io.Writer) error { + // Create URL. + u := nodePathToURL(node, "/export") + u.RawQuery = url.Values{ + "index": {index}, + "field": {field}, + "slice": {strconv.FormatUint(slice, 10)}, + }.Encode() + + // Generate HTTP request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Accept", "text/csv") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Validate status code. + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("invalid status: %d", resp.StatusCode) + } + + // Copy body to writer. + if _, err := io.Copy(w, resp.Body); err != nil { + return errors.Wrap(err, "copying") + } + + return nil +} + +func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri pilosa.URI) (io.ReadCloser, error) { + node := &pilosa.Node{ + URI: uri, + } + return c.backupSliceNode(ctx, index, field, slice, node) +} + +func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, field string, slice uint64, node *pilosa.Node) (io.ReadCloser, error) { + u := nodePathToURL(node, "/fragment/data") + u.RawQuery = url.Values{ + "index": {index}, + "field": {field}, + "slice": {strconv.FormatUint(slice, 10)}, + }.Encode() + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + + // Return error if status is not OK. + if resp.StatusCode == http.StatusNotFound { + resp.Body.Close() + return nil, pilosa.ErrFragmentNotFound + } else if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode) + } + + return resp.Body, nil +} + +// CreateField creates a new field on the server. +func (c *InternalHTTPClient) CreateField(ctx context.Context, index, field string, opt pilosa.FieldOptions) error { + if index == "" { + return pilosa.ErrIndexRequired + } + + // Encode query request. + buf, err := json.Marshal(&postFieldRequest{ + Options: opt, + }) + if err != nil { + return errors.Wrap(err, "marshaling") + } + + // Create URL & HTTP request. + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/field/%s", index, field)) + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } + + // Handle response based on status code. + switch resp.StatusCode { + case http.StatusOK: + return nil // ok + case http.StatusConflict: + return pilosa.ErrFieldExists + default: + return errors.New(string(body)) + } +} + +// FragmentBlocks returns a list of block checksums for a fragment on a host. +// Only returns blocks which contain data. +func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64) ([]pilosa.FragmentBlock, error) { + if uri == nil { + uri = c.defaultURI + } + u := uriPathToURL(uri, "/fragment/blocks") + u.RawQuery = url.Values{ + "index": {index}, + "field": {field}, + "slice": {strconv.FormatUint(slice, 10)}, + }.Encode() + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + case http.StatusNotFound: + return nil, pilosa.ErrFragmentNotFound + default: + return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp getFragmentBlocksResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, errors.Wrap(err, "decoding") + } + return rsp.Blocks, nil +} + +// BlockData returns row/column id pairs for a block. +func (c *InternalHTTPClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { + buf, err := proto.Marshal(&internal.BlockDataRequest{ + Index: index, + Field: field, + Slice: slice, + Block: uint64(block), + }) + if err != nil { + return nil, nil, errors.Wrap(err, "marshaling") + } + + u := uriPathToURL(c.defaultURI, "/fragment/block/data") + req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) + if err != nil { + return nil, nil, errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Type", "application/protobuf") + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Accept", "application/protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // fallthrough + case http.StatusNotFound: + return nil, nil, nil + default: + return nil, nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp internal.BlockDataResponse + if body, err := ioutil.ReadAll(resp.Body); err != nil { + return nil, nil, errors.Wrap(err, "reading") + } else if err := proto.Unmarshal(body, &rsp); err != nil { + return nil, nil, errors.Wrap(err, "unmarshalling") + } + return rsp.RowIDs, rsp.ColumnIDs, nil +} + +// ColumnAttrDiff returns data from differing blocks on a remote host. +func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { + if uri == nil { + uri = c.defaultURI + } + u := uriPathToURL(uri, fmt.Sprintf("/index/%s/attr/diff", index)) + + // Encode request. + buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks}) + if err != nil { + return nil, errors.Wrap(err, "marshaling") + } + + // Build request. + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + default: + return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp postIndexAttrDiffResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, errors.Wrap(err, "decoding") + } + return rsp.Attrs, nil +} + +// RowAttrDiff returns data from differing blocks on a remote host. +func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { + if uri == nil { + uri = c.defaultURI + } + u := uriPathToURL(uri, fmt.Sprintf("/index/%s/field/%s/attr/diff", index, field)) + + // Encode request. + buf, err := json.Marshal(postFieldAttrDiffRequest{Blocks: blks}) + if err != nil { + return nil, errors.Wrap(err, "marshaling") + } + + // Build request. + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + case http.StatusNotFound: + return nil, pilosa.ErrFieldNotFound + default: + return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp postFieldAttrDiffResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, errors.Wrap(err, "decoding") + } + return rsp.Attrs, nil +} + +// SendMessage posts a message synchronously. +func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb proto.Message) error { + msg, err := pilosa.MarshalMessage(pb) + if err != nil { + return fmt.Errorf("marshaling message: %v", err) + } + + u := uriPathToURL(uri, "/cluster/message") + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) + if err != nil { + return errors.Wrap(err, "making new request") + } + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return fmt.Errorf("executing http request: %v", err) + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response body: %v", err) + } + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + default: + return fmt.Errorf("unexpected response status code: %d: %s", resp.StatusCode, body) + } + + return nil +} + +// Bits is a slice of Bit. +type Bits []pilosa.Bit + +func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p Bits) Len() int { return len(p) } + +func (p Bits) Less(i, j int) bool { + if p[i].RowID == p[j].RowID { + if p[i].ColumnID < p[j].ColumnID { + return p[i].Timestamp < p[j].Timestamp + } + return p[i].ColumnID < p[j].ColumnID + } + return p[i].RowID < p[j].RowID +} + +// RowIDs returns a slice of all the row IDs. +func (p Bits) RowIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].RowID + } + return other +} + +// ColumnIDs returns a slice of all the column IDs. +func (p Bits) ColumnIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].ColumnID + } + return other +} + +// RowKeys returns a slice of all the row keys. +func (p Bits) RowKeys() []string { + other := make([]string, len(p)) + for i := range p { + other[i] = p[i].RowKey + } + return other +} + +// ColumnKeys returns a slice of all the column keys. +func (p Bits) ColumnKeys() []string { + other := make([]string, len(p)) + for i := range p { + other[i] = p[i].ColumnKey + } + return other +} + +// Timestamps returns a slice of all the timestamps. +func (p Bits) Timestamps() []int64 { + other := make([]int64, len(p)) + for i := range p { + other[i] = p[i].Timestamp + } + return other +} + +// GroupBySlice returns a map of bits by slice. +func (p Bits) GroupBySlice() map[uint64][]pilosa.Bit { + m := make(map[uint64][]pilosa.Bit) + for _, bit := range p { + slice := bit.ColumnID / pilosa.SliceWidth + m[slice] = append(m[slice], bit) + } + + for slice, bits := range m { + sort.Sort(Bits(bits)) + m[slice] = bits + } + + return m +} + +// FieldValues represents a slice of field values. +type FieldValues []pilosa.FieldValue + +func (p FieldValues) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p FieldValues) Len() int { return len(p) } + +func (p FieldValues) Less(i, j int) bool { + return p[i].ColumnID < p[j].ColumnID +} + +// ColumnIDs returns a slice of all the column IDs. +func (p FieldValues) ColumnIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].ColumnID + } + return other +} + +// Values returns a slice of all the values. +func (p FieldValues) Values() []int64 { + other := make([]int64, len(p)) + for i := range p { + other[i] = p[i].Value + } + return other +} + +// GroupBySlice returns a map of field values by slice. +func (p FieldValues) GroupBySlice() map[uint64][]pilosa.FieldValue { + m := make(map[uint64][]pilosa.FieldValue) + for _, val := range p { + slice := val.ColumnID / pilosa.SliceWidth + m[slice] = append(m[slice], val) + } + + for slice, vals := range m { + sort.Sort(FieldValues(vals)) + m[slice] = vals + } + + return m +} + +// BitsByPos is a slice of bits sorted row then column. +type BitsByPos []pilosa.Bit + +func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p BitsByPos) Len() int { return len(p) } +func (p BitsByPos) Less(i, j int) bool { + p0, p1 := pilosa.Pos(p[i].RowID, p[i].ColumnID), pilosa.Pos(p[j].RowID, p[j].ColumnID) + if p0 == p1 { + return p[i].Timestamp < p[j].Timestamp + } + return p0 < p1 +} + +func uriPathToURL(uri *pilosa.URI, path string) url.URL { + return url.URL{ + Scheme: uri.Scheme(), + Host: uri.HostPort(), + Path: path, + } +} + +func nodePathToURL(node *pilosa.Node, path string) url.URL { + return url.URL{ + Scheme: node.URI.Scheme(), + Host: node.URI.HostPort(), + Path: path, + } +} diff --git a/client_test.go b/http/client_test.go similarity index 93% rename from client_test.go rename to http/client_test.go index d2e194805..93b16a177 100644 --- a/client_test.go +++ b/http/client_test.go @@ -12,20 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package http_test import ( "context" "fmt" - "net/http" + gohttp "net/http" "reflect" "testing" "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -43,10 +43,10 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { return server, hldr } -var defaultClient *http.Client +var defaultClient *gohttp.Client func init() { - defaultClient = server.GetHTTPClient(nil) + defaultClient = http.GetHTTPClient(nil) } @@ -61,21 +61,24 @@ func TestClient_MultiNode(t *testing.T) { } s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(defaultClient) + httpClient := http.NewInternalHTTPClientFromURI(&cluster.Nodes[0].URI, defaultClient) + e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) e.Holder = hldr[0].Holder e.Node = cluster.Nodes[0] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(defaultClient) + httpClient := http.NewInternalHTTPClientFromURI(&cluster.Nodes[0].URI, defaultClient) + e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) e.Holder = hldr[1].Holder e.Node = cluster.Nodes[1] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(defaultClient) + httpClient := http.NewInternalHTTPClientFromURI(&cluster.Nodes[0].URI, defaultClient) + e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) e.Holder = hldr[2].Holder e.Node = cluster.Nodes[2] e.Cluster = cluster @@ -98,9 +101,9 @@ func TestClient_MultiNode(t *testing.T) { } } - baseBit0 := SliceWidth * sliceNums[0] - baseBit1 := SliceWidth * sliceNums[1] - baseBit2 := SliceWidth * sliceNums[2] + baseBit0 := pilosa.SliceWidth * sliceNums[0] + baseBit1 := pilosa.SliceWidth * sliceNums[1] + baseBit2 := pilosa.SliceWidth * sliceNums[2] maxSlice := uint64(0) for _, x := range sliceNums { @@ -336,7 +339,7 @@ func TestClient_FragmentBlocks(t *testing.T) { // Retrieve blocks. c := test.MustNewClient(s.Host(), defaultClient) - blocks, err := c.FragmentBlocks(context.Background(), "i", "f", 0) + blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0) if err != nil { t.Fatal(err) } else if len(blocks) != 2 { diff --git a/http/handler.go b/http/handler.go new file mode 100644 index 000000000..df5108826 --- /dev/null +++ b/http/handler.go @@ -0,0 +1,1231 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "crypto/tls" + "encoding/json" + "expvar" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + // Imported for its side-effect of registering pprof endpoints with the server. + _ "net/http/pprof" + "reflect" + "runtime/debug" + "strconv" + "strings" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/gorilla/handlers" + "github.com/gorilla/mux" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + + "github.com/pkg/errors" +) + +// Handler represents an HTTP handler. +type Handler struct { + Handler http.Handler + + Logger pilosa.Logger + + // Keeps the query argument validators for each handler + validators map[string]*queryValidationSpec + + API *pilosa.API + + AllowedOrigins []string +} + +// externalPrefixFlag denotes endpoints that are intended to be exposed to clients. +// This is used for stats tagging. +var externalPrefixFlag = map[string]bool{ + "schema": true, + "query": true, + "import": true, + "export": true, + "index": true, + "field": true, + "nodes": true, + "version": true, +} + +type errorResponse struct { + Error string `json:"error"` +} + +// HandlerOption is a functional option type for pilosa.Handler +type HandlerOption func(s *Handler) error + +func OptHandlerAllowedOrigins(origins []string) HandlerOption { + return func(h *Handler) error { + h.Handler = handlers.CORS( + handlers.AllowedOrigins(origins), + handlers.AllowedHeaders([]string{"Content-Type"}), + )(h.Handler) + return nil + } +} + +func OptHandlerAPI(api *pilosa.API) HandlerOption { + return func(h *Handler) error { + h.API = api + return nil + } +} + +func OptHandlerLogger(logger pilosa.Logger) HandlerOption { + return func(h *Handler) error { + h.Logger = logger + return nil + } +} + +// NewHandler returns a new instance of Handler with a default logger. +func NewHandler(opts ...HandlerOption) (*Handler, error) { + handler := &Handler{ + Logger: pilosa.NopLogger, + } + handler.Handler = NewRouter(handler) + handler.populateValidators() + + for _, opt := range opts { + err := opt(handler) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + + return handler, nil +} + +func (h *Handler) populateValidators() { + h.validators = map[string]*queryValidationSpec{} + h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") + h.validators["GetSliceMax"] = queryValidationSpecRequired() + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") + h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "slice") +} + +func (h *Handler) queryArgValidator(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := mux.CurrentRoute(r).GetName() + if validator, ok := h.validators[key]; ok { + if err := validator.validate(r.URL.Query()); err != nil { + // TODO: Return the response depending on the Accept header + response := errorResponse{Error: err.Error()} + body, err := json.Marshal(response) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + http.Error(w, string(body), http.StatusBadRequest) + return + } + } + next.ServeHTTP(w, r) + }) +} + +// NewRouter creates a new mux http router. +func NewRouter(handler *Handler) *mux.Router { + router := mux.NewRouter() + router.HandleFunc("/", handler.handleHome).Methods("GET") + router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") + router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") + router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") + router.Handle("/debug/vars", expvar.Handler()).Methods("GET") + router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") + router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client + router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") + router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") + router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") + + router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") + + router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") + router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") + router.Handle("/debug/vars", expvar.Handler()).Methods("GET") + router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") + router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") + router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") + router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") + router.HandleFunc("/import", handler.handlePostImport).Methods("POST") + router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST") + router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") + router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") + router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") + router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") + router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") + //router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented. + router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") + router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") + router.HandleFunc("/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST") + router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") + router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") + + // TODO: Apply MethodNotAllowed statuses to all endpoints. + // Ideally this would be automatic, as described in this (wontfix) ticket: + // https://github.com/gorilla/mux/issues/6 + // For now we just do it for the most commonly used handler, /query + router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") + + router.Use(handler.queryArgValidator) + return router +} + +func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) +} + +// ServeHTTP handles an HTTP request. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + stack := debug.Stack() + msg := "PANIC: %s\n%s" + h.Logger.Printf(msg, err, stack) + fmt.Fprintf(w, msg, err, stack) + } + }() + + t := time.Now() + h.Handler.ServeHTTP(w, r) + dif := time.Since(t) + + // Calculate per request StatsD metrics when the handler is fully configured. + statsTags := make([]string, 0, 3) + + longQueryTime := h.API.LongQueryTime() + if longQueryTime > 0 && dif > longQueryTime { + h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) + statsTags = append(statsTags, "slow_query") + } + + pathParts := strings.Split(r.URL.Path, "/") + endpointName := strings.Join(pathParts, "_") + + if externalPrefixFlag[pathParts[1]] { + statsTags = append(statsTags, "external") + } + + // useragent tag identifies internal/external endpoints + statsTags = append(statsTags, "useragent:"+r.UserAgent()) + stats := h.API.StatsWithTags(statsTags) + if stats != nil { + stats.Histogram("http."+endpointName, float64(dif), 0.1) + } +} + +func (h *Handler) handleHome(w http.ResponseWriter, r *http.Request) { + http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.", http.StatusNotFound) +} + +// handleGetSchema handles GET /schema requests. +func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { + schema := h.API.Schema(r.Context()) + if err := json.NewEncoder(w).Encode(getSchemaResponse{ + Indexes: schema, + }); err != nil { + h.Logger.Printf("write schema response error: %s", err) + } +} + +// handleGetStatus handles GET /status requests. +func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { + status := getStatusResponse{ + State: h.API.State(), + Nodes: h.API.Hosts(r.Context()), + LocalID: h.API.LocalID(), + } + if err := json.NewEncoder(w).Encode(status); err != nil { + h.Logger.Printf("write status response error: %s", err) + } +} + +func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { + info := h.API.Info() + if err := json.NewEncoder(w).Encode(info); err != nil { + h.Logger.Printf("write info response error: %s", err) + } +} + +type getSchemaResponse struct { + Indexes []*pilosa.IndexInfo `json:"indexes"` +} + +type getStatusResponse struct { + State string `json:"state"` + Nodes []*pilosa.Node `json:"nodes"` + LocalID string `json:"localID"` +} + +// handlePostQuery handles /query requests. +func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { + // Parse incoming request. + req, err := h.readQueryRequest(r) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + return + } + // TODO: Remove + req.Index = mux.Vars(r)["index"] + + resp, err := h.API.Query(r.Context(), req) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + return + } + + // Set appropriate status code, if there is an error. + if resp.Err != nil { + switch resp.Err { + case pilosa.ErrTooManyWrites: + w.WriteHeader(http.StatusRequestEntityTooLarge) + default: + w.WriteHeader(http.StatusInternalServerError) + } + } + + // Write response back to client. + if err := h.writeQueryResponse(w, r, &resp); err != nil { + h.Logger.Printf("write query response error: %s", err) + } +} + +// handleGetSlicesMax handles GET /schema requests. +func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { + if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ + Standard: h.API.MaxSlices(r.Context()), + }); err != nil { + h.Logger.Printf("write slices-max response error: %s", err) + } +} + +type getSlicesMaxResponse struct { + Standard map[string]uint64 `json:"standard"` +} + +// handleGetIndexes handles GET /index request. +func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { + h.handleGetSchema(w, r) +} + +// handleGetIndex handles GET /index/ requests. +func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + index, err := h.API.Index(r.Context(), indexName) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + if err := json.NewEncoder(w).Encode(getIndexResponse{ + map[string]string{"name": index.Name()}, + }); err != nil { + h.Logger.Printf("write response error: %s", err) + } +} + +type getIndexResponse struct { + Index map[string]string `json:"index"` +} + +type postIndexRequest struct { + Options pilosa.IndexOptions `json:"options"` +} + +//_postIndexRequest is necessary to avoid recursion while decoding. +type _postIndexRequest postIndexRequest + +// Custom Unmarshal JSON to validate request body when creating a new index. +func (p *postIndexRequest) UnmarshalJSON(b []byte) error { + + // m is an overflow map used to capture additional, unexpected keys. + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return errors.Wrap(err, "unmarshalling unexpected values") + } + + validIndexOptions := getValidOptions(pilosa.IndexOptions{}) + err := validateOptions(m, validIndexOptions) + if err != nil { + return err + } + // Unmarshal expected values. + var _p _postIndexRequest + if err := json.Unmarshal(b, &_p); err != nil { + return errors.Wrap(err, "unmarshalling expected values") + } + + p.Options = _p.Options + + return nil +} + +// Raise errors for any unknown key +func validateOptions(data map[string]interface{}, validIndexOptions []string) error { + for k, v := range data { + switch k { + case "options": + options, ok := v.(map[string]interface{}) + if !ok { + return errors.New("options is not map[string]interface{}") + } + for kk, vv := range options { + if !foundItem(validIndexOptions, kk) { + return fmt.Errorf("Unknown key: %v:%v", kk, vv) + } + } + default: + return fmt.Errorf("Unknown key: %v:%v", k, v) + } + } + return nil +} + +func foundItem(items []string, item string) bool { + for _, i := range items { + if item == i { + return true + } + } + return false +} + +type postIndexResponse struct{} + +// handleDeleteIndex handles DELETE /index request. +func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + err := h.API.DeleteIndex(r.Context(), indexName) + if err != nil { + h.Logger.Printf("problem deleting index: %s", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type deleteIndexResponse struct{} + +// handlePostIndex handles POST /index request. +func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + + // Decode request. + var req postIndexRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err == io.EOF { + // If no data was provided (EOF), we still create the index + // with default values. + } else if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + _, err = h.API.CreateIndex(r.Context(), indexName, req.Options) + if errors.Cause(err) == pilosa.ErrIndexExists { + http.Error(w, err.Error(), http.StatusConflict) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +// handlePostIndexAttrDiff handles POST /index/attr/diff requests. +func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + + // Decode request. + var req postIndexAttrDiffRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + attrs, err := h.API.IndexAttrDiff(r.Context(), indexName, req.Blocks) + if err != nil { + if errors.Cause(err) == pilosa.ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{ + Attrs: attrs, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type postIndexAttrDiffRequest struct { + Blocks []pilosa.AttrBlock `json:"blocks"` +} + +type postIndexAttrDiffResponse struct { + Attrs map[uint64]map[string]interface{} `json:"attrs"` +} + +// handlePostField handles POST /field request. +func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + + // Decode request. + var req postFieldRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err == io.EOF { + // If no data was provided (EOF), we still create the field + // with default values. + } else if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + _, err = h.API.CreateField(r.Context(), indexName, fieldName, req.Options) + if err != nil { + switch errors.Cause(err) { + case pilosa.ErrIndexNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case pilosa.ErrFieldExists: + http.Error(w, err.Error(), http.StatusConflict) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + // Encode response. + if err := json.NewEncoder(w).Encode(postFieldResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type _postFieldRequest postFieldRequest + +// Custom Unmarshal JSON to validate request body when creating a new field. If there's new FieldOptions, +// adding it to validFieldOptions to make sure the new option is validated, otherwise the request will be failed +func (p *postFieldRequest) UnmarshalJSON(b []byte) error { + // m is an overflow map used to capture additional, unexpected keys. + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return errors.Wrap(err, "unmarshaling unexpected keys") + } + + validFieldOptions := getValidOptions(pilosa.FieldOptions{}) + err := validateOptions(m, validFieldOptions) + if err != nil { + return err + } + + // Unmarshal expected values. + var _p _postFieldRequest + if err := json.Unmarshal(b, &_p); err != nil { + return errors.Wrap(err, "unmarshalling expected keys") + } + + p.Options = _p.Options + return nil + +} + +func getValidOptions(option interface{}) []string { + validOptions := []string{} + val := reflect.ValueOf(option) + for i := 0; i < val.Type().NumField(); i++ { + jsonTag := val.Type().Field(i).Tag.Get("json") + s := strings.Split(jsonTag, ",") + validOptions = append(validOptions, s[0]) + } + return validOptions +} + +type postFieldRequest struct { + Options pilosa.FieldOptions `json:"options"` +} + +type postFieldResponse struct{} + +// handleDeleteField handles DELETE /field request. +func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + + err := h.API.DeleteField(r.Context(), indexName, fieldName) + if err != nil { + if errors.Cause(err) == pilosa.ErrIndexNotFound { + if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } + return + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(deleteFieldResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type deleteFieldResponse struct{} + +// handlePostFieldAttrDiff handles POST /field/attr/diff requests. +func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + + // Decode request. + var req postFieldAttrDiffRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + attrs, err := h.API.FieldAttrDiff(r.Context(), indexName, fieldName, req.Blocks) + if err != nil { + switch errors.Cause(err) { + case pilosa.ErrFragmentNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(postFieldAttrDiffResponse{ + Attrs: attrs, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type postFieldAttrDiffRequest struct { + Blocks []pilosa.AttrBlock `json:"blocks"` +} + +type postFieldAttrDiffResponse struct { + Attrs map[uint64]map[string]interface{} `json:"attrs"` +} + +// readQueryRequest parses an query parameters from r. +func (h *Handler) readQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { + switch r.Header.Get("Content-Type") { + case "application/x-protobuf": + return h.readProtobufQueryRequest(r) + default: + return h.readURLQueryRequest(r) + } +} + +// readProtobufQueryRequest parses query parameters in protobuf from r. +func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { + // Slurp the body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } + + // Unmarshal into object. + var req internal.QueryRequest + if err := proto.Unmarshal(body, &req); err != nil { + return nil, errors.Wrap(err, "unmarshalling") + } + + return decodeQueryRequest(&req), nil +} + +// readURLQueryRequest parses query parameters from URL parameters from r. +func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { + q := r.URL.Query() + + // Parse query string. + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } + query := string(buf) + + // Parse list of slices. + slices, err := parseUint64Slice(q.Get("slices")) + if err != nil { + return nil, errors.New("invalid slice argument") + } + + return &pilosa.QueryRequest{ + Query: query, + Slices: slices, + ColumnAttrs: q.Get("columnAttrs") == "true", + ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true", + ExcludeColumns: q.Get("excludeColumns") == "true", + }, nil +} + +// writeQueryResponse writes the response from the executor to w. +func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *pilosa.QueryResponse) error { + if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { + return h.writeProtobufQueryResponse(w, resp) + } + return h.writeJSONQueryResponse(w, resp) +} + +// writeProtobufQueryResponse writes the response from the executor to w as protobuf. +func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error { + if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil { + return errors.Wrap(err, "marshalling") + } else if _, err := w.Write(buf); err != nil { + return errors.Wrap(err, "writing") + } + return nil +} + +// writeJSONQueryResponse writes the response from the executor to w as JSON. +func (h *Handler) writeJSONQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error { + return json.NewEncoder(w).Encode(resp) +} + +// handlePostImport handles /import requests. +func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } else if r.Header.Get("Accept") != "application/x-protobuf" { + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + return + } + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Marshal into request object. + var req internal.ImportRequest + if err := proto.Unmarshal(body, &req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.API.Import(r.Context(), req); err != nil { + switch errors.Cause(err) { + case pilosa.ErrIndexNotFound: + fallthrough + case pilosa.ErrFieldNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case pilosa.ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Marshal response object. + buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) + if e != nil { + http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) + return + } + + // Write response. + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + } + w.Write(buf) +} + +// handlePostImportValue handles /import-value requests. +func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } else if r.Header.Get("Accept") != "application/x-protobuf" { + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + return + } + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Marshal into request object. + var req internal.ImportValueRequest + if err := proto.Unmarshal(body, &req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err = h.API.ImportValue(r.Context(), req); err != nil { + switch errors.Cause(err) { + case pilosa.ErrIndexNotFound: + fallthrough + case pilosa.ErrFieldNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case pilosa.ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Marshal response object. + buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) + if e != nil { + http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) + return + } + + // Write response. + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + } + w.Write(buf) +} + +// handleGetExport handles /export requests. +func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { + switch r.Header.Get("Accept") { + case "text/csv": + h.handleGetExportCSV(w, r) + default: + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + } +} + +func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { + // Parse query parameters. + q := r.URL.Query() + index, field := q.Get("index"), q.Get("field") + + slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + if err != nil { + http.Error(w, "invalid slice", http.StatusBadRequest) + return + } + + if err = h.API.ExportCSV(r.Context(), index, field, slice, w); err != nil { + switch errors.Cause(err) { + case pilosa.ErrFragmentNotFound: + break + case pilosa.ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } +} + +// handleGetFragmentNodes handles /fragment/nodes requests. +func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + index := q.Get("index") + + // Read slice parameter. + slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + if err != nil { + http.Error(w, "slice should be an unsigned integer", http.StatusBadRequest) + return + } + + // Retrieve fragment owner nodes. + nodes, err := h.API.SliceNodes(r.Context(), index, slice) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Write to response. + if err := json.NewEncoder(w).Encode(nodes); err != nil { + h.Logger.Printf("json write error: %s", err) + } +} + +// handleGetFragmentBlockData handles GET /fragment/block/data requests. +func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { + buf, err := h.API.FragmentBlockData(r.Context(), r.Body) + if err != nil { + if _, ok := err.(pilosa.BadRequestError); ok { + http.Error(w, err.Error(), http.StatusBadRequest) + } else if errors.Cause(err) == pilosa.ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Write response. + w.Header().Set("Content-Type", "application/protobuf") + w.Header().Set("Content-Length", strconv.Itoa(len(buf))) + w.Write(buf) +} + +// handleGetFragmentBlocks handles GET /fragment/blocks requests. +func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request) { + // Read slice parameter. + q := r.URL.Query() + slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + if err != nil { + http.Error(w, "slice required", http.StatusBadRequest) + return + } + + blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), slice) + if err != nil { + if errors.Cause(err) == pilosa.ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ + Blocks: blocks, + }); err != nil { + h.Logger.Printf("block response encoding error: %s", err) + } +} + +type getFragmentBlocksResponse struct { + Blocks []pilosa.FragmentBlock `json:"blocks"` +} + +// handleGetVersion handles /version requests. +func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { + err := json.NewEncoder(w).Encode(struct { + Version string `json:"version"` + }{ + Version: h.API.Version(), + }) + if err != nil { + h.Logger.Printf("write version response error: %s", err) + } +} + +// QueryResult types. +const ( + QueryResultTypeNil uint32 = iota + QueryResultTypeRow + QueryResultTypePairs + QueryResultTypeValCount + QueryResultTypeUint64 + QueryResultTypeBool +) + +func decodeQueryRequest(pb *internal.QueryRequest) *pilosa.QueryRequest { + req := &pilosa.QueryRequest{ + Query: pb.Query, + Slices: pb.Slices, + ColumnAttrs: pb.ColumnAttrs, + Remote: pb.Remote, + ExcludeRowAttrs: pb.ExcludeRowAttrs, + ExcludeColumns: pb.ExcludeColumns, + } + + return req +} + +func encodeQueryResponse(resp *pilosa.QueryResponse) *internal.QueryResponse { + pb := &internal.QueryResponse{ + Results: make([]*internal.QueryResult, len(resp.Results)), + ColumnAttrSets: pilosa.EncodeColumnAttrSets(resp.ColumnAttrSets), + } + + for i := range resp.Results { + pb.Results[i] = &internal.QueryResult{} + + switch result := resp.Results[i].(type) { + case *pilosa.Row: + pb.Results[i].Type = QueryResultTypeRow + pb.Results[i].Row = pilosa.EncodeRow(result) + case []pilosa.Pair: + pb.Results[i].Type = QueryResultTypePairs + pb.Results[i].Pairs = pilosa.EncodePairs(result) + case pilosa.ValCount: + pb.Results[i].Type = QueryResultTypeValCount + pb.Results[i].ValCount = pilosa.EncodeValCount(result) + case uint64: + pb.Results[i].Type = QueryResultTypeUint64 + pb.Results[i].N = result + case bool: + pb.Results[i].Type = QueryResultTypeBool + pb.Results[i].Changed = result + case nil: + pb.Results[i].Type = QueryResultTypeNil + } + } + + if resp.Err != nil { + pb.Err = resp.Err.Error() + } + + return pb +} + +// parseUint64Slice returns a slice of uint64s from a comma-delimited string. +func parseUint64Slice(s string) ([]uint64, error) { + var a []uint64 + for _, str := range strings.Split(s, ",") { + // Ignore blanks. + if str == "" { + continue + } + + // Parse number. + num, err := strconv.ParseUint(str, 10, 64) + if err != nil { + return nil, errors.Wrap(err, "parsing int") + } + a = append(a, num) + } + return a, nil +} + +// errorString returns the string representation of err. +func errorString(err error) string { + if err == nil { + return "" + } + return err.Error() +} + +func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req setCoordinatorRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, "decoding request "+err.Error(), http.StatusBadRequest) + return + } + + oldNode, newNode, err := h.API.SetCoordinator(r.Context(), req.ID) + if err != nil { + if errors.Cause(err) == pilosa.ErrNodeIDNotExists { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusInternalServerError) + } + return + } + // Encode response. + if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ + Old: oldNode, + New: newNode, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type setCoordinatorRequest struct { + ID string `json:"id"` +} + +type setCoordinatorResponse struct { + Old *pilosa.Node `json:"old"` + New *pilosa.Node `json:"new"` +} + +// handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request. +func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req removeNodeRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + removeNode, err := h.API.RemoveNode(req.ID) + if err != nil { + if errors.Cause(err) == pilosa.ErrNodeIDNotExists { + http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "removing node: "+err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(removeNodeResponse{ + Remove: removeNode, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type removeNodeRequest struct { + ID string `json:"id"` +} + +type removeNodeResponse struct { + Remove *pilosa.Node `json:"remove"` +} + +// handlePostClusterResizeAbort handles POST /cluster/resize/abort request. +func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { + err := h.API.ResizeAbort() + var msg string + if err != nil { + switch errors.Cause(err) { + case pilosa.ErrNodeNotCoordinator: + http.Error(w, err.Error(), http.StatusBadRequest) + return + case pilosa.ErrResizeNotRunning: + msg = err.Error() + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + // Encode response. + if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ + Info: msg, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type clusterResizeAbortResponse struct { + Info string `json:"info"` +} + +func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { + err := h.API.RecalculateCaches(r.Context()) + if err != nil { + http.Error(w, "recalculating caches: "+err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } + + err := h.API.ClusterMessage(r.Context(), r.Body) + if err != nil { + // TODO this was the previous behavior, but perhaps not everything is a bad request + http.Error(w, err.Error(), http.StatusBadRequest) + } + + if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +func (h *Handler) GetAPI() *pilosa.API { + return h.API +} + +type defaultClusterMessageResponse struct{} + +type queryValidationSpec struct { + required []string + args map[string]struct{} +} + +func queryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { + args := map[string]struct{}{} + for _, arg := range requiredArgs { + args[arg] = struct{}{} + } + + return &queryValidationSpec{ + required: requiredArgs, + args: args, + } +} + +func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec { + for _, arg := range args { + s.args[arg] = struct{}{} + } + return s +} + +func (s queryValidationSpec) validate(query url.Values) error { + for _, req := range s.required { + if query.Get(req) == "" { + return errors.Errorf("%s is required", req) + } + } + for k := range query { + if _, ok := s.args[k]; !ok { + return errors.Errorf("%s is not a valid argument", k) + } + } + return nil +} + +func GetHTTPClient(t *tls.Config) *http.Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 1000, + MaxIdleConnsPerHost: 200, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + if t != nil { + transport.TLSClientConfig = t + } + return &http.Client{Transport: transport} +} diff --git a/handler_internal_test.go b/http/handler_internal_test.go similarity index 94% rename from handler_internal_test.go rename to http/handler_internal_test.go index 837c456ca..fa7c23060 100644 --- a/handler_internal_test.go +++ b/http/handler_internal_test.go @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa +package http import ( "encoding/json" "reflect" "testing" + + "github.com/pilosa/pilosa" ) // Test custom UnmarshalJSON for postIndexRequest object @@ -27,7 +29,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { expected postIndexRequest err string }{ - {json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}}, + {json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, @@ -62,12 +64,12 @@ func TestPostFieldRequestUnmarshalJSON(t *testing.T) { expected postFieldRequest err string }{ - {json: `{"options": {}}`, expected: postFieldRequest{Options: FieldOptions{}}}, + {json: `{"options": {}}`, expected: postFieldRequest{Options: pilosa.FieldOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, {json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"}, - {json: `{"options": {"cacheType": "type"}}`, expected: postFieldRequest{Options: FieldOptions{CacheType: "type"}}}, + {json: `{"options": {"cacheType": "type"}}`, expected: postFieldRequest{Options: pilosa.FieldOptions{CacheType: "type"}}}, {json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, } for _, test := range tests { diff --git a/handler_test.go b/http/handler_test.go similarity index 93% rename from handler_test.go rename to http/handler_test.go index f9e490d9d..39941a669 100644 --- a/handler_test.go +++ b/http/handler_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package http_test import ( "bytes" @@ -21,7 +21,7 @@ import ( "fmt" "io" "io/ioutil" - "net/http" + gohttp "net/http" "net/http/httptest" "reflect" "strings" @@ -29,6 +29,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" @@ -49,7 +50,7 @@ func TestHandlerPanics(t *testing.T) { if !bytes.Contains(bufbytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) { t.Fatalf("expected panic in log, but got: %s", bufbytes) } - if w.Code != http.StatusInternalServerError { + if w.Code != gohttp.StatusInternalServerError { t.Fatalf("expected internal server error, but got: %v", w.Code) } bodyBytes := w.Body.Bytes() @@ -69,7 +70,7 @@ func TestHandler_NotFound(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) - if w.Code != http.StatusNotFound { + if w.Code != gohttp.StatusNotFound { t.Fatalf("invalid status: %d", w.Code) } } @@ -101,7 +102,7 @@ func TestHandler_Schema(t *testing.T) { h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { @@ -142,7 +143,7 @@ func TestHandler_Status(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}],"localID":"node0"}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -156,9 +157,9 @@ func TestHandler_Info(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", SliceWidth) { + } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) { t.Fatalf("unexpected body: %s", body) } } @@ -173,7 +174,7 @@ func TestHandler_ClusterResizeAbort(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { bod, err := ioutil.ReadAll(w.Body) t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err) } else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" { @@ -188,20 +189,20 @@ func TestHandler_MaxSlices(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*pilosa.SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*pilosa.SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*pilosa.SliceWidth)+4) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*pilosa.SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*pilosa.SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*pilosa.SliceWidth)+8) h := test.MustNewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -229,7 +230,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -270,7 +271,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, req) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } } @@ -286,7 +287,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { h.API.Holder = hldr.Holder h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -295,7 +296,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { func TestHandler_Query_Params_Err(t *testing.T) { w := httptest.NewRecorder() test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -317,7 +318,7 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -340,14 +341,14 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeUint64 { + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if n := resp.Results[0].N; n != 100 { t.Fatalf("unexpected n: %d", n) @@ -370,7 +371,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -403,7 +404,7 @@ func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -428,16 +429,16 @@ func TestHandler_Query_Row_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow { + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) { + } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) @@ -486,7 +487,7 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { r.Header.Set("Content-Type", "application/x-protobuf") r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } @@ -494,9 +495,9 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) { + if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow { + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) @@ -536,7 +537,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -562,14 +563,14 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypePairs { + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if a := resp.Results[0].GetPairs(); len(a) != 2 { t.Fatalf("unexpected pair length: %d", len(a)) @@ -590,7 +591,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"executing: marker"}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -613,7 +614,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } @@ -635,7 +636,7 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil)) - if w.Code != http.StatusMethodNotAllowed { + if w.Code != gohttp.StatusMethodNotAllowed { t.Fatalf("invalid status: %d", w.Code) } } @@ -650,7 +651,7 @@ func TestHandler_Query_ErrParse(t *testing.T) { h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -672,14 +673,14 @@ func TestHandler_Index_Delete(t *testing.T) { } // Send request to delete index. - resp, err := http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) + resp, err := gohttp.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) if err != nil { t.Fatal(err) } defer resp.Body.Close() // Verify body response. - if resp.StatusCode != http.StatusOK { + if resp.StatusCode != gohttp.StatusOK { t.Fatalf("unexpected status: %d", resp.StatusCode) } else if buf, err := ioutil.ReadAll(resp.Body); err != nil { t.Fatal(err) @@ -707,7 +708,7 @@ func TestHandler_DeleteField(t *testing.T) { h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/field/f1", strings.NewReader(""))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -749,7 +750,7 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) { blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") // Send block checksums to determine diff. - resp, err := http.Post( + resp, err := gohttp.Post( s.URL+"/index/i/attr/diff", "application/json", strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), @@ -799,7 +800,7 @@ func TestHandler_Field_AttrStore_Diff(t *testing.T) { blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") // Send block checksums to determine diff. - resp, err := http.Post( + resp, err := gohttp.Post( s.URL+"/index/i/field/meta/attr/diff", "application/json", strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), @@ -831,7 +832,7 @@ func TestHandler_Version(t *testing.T) { if strings.HasPrefix(version, "v") { version = version[1:] } - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if w.Body.String() != `{"version":"`+version+`"}`+"\n" { t.Fatalf("unexpected body: %q", w.Body.String()) @@ -851,7 +852,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -861,7 +862,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { w = httptest.NewRecorder() r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil) h.ServeHTTP(w, r) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } @@ -869,7 +870,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { w = httptest.NewRecorder() r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil) h.ServeHTTP(w, r) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } } @@ -885,7 +886,7 @@ func TestHandler_Expvars(t *testing.T) { w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } } @@ -908,7 +909,7 @@ func TestHandler_RecalculateCaches(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) - if w.Code != http.StatusNoContent { + if w.Code != gohttp.StatusNoContent { t.Fatalf("unexpected status code: %d", w.Code) } @@ -939,7 +940,7 @@ func TestHandler_CORS(t *testing.T) { } // CORS config should allow preflight response - handler = test.MustNewHandler(pilosa.OptHandlerAllowedOrigins([]string{"http://test/"})) + handler = test.MustNewHandler(http.OptHandlerAllowedOrigins([]string{"http://test/"})) w = httptest.NewRecorder() handler.ServeHTTP(w, req) result = w.Result() diff --git a/pilosa.go b/pilosa.go index 6a7e74dfb..a87bab4fc 100644 --- a/pilosa.go +++ b/pilosa.go @@ -88,17 +88,17 @@ type ColumnAttrSet struct { Attrs map[string]interface{} `json:"attrs,omitempty"` } -// encodeColumnAttrSets converts a into its internal representation. -func encodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet { +// EncodeColumnAttrSets converts a into its internal representation. +func EncodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet { other := make([]*internal.ColumnAttrSet, len(a)) for i := range a { - other[i] = encodeColumnAttrSet(a[i]) + other[i] = EncodeColumnAttrSet(a[i]) } return other } -// encodeColumnAttrSet converts set into its internal representation. -func encodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { +// EncodeColumnAttrSet converts set into its internal representation. +func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { return &internal.ColumnAttrSet{ ID: set.ID, Attrs: encodeAttrs(set.Attrs), diff --git a/row.go b/row.go index ec2c42726..b7643a975 100644 --- a/row.go +++ b/row.go @@ -261,8 +261,8 @@ func (r *Row) Columns() []uint64 { return a } -// encodeRow converts r into its internal representation. -func encodeRow(r *Row) *internal.Row { +// EncodeRow converts r into its internal representation. +func EncodeRow(r *Row) *internal.Row { if r == nil { return nil } @@ -273,8 +273,8 @@ func encodeRow(r *Row) *internal.Row { } } -// decodeRow converts r from its internal representation. -func decodeRow(pr *internal.Row) *Row { +// DecodeRow converts r from its internal representation. +func DecodeRow(pr *internal.Row) *Row { if pr == nil { return nil } diff --git a/server.go b/server.go index 471319dd7..a324ebd49 100644 --- a/server.go +++ b/server.go @@ -59,7 +59,7 @@ type Server struct { executor *Executor // External - handler *Handler + handler Handlerer Broadcaster Broadcaster BroadcastReceiver BroadcastReceiver Gossiper Gossiper @@ -127,7 +127,7 @@ func OptServerLongQueryTime(dur time.Duration) ServerOption { } } -func OptServerHandler(h *Handler) ServerOption { +func OptServerHandler(h Handlerer) ServerOption { return func(s *Server) error { s.handler = h return nil @@ -162,16 +162,24 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { } } +// TODO: Remove RemoteClient func OptServerRemoteClient(c *http.Client) ServerOption { return func(s *Server) error { - s.executor = NewExecutor(c) s.remoteClient = c - s.defaultClient = NewInternalHTTPClientFromURI(nil, c) s.Cluster.RemoteClient = c return nil } } +func OptServerInternalClient(c InternalClient) ServerOption { + return func(s *Server) error { + s.executor = NewExecutor(ExecutorOptInternalQueryClient(c)) + s.defaultClient = c + s.Cluster.InternalClient = c + return nil + } +} + func OptServerStatsClient(sc StatsClient) ServerOption { return func(s *Server) error { s.Holder.Stats = sc @@ -203,15 +211,15 @@ func OptServerURI(uri *URI) ServerOption { // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { - handler, err := NewHandler() - if err != nil { - return nil, errors.Wrap(err, "initializing handler") - } + //handler, err := NewNopHandler() + //if err != nil { + // return nil, errors.Wrap(err, "initializing handler") + //} s := &Server{ - closing: make(chan struct{}), - Cluster: NewCluster(), - Holder: NewHolder(), - handler: handler, + closing: make(chan struct{}), + Cluster: NewCluster(), + Holder: NewHolder(), + //handler: handler, Broadcaster: NopBroadcaster, BroadcastReceiver: NopBroadcastReceiver, diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), @@ -268,7 +276,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.executor.Node = node s.executor.Cluster = s.Cluster s.executor.MaxWritesPerRequest = s.maxWritesPerRequest - s.handler.API.Executor = s.executor + s.handler.GetAPI().Executor = s.executor return s, nil } @@ -300,11 +308,12 @@ func (s *Server) Open() error { s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest // Initialize HTTP handler. - s.handler.API.Holder = s.Holder - s.handler.API.Broadcaster = s.Broadcaster - s.handler.API.BroadcastHandler = s - s.handler.API.StatusHandler = s - s.handler.API.Cluster = s.Cluster + api := s.handler.GetAPI() + api.Holder = s.Holder + api.Broadcaster = s.Broadcaster + api.BroadcastHandler = s + api.StatusHandler = s + api.Cluster = s.Cluster // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster diff --git a/server/server.go b/server/server.go index bb75a1d2e..febb79529 100644 --- a/server/server.go +++ b/server/server.go @@ -25,7 +25,6 @@ import ( "log" "math/rand" "net" - "net/http" "os" "os/signal" "strconv" @@ -39,6 +38,7 @@ import ( "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gopsutil" "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/statsd" "github.com/pkg/errors" ) @@ -164,13 +164,17 @@ func (m *Command) SetupServer() error { } m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime) - handler, err := pilosa.NewHandler(pilosa.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins)) + api := pilosa.NewAPI() + api.Logger = m.logger + + handler, err := http.NewHandler( + http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), + http.OptHandlerAPI(api), + http.OptHandlerLogger(m.logger), + ) if err != nil { return errors.Wrap(err, "wrapping handler") } - handler.Logger = m.logger - handler.API = pilosa.NewAPI() - handler.API.Logger = m.logger uri, err := pilosa.AddressWithDefaults(m.Config.Bind) if err != nil { @@ -211,8 +215,8 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "getting listener") } - c := GetHTTPClient(TLSConfig) - handler.API.RemoteClient = c + c := http.GetHTTPClient(TLSConfig) + api.RemoteClient = c m.Server, err = pilosa.NewServer( pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)), @@ -232,31 +236,12 @@ func (m *Command) SetupServer() error { pilosa.OptServerListener(ln), pilosa.OptServerURI(uri), pilosa.OptServerRemoteClient(c), + pilosa.OptServerInternalClient(http.NewInternalHTTPClientFromURI(uri, c)), ) return errors.Wrap(err, "new server") } -func GetHTTPClient(t *tls.Config) *http.Client { - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 1000, - MaxIdleConnsPerHost: 200, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - if t != nil { - transport.TLSClientConfig = t - } - return &http.Client{Transport: transport} -} - // SetupNetworking sets up internode communication based on the configuration. func (m *Command) SetupNetworking() error { diff --git a/server/server_test.go b/server/server_test.go index 4356ad241..32d57b01c 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -29,6 +29,7 @@ import ( "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -44,7 +45,7 @@ func TestMain_Set_Quick(t *testing.T) { defer m.Close() // Create client. - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) + client, err := http.NewInternalHTTPClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil)) if err != nil { t.Fatal(err) } diff --git a/test/client.go b/test/client.go index 9e391e88b..faef0b480 100644 --- a/test/client.go +++ b/test/client.go @@ -15,19 +15,19 @@ package test import ( - "net/http" + gohttp "net/http" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" ) // Client represents a test wrapper for pilosa.Client. type Client struct { - *pilosa.InternalHTTPClient + *http.InternalHTTPClient } // MustNewClient returns a new instance of Client. Panic on error. -func MustNewClient(host string, h *http.Client) *Client { - c, err := pilosa.NewInternalHTTPClient(host, h) +func MustNewClient(host string, h *gohttp.Client) *Client { + c, err := http.NewInternalHTTPClient(host, h) if err != nil { panic(err) } diff --git a/test/executor.go b/test/executor.go index 8cd6391c3..2e2912d60 100644 --- a/test/executor.go +++ b/test/executor.go @@ -15,12 +15,12 @@ package test import ( - "net/http" + gohttp "net/http" "strings" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/server" ) // Executor represents a test wrapper for pilosa.Executor. @@ -28,16 +28,17 @@ type Executor struct { *pilosa.Executor } -var remoteClient *http.Client +var remoteClient *gohttp.Client func init() { - remoteClient = server.GetHTTPClient(nil) + remoteClient = http.GetHTTPClient(nil) } // NewExecutor returns a new instance of Executor. // The executor always matches the uri of the first cluster node. func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { - executor := pilosa.NewExecutor(remoteClient) + client := http.NewInternalHTTPClientFromURI(nil, remoteClient) + executor := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(client)) e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster diff --git a/test/handler.go b/test/handler.go index 401fa64ab..27fa20503 100644 --- a/test/handler.go +++ b/test/handler.go @@ -19,25 +19,26 @@ import ( "encoding/json" "io" "io/ioutil" - "net/http" + gohttp "net/http" "net/http/httptest" "net/url" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) // Handler represents a test wrapper for pilosa.Handler. type Handler struct { - *pilosa.Handler + *http.Handler Executor HandlerExecutor } // NewHandler returns a new instance of Handler. -func NewHandler(opts ...pilosa.HandlerOption) (*Handler, error) { - handler, err := pilosa.NewHandler(opts...) +func NewHandler(opts ...http.HandlerOption) (*Handler, error) { + handler, err := http.NewHandler(opts...) if err != nil { return nil, err } @@ -55,7 +56,7 @@ func NewHandler(opts ...pilosa.HandlerOption) (*Handler, error) { } // MustNewHandler returns a new instance of Handler. -func MustNewHandler(opts ...pilosa.HandlerOption) *Handler { +func MustNewHandler(opts ...http.HandlerOption) *Handler { h, err := NewHandler(opts...) if err != nil { panic(err) @@ -145,8 +146,8 @@ func MustParseURLHost(rawurl string) string { } // MustNewHTTPRequest creates a new HTTP request. Panic on error. -func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request { - req, err := http.NewRequest(method, urlStr, body) +func MustNewHTTPRequest(method, urlStr string, body io.Reader) *gohttp.Request { + req, err := gohttp.NewRequest(method, urlStr, body) if err != nil { panic(err) } diff --git a/test/pilosa.go b/test/pilosa.go index 4a741ea8b..426988f3f 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -19,15 +19,15 @@ import ( "fmt" "io" "io/ioutil" - "net/http" + gohttp "net/http" "os" "strings" "testing" "time" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/boltdb" "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" @@ -238,8 +238,8 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) ( func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } // Client returns a client to connect to the program. -func (m *Main) Client() *pilosa.InternalHTTPClient { - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) +func (m *Main) Client() *http.InternalHTTPClient { + client, err := http.NewInternalHTTPClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil)) if err != nil { panic(err) } @@ -249,7 +249,7 @@ func (m *Main) Client() *pilosa.InternalHTTPClient { // Query executes a query against the program through the HTTP API. func (m *Main) Query(index, rawQuery, query string) (string, error) { resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query) - if resp.StatusCode != http.StatusOK { + if resp.StatusCode != gohttp.StatusOK { return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) } return resp.Body, nil @@ -267,11 +267,11 @@ func (m *Main) RecalculateCaches() error { // MustDo executes http.Do() with an http.NewRequest(). Panic on error. func MustDo(method, urlStr string, body string) *httpResponse { - req, err := http.NewRequest(method, urlStr, strings.NewReader(body)) + req, err := gohttp.NewRequest(method, urlStr, strings.NewReader(body)) if err != nil { panic(err) } - resp, err := http.DefaultClient.Do(req) + resp, err := gohttp.DefaultClient.Do(req) if err != nil { panic(err) } @@ -287,6 +287,6 @@ func MustDo(method, urlStr string, body string) *httpResponse { // httpResponse is a wrapper for http.Response that holds the Body as a string. type httpResponse struct { - *http.Response + *gohttp.Response Body string }