diff --git a/api.go b/api.go
index f86a1db28..3f3e8bd50 100644
--- a/api.go
+++ b/api.go
@@ -46,7 +46,6 @@ type API struct {
BroadcastHandler BroadcastHandler
StatusHandler StatusHandler
Cluster *Cluster
- URI URI
RemoteClient *http.Client
Logger Logger
}
@@ -294,7 +293,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin
// Validate that this handler owns the slice.
if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) {
- api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice)
+ api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName)
return ErrClusterDoesNotOwnSlice
}
@@ -957,7 +956,7 @@ func (api *API) LongQueryTime() time.Duration {
func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) {
// Validate that this handler owns the slice.
if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) {
- api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice)
+ api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName)
return nil, nil, ErrClusterDoesNotOwnSlice
}
@@ -1118,6 +1117,17 @@ func (api *API) Version() string {
return strings.TrimPrefix(Version, "v")
}
+// Info returns information about this server instance
+func (api *API) Info() ServerInfo {
+ return ServerInfo{
+ SliceWidth: SliceWidth,
+ }
+}
+
+type ServerInfo struct {
+ SliceWidth uint64 `json:"sliceWidth"`
+}
+
type apiMethod int
// API validation constants.
diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go
index ebb539903..3604de5b6 100644
--- a/boltdb/attrstore.go
+++ b/boltdb/attrstore.go
@@ -27,6 +27,7 @@ import (
"github.com/boltdb/bolt"
"github.com/pilosa/pilosa"
+ "github.com/pkg/errors"
)
// AttrBlockSize is the size of attribute blocks for anti-entropy.
@@ -93,7 +94,7 @@ func (s *AttrStore) Open() error {
// Open storage.
db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
- return err
+ return errors.Wrap(err, "opening storage")
}
s.db = db
@@ -104,7 +105,7 @@ func (s *AttrStore) Open() error {
}
return nil
}); err != nil {
- return err
+ return errors.Wrap(err, "initializing")
}
return nil
@@ -136,7 +137,7 @@ func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
}
return nil
}); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "finding attributes")
}
// Add to cache.
@@ -154,7 +155,7 @@ func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
// Check if the attributes already exist under a read-only lock.
if attr, err := s.Attrs(id); err != nil {
- return err
+ return errors.Wrap(err, "checking attrs")
} else if attr != nil && mapContains(attr, m) {
return nil
}
@@ -173,7 +174,7 @@ func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
return nil
}); err != nil {
- return err
+ return errors.Wrap(err, "updating store")
}
// Swap attributes map in cache.
@@ -222,7 +223,7 @@ func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) {
tx, err := s.db.Begin(false)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "starting transaction")
}
defer tx.Rollback()
@@ -256,7 +257,7 @@ func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, erro
// Start read-only transaction.
tx, err := s.db.Begin(false)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "starting transaction")
}
defer tx.Rollback()
@@ -273,7 +274,7 @@ func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, erro
// Decode attribute map and associate with id.
attrs, err := pilosa.DecodeAttrs(v)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "decoding attrs")
}
m[btou64(k)] = attrs
@@ -329,10 +330,10 @@ func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string
// Marshal and save new values.
buf, err := pilosa.EncodeAttrs(attr)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "encoding attrs")
}
if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "saving attrs")
}
return attr, nil
}
diff --git a/broadcast.go b/broadcast.go
index de43f3b85..2be5e08db 100644
--- a/broadcast.go
+++ b/broadcast.go
@@ -20,6 +20,7 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
+ "github.com/pkg/errors"
)
// MemberSet represents an interface for Node membership and inter-node communication.
@@ -187,7 +188,7 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
}
buf, err := proto.Marshal(m)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "marshalling")
}
return append([]byte{typ}, buf...), nil
}
@@ -241,7 +242,7 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
}
if err := proto.Unmarshal(buf, m); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "unmarshalling")
}
return m, nil
}
diff --git a/broadcast_test.go b/broadcast_test.go
index 5654f7c28..970a249cb 100644
--- a/broadcast_test.go
+++ b/broadcast_test.go
@@ -56,9 +56,14 @@ func testMessageMarshal(t *testing.T, m proto.Message) {
// Ensure that BroadcastReceiver can register a BroadcastHandler.
func TestBroadcast_BroadcastReceiver(t *testing.T) {
+ path, err := ioutil.TempDir("", "pilosa-")
+ if err != nil {
+ panic(err)
+ }
com := server.NewCommand(bytes.NewBuffer([]byte{}), ioutil.Discard, ioutil.Discard)
com.Config.Bind = "localhost:0"
- err := com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO
+ com.Config.DataDir = path
+ err = com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO
if err != nil {
t.Fatalf("setting up server: %v", err)
}
diff --git a/client.go b/client.go
index 5997cf4ef..72225aef2 100644
--- a/client.go
+++ b/client.go
@@ -19,7 +19,6 @@ import (
"bytes"
"context"
"encoding/json"
- "errors"
"fmt"
"io"
"io/ioutil"
@@ -35,6 +34,7 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
+ "github.com/pkg/errors"
)
// ClientOptions represents the configuration for a InternalHTTPClient
@@ -58,7 +58,7 @@ func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTT
uri, err := NewURIFromAddress(host)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "getting URI")
}
client := NewInternalHTTPClientFromURI(uri, remoteClient)
@@ -93,7 +93,7 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool)
// Build request.
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+Version)
@@ -101,7 +101,7 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool)
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@@ -126,7 +126,7 @@ func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) {
// Build request.
req, err := http.NewRequest("GET", u, nil)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+Version)
@@ -134,7 +134,7 @@ func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) {
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@@ -154,14 +154,14 @@ func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt
Options: opt,
})
if err != nil {
- return err
+ return errors.Wrap(err, "encoding request")
}
// Create URL & HTTP request.
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index))
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
- return err
+ return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/json")
@@ -171,14 +171,14 @@ func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return err
+ return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
// Read body.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
- return err
+ return errors.Wrap(err, "reading")
}
// Handle response based on status code.
@@ -201,7 +201,7 @@ func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, sl
// Build request.
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+Version)
@@ -209,7 +209,7 @@ func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, sl
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@@ -239,14 +239,14 @@ func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index stri
// Encode request object.
buf, err := proto.Marshal(queryRequest)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "marshaling")
}
// Create HTTP request.
u := uri.Path(fmt.Sprintf("/index/%s/query", index))
req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
@@ -257,14 +257,14 @@ func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index stri
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
// Read body and unmarshal response.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "reading")
} else if resp.StatusCode != http.StatusOK {
return nil, errors.New(string(body))
}
@@ -398,7 +398,7 @@ func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []b
u := nodePathToURL(node, "/import")
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
- return err
+ return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/x-protobuf")
@@ -408,14 +408,14 @@ func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []b
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return err
+ return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
// Read body and unmarshal response.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
- return err
+ return errors.Wrap(err, "reading")
} else if resp.StatusCode != http.StatusOK {
return errors.New(string(body))
}
@@ -486,7 +486,7 @@ func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, bu
u := nodePathToURL(node, "/import-value")
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
- return err
+ return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/x-protobuf")
@@ -496,14 +496,14 @@ func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, bu
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return err
+ return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
// Read body and unmarshal response.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
- return err
+ return errors.Wrap(err, "reading")
} else if resp.StatusCode != http.StatusOK {
return errors.New(string(body))
}
@@ -564,7 +564,7 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde
// Generate HTTP request.
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
- return err
+ return errors.Wrap(err, "creating request")
}
req.Header.Set("Accept", "text/csv")
req.Header.Set("User-Agent", "pilosa/"+Version)
@@ -572,7 +572,7 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return err
+ return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@@ -583,7 +583,7 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde
// Copy body to writer.
if _, err := io.Copy(w, resp.Body); err != nil {
- return err
+ return errors.Wrap(err, "copying")
}
return nil
@@ -618,13 +618,13 @@ func (c *InternalHTTPClient) BackupTo(ctx context.Context, w io.Writer, index, f
// Backup every slice to the tar file.
for i := uint64(0); i <= maxSlices[index]; i++ {
if err := c.backupSliceTo(ctx, tw, index, frame, view, i); err != nil {
- return err
+ return errors.Wrap(err, "backing up slice")
}
}
// Close tar file.
if err := tw.Close(); err != nil {
- return err
+ return errors.Wrap(err, "closing")
}
return nil
@@ -644,9 +644,9 @@ func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer,
// Read entire buffer to determine file size.
data, err := ioutil.ReadAll(r)
if err != nil {
- return err
+ return errors.Wrap(err, "reading")
} else if err := r.Close(); err != nil {
- return err
+ return errors.Wrap(err, "closing")
}
// Write slice file header.
@@ -656,12 +656,12 @@ func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer,
Size: int64(len(data)),
ModTime: time.Now(),
}); err != nil {
- return err
+ return errors.Wrap(err, "writing header")
}
// Write buffer to file.
if _, err := tw.Write(data); err != nil {
- return fmt.Errorf("write buffer: %s", err)
+ return errors.Wrap(err, "writing buffer")
}
return nil
@@ -711,7 +711,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame,
// Build request.
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+Version)
@@ -719,7 +719,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame,
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "executing request")
}
// Return error if status is not OK.
@@ -751,7 +751,7 @@ func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index
if err == io.EOF {
return nil
} else if err != nil {
- return err
+ return errors.Wrap(err, "opening")
}
// Parse slice from entry name.
@@ -763,12 +763,12 @@ func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index
// Read file into buffer.
var buf bytes.Buffer
if _, err := io.CopyN(&buf, tr, hdr.Size); err != nil {
- return err
+ return errors.Wrap(err, "copying")
}
// Restore file to all nodes that own it.
if err := c.restoreSliceFrom(ctx, buf.Bytes(), index, frame, view, slice); err != nil {
- return err
+ return errors.Wrap(err, "restoring")
}
}
}
@@ -794,14 +794,14 @@ func (c *InternalHTTPClient) restoreSliceFrom(ctx context.Context, buf []byte, i
// Build request.
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
- return err
+ return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("User-Agent", "pilosa/"+Version)
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return err
+ return errors.Wrap(err, "executing request")
}
resp.Body.Close()
@@ -825,14 +825,14 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin
Options: opt,
})
if err != nil {
- return err
+ return errors.Wrap(err, "marshaling")
}
// Create URL & HTTP request.
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s", index, frame))
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
- return err
+ return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/json")
@@ -842,14 +842,14 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return err
+ return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
// Read body.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
- return err
+ return errors.Wrap(err, "reading")
}
// Handle response based on status code.
@@ -873,7 +873,7 @@ func (c *InternalHTTPClient) RestoreFrame(ctx context.Context, host, index, fram
// Build request.
req, err := http.NewRequest("POST", u.String(), nil)
if err != nil {
- return err
+ return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("User-Agent", "pilosa/"+Version)
@@ -881,7 +881,7 @@ func (c *InternalHTTPClient) RestoreFrame(ctx context.Context, host, index, fram
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return err
+ return errors.Wrap(err, "executing request")
}
resp.Body.Close()
@@ -899,7 +899,7 @@ func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/views", index, frame))
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Accept", "application/json")
req.Header.Set("User-Agent", "pilosa/"+Version)
@@ -907,7 +907,7 @@ func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@@ -924,7 +924,7 @@ func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string
// Decode response.
var rsp getFrameViewsResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "decoding")
}
return rsp.Views, nil
}
@@ -943,7 +943,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v
// Build request.
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+Version)
@@ -951,7 +951,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@@ -967,7 +967,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v
// Decode response object.
var rsp getFragmentBlocksResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "decoding")
}
return rsp.Blocks, nil
}
@@ -982,13 +982,13 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view s
Block: uint64(block),
})
if err != nil {
- return nil, nil, err
+ return nil, nil, errors.Wrap(err, "marshaling")
}
u := uriPathToURL(c.defaultURI, "/fragment/block/data")
req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf))
if err != nil {
- return nil, nil, err
+ return nil, nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/protobuf")
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
@@ -997,7 +997,7 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view s
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return nil, nil, err
+ return nil, nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@@ -1013,9 +1013,9 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view s
// Decode response object.
var rsp internal.BlockDataResponse
if body, err := ioutil.ReadAll(resp.Body); err != nil {
- return nil, nil, err
+ return nil, nil, errors.Wrap(err, "reading")
} else if err := proto.Unmarshal(body, &rsp); err != nil {
- return nil, nil, err
+ return nil, nil, errors.Wrap(err, "unmarshalling")
}
return rsp.RowIDs, rsp.ColumnIDs, nil
}
@@ -1027,13 +1027,13 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b
// Encode request.
buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks})
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "marshaling")
}
// Build request.
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", "pilosa/"+Version)
@@ -1041,7 +1041,7 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@@ -1055,7 +1055,7 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b
// Decode response object.
var rsp postIndexAttrDiffResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "decoding")
}
return rsp.Attrs, nil
}
@@ -1067,13 +1067,13 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin
// Encode request.
buf, err := json.Marshal(postFrameAttrDiffRequest{Blocks: blks})
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "marshaling")
}
// Build request.
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", "pilosa/"+Version)
@@ -1081,7 +1081,7 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@@ -1097,7 +1097,7 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin
// Decode response object.
var rsp postFrameAttrDiffResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "decoding")
}
return rsp.Attrs, nil
}
diff --git a/client_test.go b/client_test.go
index 88dc3accb..e2b806412 100644
--- a/client_test.go
+++ b/client_test.go
@@ -37,7 +37,6 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) {
for i := 0; i < numNodes; i++ {
hldr[i] = test.MustOpenHolder()
server[i] = test.NewServer()
- server[i].Handler.API.URI = server[i].HostURI()
server[i].Handler.API.Cluster = c
server[i].Handler.API.Cluster.Nodes[i].URI = server[i].HostURI()
server[i].Handler.API.Holder = hldr[i].Holder
@@ -218,7 +217,6 @@ func TestClient_Import(t *testing.T) {
s := test.NewServer()
defer s.Close()
- s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
@@ -269,7 +267,6 @@ func TestClient_ImportInverseEnabled(t *testing.T) {
s := test.NewServer()
defer s.Close()
- s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
@@ -318,7 +315,6 @@ func TestClient_ImportValue(t *testing.T) {
s := test.NewServer()
defer s.Close()
- s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
@@ -386,7 +382,6 @@ func TestClient_BackupRestore(t *testing.T) {
s := test.NewServer()
defer s.Close()
- s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
@@ -452,7 +447,6 @@ func TestClient_BackupInverseView(t *testing.T) {
s := test.NewServer()
defer s.Close()
- s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
@@ -489,7 +483,6 @@ func TestClient_BackupInvalidView(t *testing.T) {
s := test.NewServer()
defer s.Close()
- s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
@@ -518,7 +511,6 @@ func TestClient_FragmentBlocks(t *testing.T) {
s := test.NewServer()
defer s.Close()
- s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
diff --git a/cluster.go b/cluster.go
index b02bd50d2..9dfeac094 100644
--- a/cluster.go
+++ b/cluster.go
@@ -307,16 +307,16 @@ func (c *Cluster) isCoordinator() bool {
// nodes with its version of Cluster.Status.
func (c *Cluster) SetCoordinator(n *Node) error {
c.mu.Lock()
- defer c.mu.Unlock()
// Verify that the new Coordinator value matches
// this node.
if c.Node.ID != n.ID {
+ c.mu.Unlock()
return fmt.Errorf("coordinator node does not match this node")
}
// Update IsCoordinator on all nodes (locally).
_ = c.updateCoordinator(n)
-
+ c.mu.Unlock()
// Send the update coordinator message to all nodes.
err := c.Broadcaster.SendSync(
&internal.UpdateCoordinatorMessage{
@@ -721,7 +721,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R
// Determine if a node is being added or removed.
action, diffNodeID, err := c.diff(to)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "diffing")
}
// Initialize the map with all the nodes in `to`.
@@ -905,7 +905,7 @@ func (c *Cluster) Open() error {
// Load topology file if it exists.
if err := c.loadTopology(); err != nil {
- return fmt.Errorf("load topology: %v", err)
+ return errors.Wrap(err, "loading topology")
}
c.ID = c.Topology.ClusterID
@@ -1006,7 +1006,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error {
if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil {
c.Logger.Printf("setStateAndBroadcast error: err=%s", err)
}
- return err
+ return errors.Wrap(err, "setting state")
}
// j.Run() runs in a goroutine because in the case where the
@@ -1023,14 +1023,14 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error {
// Make sure j.Run() didn't return an error.
if eg.Wait() != nil {
- return err
+ return errors.Wrap(err, "running job")
}
c.Logger.Printf("received jobResult: %s", jobResult)
switch jobResult {
case ResizeJobStateDone:
if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil {
- return err
+ return errors.Wrap(err, "completing finished job")
}
// Add/remove uri to/from the cluster.
if j.action == ResizeJobActionRemove {
@@ -1040,7 +1040,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error {
}
case ResizeJobStateAborted:
if err := c.CompleteCurrentJob(ResizeJobStateAborted); err != nil {
- return err
+ return errors.Wrap(err, "completing aborted job")
}
}
return nil
@@ -1055,7 +1055,7 @@ func (c *Cluster) setStateAndBroadcast(state string) error {
func (c *Cluster) sendTo(node *Node, msg proto.Message) error {
if err := c.Broadcaster.SendTo(node, msg); err != nil {
- return err
+ return errors.Wrap(err, "sending")
}
return nil
}
@@ -1124,7 +1124,7 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) {
j, err := c.generateResizeJobByAction(nodeAction)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "generating job")
}
c.Logger.Printf("generated ResizeJob: %d", j.ID)
@@ -1171,7 +1171,7 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob,
for _, idx := range c.Holder.Indexes() {
fragSources, err := c.fragSources(toCluster, idx)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "getting sources")
}
for id, sources := range fragSources {
@@ -1223,7 +1223,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
// Make sure the cluster status on this node agrees with the Coordinator
// before attempting a resize.
if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil {
- return err
+ return errors.Wrap(err, "merging cluster status")
}
c.Logger.Printf("MergeClusterStatus done, start goroutine")
@@ -1248,7 +1248,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
// Sync the schema received in the resize instruction.
c.Logger.Printf("Holder ApplySchema")
if err := c.Holder.ApplySchema(instr.Schema); err != nil {
- return err
+ return errors.Wrap(err, "applying schema")
}
// Create a client for calling remote nodes.
@@ -1269,13 +1269,13 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
// Create view.
v, err := f.CreateViewIfNotExists(src.View)
if err != nil {
- return err
+ return errors.Wrap(err, "creating view")
}
// Create the local fragment.
frag, err := v.CreateFragmentIfNotExists(src.Slice)
if err != nil {
- return err
+ return errors.Wrap(err, "creating fragment")
}
// Stream slice from remote node.
@@ -1291,7 +1291,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
if err == ErrFragmentNotFound {
return nil
}
- return err
+ return errors.Wrap(err, "retrieving slice")
} else if rd == nil {
return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.Node.URI)
}
@@ -1304,7 +1304,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
}
return nil
}(); err != nil {
- return err
+ return errors.Wrap(err, "copying remote slice")
}
}
return nil
@@ -1438,7 +1438,7 @@ func (j *ResizeJob) Run() error {
err := j.distributeResizeInstructions()
if err != nil {
j.result <- ResizeJobStateAborted
- return err
+ return errors.Wrap(err, "distributing instructions")
}
return nil
}
@@ -1475,7 +1475,7 @@ func (j *ResizeJob) distributeResizeInstructions() error {
}
j.Logger.Printf("send resize instructions: %v", instr)
if err := j.Broadcaster.SendTo(node, instr); err != nil {
- return err
+ return errors.Wrap(err, "sending instruction")
}
}
return nil
@@ -1581,16 +1581,16 @@ func (c *Cluster) loadTopology() error {
c.Topology = NewTopology()
return nil
} else if err != nil {
- return err
+ return errors.Wrap(err, "reading file")
}
var pb internal.Topology
if err := proto.Unmarshal(buf, &pb); err != nil {
- return err
+ return errors.Wrap(err, "unmarshalling")
}
top, err := decodeTopology(&pb)
if err != nil {
- return err
+ return errors.Wrap(err, "decoding")
}
c.Topology = top
@@ -1601,13 +1601,13 @@ func (c *Cluster) loadTopology() error {
func (c *Cluster) saveTopology() error {
if err := os.MkdirAll(c.Path, 0777); err != nil {
- return err
+ return errors.Wrap(err, "creating directory")
}
if buf, err := proto.Marshal(encodeTopology(c.Topology)); err != nil {
- return err
+ return errors.Wrap(err, "marshalling")
} else if err := ioutil.WriteFile(filepath.Join(c.Path, ".topology"), buf, 0666); err != nil {
- return err
+ return errors.Wrap(err, "writing file")
}
return nil
}
@@ -1704,7 +1704,7 @@ func (c *Cluster) nodeJoin(node *Node) error {
}
if err := c.AddNode(node); err != nil {
- return err
+ return errors.Wrap(err, "adding node for agreement")
}
// Only change to normal if there is no existing data. Otherwise,
@@ -1742,7 +1742,7 @@ func (c *Cluster) nodeJoin(node *Node) error {
// If the holder does not yet contain data, go ahead and add the node.
if ok, err := c.Holder.HasData(); !ok && err == nil {
if err := c.AddNode(node); err != nil {
- return err
+ return errors.Wrap(err, "adding node")
}
return c.setStateAndBroadcast(ClusterStateNormal)
} else if err != nil {
@@ -1752,7 +1752,7 @@ func (c *Cluster) nodeJoin(node *Node) error {
// If the cluster has data, we need to change to RESIZING and
// kick off the resizing process.
if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil {
- return err
+ return errors.Wrap(err, "broadcasting state")
}
c.joiningLeavingNodes <- nodeAction{node, ResizeJobActionAdd}
@@ -1784,7 +1784,7 @@ func (c *Cluster) NodeLeave(node *Node) error {
_, err := c.generateResizeJobByAction(nodeAction{c.nodeByID(node.ID), ResizeJobActionRemove})
if err != nil {
- return err
+ return errors.Wrap(err, "generating job")
}
return c.nodeLeave(node)
@@ -1802,7 +1802,7 @@ func (c *Cluster) nodeLeave(node *Node) error {
// If the holder does not yet contain data, go ahead and remove the node.
if ok, err := c.Holder.HasData(); !ok && err == nil {
if err := c.RemoveNode(n); err != nil {
- return err
+ return errors.Wrap(err, "removing node")
}
return c.setStateAndBroadcast(ClusterStateNormal)
} else if err != nil {
@@ -1812,7 +1812,7 @@ func (c *Cluster) nodeLeave(node *Node) error {
// If the cluster has data then change state to RESIZING and
// kick off the resizing process.
if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil {
- return err
+ return errors.Wrap(err, "broadcasting state")
}
c.joiningLeavingNodes <- nodeAction{n, ResizeJobActionRemove}
@@ -1836,7 +1836,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error {
// Add all nodes from the coordinator.
for _, node := range officialNodes {
if err := c.AddNode(node); err != nil {
- return err
+ return errors.Wrap(err, "adding node")
}
}
@@ -1857,7 +1857,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error {
for _, nodeID := range nodeIDsToRemove {
if err := c.RemoveNode(c.nodeByID(nodeID)); err != nil {
- return err
+ return errors.Wrap(err, "removing node")
}
}
diff --git a/cluster_internal_test.go b/cluster_internal_test.go
index da9989760..6b1f6e72b 100644
--- a/cluster_internal_test.go
+++ b/cluster_internal_test.go
@@ -17,6 +17,7 @@ package pilosa
import (
"io/ioutil"
"reflect"
+ "strings"
"testing"
"github.com/pilosa/pilosa/internal"
@@ -243,8 +244,8 @@ func TestFragSources(t *testing.T) {
actual, err := (test.from).fragSources(test.to, test.idx)
if test.err != "" {
- if err.Error() != test.err {
- t.Fatalf("expected error: %s", test.err)
+ if !strings.Contains(err.Error(), test.err) {
+ t.Fatalf("expected error: %s, got: %s", test.err, err.Error())
}
} else {
if err != nil {
diff --git a/ctl/backup.go b/ctl/backup.go
index 1b16391e0..54a760695 100644
--- a/ctl/backup.go
+++ b/ctl/backup.go
@@ -16,12 +16,12 @@ package ctl
import (
"context"
- "errors"
"io"
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
+ "github.com/pkg/errors"
)
// BackupCommand represents a command for backing up a view.
@@ -60,26 +60,26 @@ func (cmd *BackupCommand) Run(ctx context.Context) error {
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
- return err
+ return errors.Wrap(err, "creating client")
}
// Open output file.
f, err := os.Create(cmd.Path)
if err != nil {
- return err
+ return errors.Wrap(err, "creating file")
}
defer f.Close()
// Begin streaming backup.
if err := client.BackupTo(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
- return err
+ return errors.Wrap(err, "backing up")
}
// Sync & close file to ensure durability.
if err := f.Sync(); err != nil {
- return err
+ return errors.Wrap(err, "syncing")
} else if err = f.Close(); err != nil {
- return err
+ return errors.Wrap(err, "closing file")
}
return nil
diff --git a/ctl/backup_test.go b/ctl/backup_test.go
index d80b72475..a073bc8a8 100644
--- a/ctl/backup_test.go
+++ b/ctl/backup_test.go
@@ -46,11 +46,7 @@ func TestBackupCommand_Run(t *testing.T) {
s := test.NewServer()
defer s.Close()
- uri, err := pilosa.NewURIFromAddress(s.Host())
- if err != nil {
- t.Fatal(err)
- }
- s.Handler.API.URI = *uri
+
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
diff --git a/ctl/bench.go b/ctl/bench.go
index b9743bb5e..425894738 100644
--- a/ctl/bench.go
+++ b/ctl/bench.go
@@ -16,7 +16,6 @@ package ctl
import (
"context"
- "errors"
"fmt"
"io"
"math/rand"
@@ -25,6 +24,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/server"
+ "github.com/pkg/errors"
)
// BenchCommand represents a command for benchmarking index operations.
@@ -58,7 +58,7 @@ func (cmd *BenchCommand) Run(ctx context.Context) error {
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
- return err
+ return errors.Wrap(err, "creating client")
}
switch cmd.Op {
diff --git a/ctl/bench_test.go b/ctl/bench_test.go
index c3014e6ca..4790ccb44 100644
--- a/ctl/bench_test.go
+++ b/ctl/bench_test.go
@@ -23,6 +23,7 @@ import (
"testing"
"github.com/pilosa/pilosa"
+ "github.com/pkg/errors"
)
func TestBenchCommand_InvalidOption(t *testing.T) {
@@ -31,7 +32,7 @@ func TestBenchCommand_InvalidOption(t *testing.T) {
cm := NewBenchCommand(stdin, stdout, stderr)
err := cm.Run(context.Background())
- if err != pilosa.ErrHostRequired {
+ if errors.Cause(err) != pilosa.ErrHostRequired {
t.Fatalf("Expect err: %s, actual err: %s", pilosa.ErrHostRequired, err)
}
diff --git a/ctl/check.go b/ctl/check.go
index 34b7033fc..04f947eb4 100644
--- a/ctl/check.go
+++ b/ctl/check.go
@@ -24,6 +24,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/roaring"
+ "github.com/pkg/errors"
)
// CheckCommand represents a command for performing consistency checks on data files.
@@ -48,17 +49,17 @@ func (cmd *CheckCommand) Run(ctx context.Context) error {
switch filepath.Ext(path) {
case "":
if err := cmd.checkBitmapFile(path); err != nil {
- return err
+ return errors.Wrap(err, "checking bitmap")
}
case ".cache":
if err := cmd.checkCacheFile(path); err != nil {
- return err
+ return errors.Wrap(err, "checking cache")
}
case ".snapshotting":
if err := cmd.checkSnapshotFile(path); err != nil {
- return err
+ return errors.Wrap(err, "checking snapshot")
}
}
}
@@ -71,26 +72,26 @@ func (cmd *CheckCommand) checkBitmapFile(path string) error {
// Open file handle.
f, err := os.Open(path)
if err != nil {
- return err
+ return errors.Wrap(err, "opening file")
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
- return err
+ return errors.Wrap(err, "statting file")
}
// Memory map the file.
data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
if err != nil {
- return err
+ return errors.Wrap(err, "mmapping")
}
defer syscall.Munmap(data)
// Attach the mmap file to the bitmap.
bm := roaring.NewBitmap()
if err := bm.UnmarshalBinary(data); err != nil {
- return err
+ return errors.Wrap(err, "unmarshalling")
}
// Perform consistency check.
diff --git a/ctl/check_test.go b/ctl/check_test.go
index 33a19feb3..56d70f97a 100644
--- a/ctl/check_test.go
+++ b/ctl/check_test.go
@@ -85,7 +85,7 @@ func TestCheckCommand_Run(t *testing.T) {
var buf bytes.Buffer
io.Copy(&buf, r)
- if !strings.HasPrefix(err.Error(), "invalid roaring file") {
+ if !strings.HasPrefix(err.Error(), "checking bitmap: unmarshalling: invalid roaring file") {
t.Fatalf("expect error: invalid roaring file, actual: '%s'", err)
}
// Todo: need correct roaring file for happy path
diff --git a/ctl/common.go b/ctl/common.go
index 3629e233e..11042a704 100644
--- a/ctl/common.go
+++ b/ctl/common.go
@@ -19,6 +19,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
+ "github.com/pkg/errors"
"github.com/spf13/pflag"
)
@@ -42,7 +43,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error
if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" {
cert, err := tls.LoadX509KeyPair(tlsConfig.CertificatePath, tlsConfig.CertificateKeyPath)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "loading keypair")
}
TLSConfig = &tls.Config{
Certificates: []tls.Certificate{cert},
@@ -51,7 +52,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error
}
client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig))
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "getting internal client")
}
return client, err
}
diff --git a/ctl/export.go b/ctl/export.go
index 8c219a078..546d1ae1b 100644
--- a/ctl/export.go
+++ b/ctl/export.go
@@ -22,6 +22,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
+ "github.com/pkg/errors"
)
// ExportCommand represents a command for bulk exporting data from a server.
@@ -68,7 +69,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
if cmd.Path != "" {
f, err := os.Create(cmd.Path)
if err != nil {
- return err
+ return errors.Wrap(err, "creating file")
}
defer f.Close()
@@ -78,7 +79,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
- return err
+ return errors.Wrap(err, "creating client")
}
// Determine slice count.
@@ -90,21 +91,21 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
}
if err != nil {
- return err
+ return errors.Wrap(err, "getting slice count")
}
// Export each slice.
for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ {
logger.Printf("exporting slice: %d", slice)
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, cmd.View, slice, w); err != nil {
- return err
+ return errors.Wrap(err, "exporting")
}
}
// Close writer, if applicable.
if w, ok := w.(io.Closer); ok {
if err := w.Close(); err != nil {
- return err
+ return errors.Wrap(err, "closing")
}
}
diff --git a/ctl/export_test.go b/ctl/export_test.go
index b414f711f..48a90a065 100644
--- a/ctl/export_test.go
+++ b/ctl/export_test.go
@@ -59,11 +59,7 @@ func TestExportCommand_Run(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
defer s.Close()
- uri, err := pilosa.NewURIFromAddress(s.Host())
- if err != nil {
- t.Fatal(err)
- }
- s.Handler.API.URI = *uri
+
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
@@ -75,8 +71,7 @@ func TestExportCommand_Run(t *testing.T) {
cm.Index = "i"
cm.Frame = "f"
cm.View = pilosa.ViewStandard
- err = cm.Run(context.Background())
- if err != nil {
+ if err := cm.Run(context.Background()); err != nil {
t.Fatalf("Export Run doesn't work: %s", err)
}
}
diff --git a/ctl/import.go b/ctl/import.go
index c3c117eb3..004e47e96 100644
--- a/ctl/import.go
+++ b/ctl/import.go
@@ -17,7 +17,6 @@ package ctl
import (
"context"
"encoding/csv"
- "errors"
"fmt"
"io"
"log"
@@ -28,6 +27,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
+ "github.com/pkg/errors"
)
// ImportCommand represents a command for bulk importing data.
@@ -94,14 +94,14 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
- return err
+ return errors.Wrap(err, "creating client")
}
cmd.Client = client
if cmd.CreateSchema {
err := cmd.ensureSchema(ctx)
if err != nil {
- return err
+ return errors.Wrap(err, "ensuring schema")
}
}
@@ -153,7 +153,7 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error {
// Open file for reading.
f, err := os.Open(path)
if err != nil {
- return err
+ return errors.Wrap(err, "opening file")
}
defer f.Close()
@@ -173,7 +173,7 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error {
if err == io.EOF {
break
} else if err != nil {
- return err
+ return errors.Wrap(err, "reading")
}
// Ignore blank rows.
@@ -243,7 +243,7 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err
logger.Printf("importing slice: %d, n=%d", slice, len(bits))
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, bits); err != nil {
- return err
+ return errors.Wrap(err, "importing")
}
}
@@ -260,7 +260,7 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error {
// Open file for reading.
f, err := os.Open(path)
if err != nil {
- return err
+ return errors.Wrap(err, "opening file")
}
defer f.Close()
@@ -280,7 +280,7 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error {
if err == io.EOF {
break
} else if err != nil {
- return err
+ return errors.Wrap(err, "reading")
}
// Ignore blank rows.
@@ -340,7 +340,7 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er
logger.Printf("importing keys: n=%d", len(bits))
if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, bits); err != nil {
- return err
+ return errors.Wrap(err, "importing keys")
}
return nil
@@ -356,7 +356,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
// Open file for reading.
f, err := os.Open(path)
if err != nil {
- return err
+ return errors.Wrap(err, "opening file")
}
defer f.Close()
@@ -376,7 +376,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
if err == io.EOF {
break
} else if err != nil {
- return err
+ return errors.Wrap(err, "reading")
}
// Ignore blank rows.
@@ -437,7 +437,7 @@ func (cmd *ImportCommand) importFieldValues(ctx context.Context, vals []pilosa.F
logger.Printf("importing slice: %d, n=%d", slice, len(vals))
if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Frame, cmd.Field, slice, vals); err != nil {
- return err
+ return errors.Wrap(err, "importing values")
}
}
diff --git a/ctl/import_test.go b/ctl/import_test.go
index 6cbc8271a..08eea8323 100644
--- a/ctl/import_test.go
+++ b/ctl/import_test.go
@@ -65,11 +65,7 @@ func TestImportCommand_Run(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
defer s.Close()
- uri, err := pilosa.NewURIFromAddress(s.Host())
- if err != nil {
- t.Fatal(err)
- }
- s.Handler.API.URI = *uri
+
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
@@ -103,12 +99,7 @@ func TestImportCommand_RunValue(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
defer s.Close()
- uri, err := pilosa.NewURIFromAddress(s.Host())
- if err != nil {
- t.Fatal(err)
- }
- s.Handler.API.URI = *uri
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
diff --git a/ctl/inspect.go b/ctl/inspect.go
index f5825c35a..0c38c3fb5 100644
--- a/ctl/inspect.go
+++ b/ctl/inspect.go
@@ -26,6 +26,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/roaring"
+ "github.com/pkg/errors"
)
// InspectCommand represents a command for inspecting fragment data files.
@@ -49,19 +50,19 @@ func (cmd *InspectCommand) Run(ctx context.Context) error {
// Open file handle.
f, err := os.Open(cmd.Path)
if err != nil {
- return err
+ return errors.Wrap(err, "opening file")
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
- return err
+ return errors.Wrap(err, "statting file")
}
// Memory map the file.
data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
if err != nil {
- return err
+ return errors.Wrap(err, "mmapping")
}
defer syscall.Munmap(data)
@@ -70,7 +71,7 @@ func (cmd *InspectCommand) Run(ctx context.Context) error {
fmt.Fprintf(cmd.Stderr, "unmarshaling bitmap...")
bm := roaring.NewBitmap()
if err := bm.UnmarshalBinary(data); err != nil {
- return err
+ return errors.Wrap(err, "unmarshalling")
}
fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
diff --git a/ctl/restore.go b/ctl/restore.go
index 89ee8ac8d..ef863096e 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -16,12 +16,12 @@ package ctl
import (
"context"
- "errors"
"io"
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
+ "github.com/pkg/errors"
)
// RestoreCommand represents a command for restoring a frame from a backup.
@@ -60,19 +60,19 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
- return err
+ return errors.Wrap(err, "creating client")
}
// Open backup file.
f, err := os.Open(cmd.Path)
if err != nil {
- return err
+ return errors.Wrap(err, "opening file")
}
defer f.Close()
// Restore backup file to the cluster.
if err := client.RestoreFrom(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
- return err
+ return errors.Wrap(err, "restoring")
}
return nil
diff --git a/ctl/restore_test.go b/ctl/restore_test.go
index 04cb30415..896db57c5 100644
--- a/ctl/restore_test.go
+++ b/ctl/restore_test.go
@@ -48,11 +48,7 @@ func TestRestoreCommand_Run(t *testing.T) {
s := test.NewServer()
defer s.Close()
- uri, err := pilosa.NewURIFromAddress(s.Host())
- if err != nil {
- t.Fatal(err)
- }
- s.Handler.API.URI = *uri
+
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
diff --git a/diagnostics.go b/diagnostics.go
index bfebbb495..46ffc2110 100644
--- a/diagnostics.go
+++ b/diagnostics.go
@@ -23,6 +23,8 @@ import (
"strings"
"sync"
"time"
+
+ "github.com/pkg/errors"
)
// Default version check URL.
@@ -80,13 +82,13 @@ func (d *DiagnosticsCollector) Flush() error {
d.metrics["Uptime"] = (time.Now().Unix() - d.startTime)
buf, err := d.encode()
if err != nil {
- return err
+ return errors.Wrap(err, "encoding")
}
req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf))
req.Header.Set("Content-Type", "application/json")
resp, err := d.client.Do(req)
if err != nil {
- return err
+ return errors.Wrap(err, "posting")
}
// Intentionally ignoring response body, as user does not need to be notified of error.
defer resp.Body.Close()
@@ -99,7 +101,7 @@ func (d *DiagnosticsCollector) CheckVersion() error {
req, err := http.NewRequest("GET", d.VersionURL, nil)
resp, err := d.client.Do(req)
if err != nil {
- return err
+ return errors.Wrap(err, "getting version")
}
defer resp.Body.Close()
diff --git a/docs/query-language.md b/docs/query-language.md
index 8ed0d5439..7d9f22886 100644
--- a/docs/query-language.md
+++ b/docs/query-language.md
@@ -160,8 +160,8 @@ Delete url value for repo 10.
**Spec:**
```
-SetBit(, , ,
- [timestamp=TIMESTAMP])
+ClearBit(, , ,
+ [timestamp=TIMESTAMP])
```
**Description:**
diff --git a/executor.go b/executor.go
index 59e40511d..b6dc08d41 100644
--- a/executor.go
+++ b/executor.go
@@ -16,7 +16,6 @@ package pilosa
import (
"context"
- "errors"
"fmt"
"net/http"
"sort"
@@ -24,6 +23,7 @@ import (
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
+ "github.com/pkg/errors"
)
// DefaultFrame is the frame used if one is not specified.
@@ -149,7 +149,7 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// executeCall executes a call.
func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) {
if err := e.validateCallArgs(c); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "validating args")
}
indexTag := fmt.Sprintf("index:%s", index)
// Special handling for mutation and top-n calls.
@@ -344,7 +344,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil {
attrs, err := idx.ColumnAttrStore().Attrs(columnID)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "getting column attrs")
}
bm.Attrs = attrs
} else if err != nil {
@@ -354,11 +354,11 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
if fr := idx.Frame(frame); fr != nil {
rowID, _, err := c.UintArg(rowLabel)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "getting row")
}
attrs, err := fr.RowAttrStore().Attrs(rowID)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "getting row attrs")
}
bm.Attrs = attrs
}
@@ -400,7 +400,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
- return ValCount{}, err
+ return ValCount{}, errors.Wrap(err, "executing bitmap call")
}
filter = bm
}
@@ -425,7 +425,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq
vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth())
if err != nil {
- return ValCount{}, err
+ return ValCount{}, errors.Wrap(err, "computing sum")
}
return ValCount{
Val: int64(vsum) + (int64(vcount) * field.Min),
@@ -527,7 +527,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s
// Execute original query.
pairs, err := e.executeTopNSlices(ctx, index, c, slices, opt)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "finding top results")
}
// If this call is against specific ids, or we didn't get results,
@@ -544,7 +544,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s
trimmedList, err := e.executeTopNSlices(ctx, index, other, slices, opt)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "retrieving full counts")
}
if n != 0 && int(n) < len(trimmedList) {
@@ -883,7 +883,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
predicates, err := cond.IntSliceValue()
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "getting condition value")
}
// Only support two integers for the between operation.
@@ -1573,7 +1573,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64,
// Start mapping across all primary owners.
if err := e.mapper(ctx, ch, nodes, index, slices, c, opt, mapFn, reduceFn); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "starting mapper")
}
// Iterate over all map responses and reduce.
diff --git a/fragment_test.go b/fragment_test.go
index d7c97a5cf..ac082512b 100644
--- a/fragment_test.go
+++ b/fragment_test.go
@@ -1171,3 +1171,68 @@ func BenchmarkFragment_Snapshot(b *testing.B) {
}
}
}
+func BenchmarkFragment_FullSnapshot(b *testing.B) {
+ f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+ defer f.Close()
+ // Generate some intersecting data.
+ maxX := 1048576 / 2
+ sz := maxX
+ rows := make([]uint64, sz, sz)
+ cols := make([]uint64, sz, sz)
+
+ max := 0
+ for row := 0; row < 100; row++ {
+ val := 1
+ i := 0
+ for col := 0; col < SliceWidth/2; col++ {
+ rows[i] = uint64(row)
+ cols[i] = uint64(val)
+ val += 2
+ i++
+ }
+ if err := f.Import(rows, cols); err != nil {
+ b.Fatalf("Error Building Sample: %s", err)
+ }
+ if row > max {
+ max = row
+ }
+ }
+
+ b.ResetTimer()
+ b.ReportAllocs()
+
+ for i := 0; i < b.N; i++ {
+ if err := f.Snapshot(); err != nil {
+ b.Fatal(err)
+ }
+ }
+}
+
+func BenchmarkFragment_Import(b *testing.B) {
+ f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+ defer f.Close()
+ maxX := 1048576 * 5 * 2
+ sz := maxX
+ rows := make([]uint64, sz, sz)
+ cols := make([]uint64, sz, sz)
+ i := 0
+ for row := 0; row < 100; row++ {
+ val := 1
+ for col := 0; col < SliceWidth/2; col++ {
+ rows[i] = uint64(row)
+ cols[i] = uint64(val)
+ val += 2
+ i++
+ }
+ if i == maxX {
+ break
+ }
+ }
+ b.ResetTimer()
+ b.ReportAllocs()
+ for i := 0; i < b.N; i++ {
+ if err := f.Import(rows, cols); err != nil {
+ b.Fatalf("Error Building Sample: %s", err)
+ }
+ }
+}
diff --git a/frame.go b/frame.go
index 0c87210bf..3de3f3613 100644
--- a/frame.go
+++ b/frame.go
@@ -834,7 +834,11 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
// Split import data by fragment.
dataByFragment := make(map[importKey]importData)
for i := range rowIDs {
- rowID, columnID, timestamp := rowIDs[i], columnIDs[i], timestamps[i]
+ rowID, columnID := rowIDs[i], columnIDs[i]
+ var timestamp *time.Time
+ if len(timestamps) > i {
+ timestamp = timestamps[i]
+ }
var standard, inverse []string
if timestamp == nil {
diff --git a/gossip/gossip.go b/gossip/gossip.go
index 25dd40cc4..2da6e3e4c 100644
--- a/gossip/gossip.go
+++ b/gossip/gossip.go
@@ -169,12 +169,14 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption {
// NewGossipMemberSet returns a new instance of GossipMemberSet based on options.
func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventReceiver, sh pilosa.StatusHandler, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
- g := &GossipMemberSet{}
+ g := &GossipMemberSet{
+ Logger: pilosa.NopLogger,
+ }
// options
for _, opt := range options {
if err := opt(g); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "executing option")
}
}
diff --git a/handler.go b/handler.go
index 7bf11af1b..b3903ef6f 100644
--- a/handler.go
+++ b/handler.go
@@ -126,6 +126,7 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups)
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
+ router.HandleFunc("/info", handler.handleGetInfo).Methods("GET")
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET")
router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST")
@@ -252,6 +253,13 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
}
}
+func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) {
+ info := h.API.Info()
+ if err := json.NewEncoder(w).Encode(info); err != nil {
+ h.Logger.Printf("write info response error: %s", err)
+ }
+}
+
type getSchemaResponse struct {
Indexes []*IndexInfo `json:"indexes"`
}
@@ -349,7 +357,7 @@ func (p *postIndexRequest) UnmarshalJSON(b []byte) error {
// m is an overflow map used to capture additional, unexpected keys.
m := make(map[string]interface{})
if err := json.Unmarshal(b, &m); err != nil {
- return err
+ return errors.Wrap(err, "unmarshalling unexpected values")
}
validIndexOptions := getValidOptions(IndexOptions{})
@@ -360,7 +368,7 @@ func (p *postIndexRequest) UnmarshalJSON(b []byte) error {
// Unmarshal expected values.
var _p _postIndexRequest
if err := json.Unmarshal(b, &_p); err != nil {
- return err
+ return errors.Wrap(err, "unmarshalling expected values")
}
p.Options = _p.Options
@@ -526,7 +534,7 @@ func (p *postFrameRequest) UnmarshalJSON(b []byte) error {
// m is an overflow map used to capture additional, unexpected keys.
m := make(map[string]interface{})
if err := json.Unmarshal(b, &m); err != nil {
- return err
+ return errors.Wrap(err, "unmarshaling unexpected keys")
}
validFrameOptions := getValidOptions(FrameOptions{})
@@ -538,7 +546,7 @@ func (p *postFrameRequest) UnmarshalJSON(b []byte) error {
// Unmarshal expected values.
var _p _postFrameRequest
if err := json.Unmarshal(b, &_p); err != nil {
- return err
+ return errors.Wrap(err, "unmarshalling expected keys")
}
p.Options = _p.Options
@@ -813,13 +821,13 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, erro
// Slurp the body.
body, err := ioutil.ReadAll(r.Body)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "reading")
}
// Unmarshal into object.
var req internal.QueryRequest
if err := proto.Unmarshal(body, &req); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "unmarshalling")
}
return decodeQueryRequest(&req), nil
@@ -832,7 +840,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
// Parse query string.
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "reading")
}
query := string(buf)
@@ -877,9 +885,9 @@ func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, res
// writeProtobufQueryResponse writes the response from the executor to w as protobuf.
func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *QueryResponse) error {
if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil {
- return err
+ return errors.Wrap(err, "marshalling")
} else if _, err := w.Write(buf); err != nil {
- return err
+ return errors.Wrap(err, "writing")
}
return nil
}
@@ -1327,7 +1335,7 @@ func parseUint64Slice(s string) ([]uint64, error) {
// Parse number.
num, err := strconv.ParseUint(str, 10, 64)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "parsing int")
}
a = append(a, num)
}
@@ -1591,7 +1599,7 @@ func GetTimeStamp(data map[string]interface{}, timeField string) (int64, error)
v, err := time.Parse(TimeFormat, timestamp)
if err != nil {
- return 0, err
+ return 0, errors.Wrap(err, "parsing timestamp")
}
return v.Unix(), nil
diff --git a/handler_test.go b/handler_test.go
index 9dfb801af..36fb77b42 100644
--- a/handler_test.go
+++ b/handler_test.go
@@ -19,6 +19,7 @@ import (
"context"
"encoding/json"
"errors"
+ "fmt"
"io"
"io/ioutil"
"net/http"
@@ -154,6 +155,20 @@ func TestHandler_Status(t *testing.T) {
}
}
+func TestHandler_Info(t *testing.T) {
+ s := test.NewServer()
+ defer s.Close()
+ h := test.NewHandler()
+
+ w := httptest.NewRecorder()
+ h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil))
+ if w.Code != http.StatusOK {
+ t.Fatalf("unexpected status code: %d", w.Code)
+ } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", SliceWidth) {
+ t.Fatalf("unexpected body: %s", body)
+ }
+}
+
// Ensure the handler can abort a cluster resize.
func TestHandler_ClusterResizeAbort(t *testing.T) {
diff --git a/holder.go b/holder.go
index f048aac44..2475d61d0 100644
--- a/holder.go
+++ b/holder.go
@@ -96,19 +96,19 @@ func (h *Holder) Open() error {
h.Logger.Printf("open holder path: %s", h.Path)
if err := os.MkdirAll(h.Path, 0777); err != nil {
- return err
+ return errors.Wrap(err, "creating directory")
}
// Open path to read all index directories.
f, err := os.Open(h.Path)
if err != nil {
- return err
+ return errors.Wrap(err, "opening directory")
}
defer f.Close()
fis, err := f.Readdir(0)
if err != nil {
- return err
+ return errors.Wrap(err, "reading directory")
}
for _, fi := range fis {
@@ -123,7 +123,7 @@ func (h *Holder) Open() error {
h.Logger.Printf("ERROR opening index: %s, err=%s", fi.Name(), err)
continue
} else if err != nil {
- return err
+ return errors.Wrap(err, "opening index")
}
if err := index.Open(); err != nil {
if err == ErrName {
@@ -158,7 +158,7 @@ func (h *Holder) Close() error {
for _, index := range h.indexes {
if err := index.Close(); err != nil {
- return err
+ return errors.Wrap(err, "closing index")
}
}
return nil
@@ -245,20 +245,20 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error {
opt := IndexOptions{}
idx, err := h.CreateIndexIfNotExists(index.Name, opt)
if err != nil {
- return err
+ return errors.Wrap(err, "creating index")
}
// Create frames that don't exist.
for _, f := range index.Frames {
opt := decodeFrameOptions(f.Meta)
frame, err := idx.CreateFrameIfNotExists(f.Name, *opt)
if err != nil {
- return err
+ return errors.Wrap(err, "creating frame")
}
// Create views that don't exist.
for _, v := range f.Views {
_, err := frame.CreateViewIfNotExists(v)
if err != nil {
- return err
+ return errors.Wrap(err, "creating view")
}
}
}
@@ -347,11 +347,11 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
// Otherwise create a new index.
index, err := h.newIndex(h.IndexPath(name), name)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "creating")
}
if err := index.Open(); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "opening")
}
// Update options.
@@ -387,12 +387,12 @@ func (h *Holder) DeleteIndex(name string) error {
// Close index.
if err := index.Close(); err != nil {
- return err
+ return errors.Wrap(err, "closing")
}
// Delete index directory.
if err := os.RemoveAll(h.IndexPath(name)); err != nil {
- return err
+ return errors.Wrap(err, "removing directory")
}
// Remove reference.
@@ -528,7 +528,7 @@ func (h *Holder) loadNodeID() (string, error) {
nodeID := ""
h.Logger.Printf("load NodeID: %s", idPath)
if err := os.MkdirAll(h.Path, 0777); err != nil {
- return "", err
+ return "", errors.Wrap(err, "creating directory")
}
nodeIDBytes, err := ioutil.ReadFile(idPath)
@@ -538,10 +538,10 @@ func (h *Holder) loadNodeID() (string, error) {
nodeID = uuid.NewV4().String()
err = ioutil.WriteFile(idPath, []byte(nodeID), 0600)
if err != nil {
- return "", err
+ return "", errors.Wrap(err, "writing file")
}
} else if err != nil {
- return "", err
+ return "", errors.Wrap(err, "reading file")
}
return nodeID, nil
@@ -667,7 +667,7 @@ func (s *HolderSyncer) syncIndex(index string) error {
// Read block checksums.
blks, err := idx.ColumnAttrStore().Blocks()
if err != nil {
- return err
+ return errors.Wrap(err, "getting blocks")
}
s.Stats.CountWithCustomTags("ColumnAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag})
@@ -679,7 +679,7 @@ func (s *HolderSyncer) syncIndex(index string) error {
// Skip update and recomputation if no attributes have changed.
m, err := client.ColumnAttrDiff(context.Background(), index, blks)
if err != nil {
- return err
+ return errors.Wrap(err, "getting differing blocks")
} else if len(m) == 0 {
continue
}
@@ -687,13 +687,13 @@ func (s *HolderSyncer) syncIndex(index string) error {
// Update local copy.
if err := idx.ColumnAttrStore().SetBulkAttrs(m); err != nil {
- return err
+ return errors.Wrap(err, "setting attrs")
}
// Recompute blocks.
blks, err = idx.ColumnAttrStore().Blocks()
if err != nil {
- return err
+ return errors.Wrap(err, "recomputing blocks")
}
}
@@ -713,7 +713,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
// Read block checksums.
blks, err := f.RowAttrStore().Blocks()
if err != nil {
- return err
+ return errors.Wrap(err, "getting blocks")
}
s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag})
@@ -727,7 +727,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
if err == ErrFrameNotFound {
continue // frame not created remotely yet, skip
} else if err != nil {
- return err
+ return errors.Wrap(err, "getting differing blocks")
} else if len(m) == 0 {
continue
}
@@ -735,13 +735,13 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
// Update local copy.
if err := f.RowAttrStore().SetBulkAttrs(m); err != nil {
- return err
+ return errors.Wrap(err, "setting attrs")
}
// Recompute blocks.
blks, err = f.RowAttrStore().Blocks()
if err != nil {
- return err
+ return errors.Wrap(err, "recomputing blocks")
}
}
@@ -759,13 +759,13 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err
// Ensure view exists locally.
v, err := f.CreateViewIfNotExists(view)
if err != nil {
- return err
+ return errors.Wrap(err, "creating view")
}
// Ensure fragment exists locally.
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
- return err
+ return errors.Wrap(err, "creating fragment")
}
// Sync fragments together.
@@ -777,7 +777,7 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err
RemoteClient: s.RemoteClient,
}
if err := fs.SyncFragment(); err != nil {
- return err
+ return errors.Wrap(err, "syncing fragment")
}
return nil
@@ -827,7 +827,7 @@ func (c *HolderCleaner) CleanHolder() error {
}
// Delete fragment.
if err := view.DeleteFragment(fragSlice); err != nil {
- return err
+ return errors.Wrap(err, "deleting fragment")
}
}
}
diff --git a/holder_test.go b/holder_test.go
index e819c6e94..b197f0d01 100644
--- a/holder_test.go
+++ b/holder_test.go
@@ -86,7 +86,7 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
}
- if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=test, err=invalid database") {
+ if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=test, err=opening storage: invalid database") {
t.Fatalf("unexpected error: %s", err)
}
})
@@ -145,7 +145,7 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
}
- if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=open frame: name=bar, err=invalid database") {
+ if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=open frame: name=bar, err=opening storage: invalid database") {
t.Fatalf("unexpected error: %s", err)
}
})
diff --git a/index.go b/index.go
index 00ee98f11..dc676c304 100644
--- a/index.go
+++ b/index.go
@@ -642,7 +642,8 @@ func (i *Index) openInputDefinitions() error {
// InputBits Process the []Bit though the Frame import process
func (i *Index) InputBits(frame string, bits []*Bit) error {
var rowIDs, columnIDs []uint64
- timestamps := make([]*time.Time, len(bits))
+ var timestamps []*time.Time
+
f := i.Frame(frame)
if f == nil {
return fmt.Errorf("Frame not found: %s", frame)
@@ -657,6 +658,11 @@ func (i *Index) InputBits(frame string, bits []*Bit) error {
// Convert timestamps to time.Time.
if bit.Timestamp > 0 {
+ // Don't create a full timestamps slice unless
+ // at least one bit contains a timestamp.
+ if len(timestamps) == 0 {
+ timestamps = make([]*time.Time, len(bits))
+ }
t := time.Unix(bit.Timestamp, 0)
timestamps[i] = &t
}
diff --git a/server.go b/server.go
index 52999b1c5..e739fe5f0 100644
--- a/server.go
+++ b/server.go
@@ -22,6 +22,7 @@ import (
"net/http"
"os"
"os/exec"
+ "path/filepath"
"runtime"
"strconv"
"strings"
@@ -77,6 +78,7 @@ type Server struct {
maxWritesPerRequest int
defaultClient InternalClient
+ dataDir string
}
// ServerOption is a functional option type for pilosa.Server
@@ -98,8 +100,7 @@ func OptServerReplicaN(n int) ServerOption {
func OptServerDataDir(dir string) ServerOption {
return func(s *Server) error {
- s.Cluster.Path = dir
- s.Holder.Path = dir
+ s.dataDir = dir
return nil
}
}
@@ -231,9 +232,16 @@ func NewServer(opts ...ServerOption) (*Server, error) {
}
}
+ path, err := expandDirName(s.dataDir)
+ if err != nil {
+ return nil, err
+ }
+
+ s.Holder.Path = path
s.Holder.Logger = s.logger
s.Holder.Stats.SetLogger(s.logger)
+ s.Cluster.Path = path
s.Cluster.Logger = s.logger
s.Cluster.Holder = s.Holder
@@ -292,7 +300,6 @@ func (s *Server) Open() error {
s.handler.API.Broadcaster = s.Broadcaster
s.handler.API.BroadcastHandler = s
s.handler.API.StatusHandler = s
- s.handler.API.URI = s.URI
s.handler.API.Cluster = s.Cluster
// Initialize Holder.
@@ -622,7 +629,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
// Sync schema.
if err := s.Holder.ApplySchema(ns.Schema); err != nil {
- return err
+ return errors.Wrap(err, "applying schema")
}
// Sync maxSlices (standard).
@@ -784,3 +791,15 @@ type StatusHandler interface {
ClusterStatus() (proto.Message, error)
HandleRemoteStatus(proto.Message) error
}
+
+func expandDirName(path string) (string, error) {
+ prefix := "~" + string(filepath.Separator)
+ if strings.HasPrefix(path, prefix) {
+ HomeDir := os.Getenv("HOME")
+ if HomeDir == "" {
+ return "", errors.New("data directory not specified and no home dir available")
+ }
+ return filepath.Join(HomeDir, strings.TrimPrefix(path, prefix)), nil
+ }
+ return path, nil
+}
diff --git a/server/server.go b/server/server.go
index 8fd699ce5..eb660fc23 100644
--- a/server/server.go
+++ b/server/server.go
@@ -28,9 +28,7 @@ import (
"net/http"
"os"
"os/signal"
- "path/filepath"
"strconv"
- "strings"
"syscall"
"time"
@@ -93,30 +91,22 @@ func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command {
// Start starts the pilosa server - it returns once the server is running.
func (m *Command) Start() (err error) {
defer close(m.Started)
- prefix := "~" + string(filepath.Separator)
- if strings.HasPrefix(m.Config.DataDir, prefix) {
- HomeDir := os.Getenv("HOME")
- if HomeDir == "" {
- return errors.New("data directory not specified and no home dir available")
- }
- m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix))
- }
// SetupServer
err = m.SetupServer()
if err != nil {
- return err
+ return errors.Wrap(err, "setting up server")
}
// SetupNetworking
err = m.SetupNetworking()
if err != nil {
- return err
+ return errors.Wrap(err, "setting up networking")
}
// Initialize server.
if err = m.Server.Open(); err != nil {
- return fmt.Errorf("server.Open: %v", err)
+ return errors.Wrap(err, "opening server")
}
m.logger.Printf("Listening as %s\n", m.Server.URI)
@@ -271,7 +261,7 @@ func (m *Command) SetupNetworking() error {
for _, address := range m.Config.Cluster.Hosts {
uri, err := pilosa.NewURIFromAddress(address)
if err != nil {
- return err
+ return errors.Wrap(err, "getting URI")
}
m.Server.Cluster.Nodes = append(m.Server.Cluster.Nodes, &pilosa.Node{
URI: *uri,
@@ -287,7 +277,7 @@ func (m *Command) SetupNetworking() error {
gossipPort, err := strconv.Atoi(m.Config.Gossip.Port)
if err != nil {
- return err
+ return errors.Wrap(err, "parsing port")
}
// get the host portion of addr to use for binding
@@ -298,7 +288,7 @@ func (m *Command) SetupNetworking() error {
} else {
transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger())
if err != nil {
- return err
+ return errors.Wrap(err, "getting transport")
}
}
@@ -310,10 +300,19 @@ func (m *Command) SetupNetworking() error {
gossipEventReceiver := gossip.NewGossipEventReceiver(m.logger)
m.Server.Cluster.EventReceiver = gossipEventReceiver
- gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, gossipEventReceiver, m.Server, gossip.WithLogger(m.logger.Logger()), gossip.WithTransport(transport))
+ gossipMemberSet, err := gossip.NewGossipMemberSet(
+ m.Server.NodeID,
+ m.Server.URI.Host(),
+ m.Config.Gossip,
+ gossipEventReceiver,
+ m.Server,
+ gossip.WithLogger(m.logger.Logger()),
+ gossip.WithTransport(transport),
+ )
if err != nil {
- return err
+ return errors.Wrap(err, "getting memberset")
}
+ gossipMemberSet.Logger = m.logger
m.Server.Cluster.MemberSet = gossipMemberSet
m.Server.Broadcaster = m.Server
m.Server.BroadcastReceiver = gossipMemberSet
diff --git a/test/handler.go b/test/handler.go
index c325d3243..50b081812 100644
--- a/test/handler.go
+++ b/test/handler.go
@@ -75,13 +75,6 @@ func NewServer() *Server {
}
s.Server = httptest.NewServer(s.Handler.Handler)
- // Update handler to use hostname.
- uri, err := pilosa.NewURIFromAddress(s.Host())
- if err != nil {
- panic(err)
- }
- s.Handler.API.URI = *uri
-
// Handler test messages can no-op.
s.Handler.API.Broadcaster = pilosa.NopBroadcaster
// Create a default cluster on the handler
diff --git a/uri.go b/uri.go
index beb5060b5..dfb82baef 100644
--- a/uri.go
+++ b/uri.go
@@ -16,13 +16,13 @@ package pilosa
import (
"encoding/json"
- "errors"
"fmt"
"regexp"
"strconv"
"strings"
"github.com/pilosa/pilosa/internal"
+ "github.com/pkg/errors"
)
var schemeRegexp = regexp.MustCompile("^[+a-z]+$")
@@ -72,7 +72,7 @@ func NewURIFromHostPort(host string, port uint16) (*URI, error) {
uri := DefaultURI()
err := uri.SetHost(host)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "setting uri host")
}
uri.SetPort(port)
return uri, nil
diff --git a/view.go b/view.go
index dbcc73b8d..4f2ae1cd7 100644
--- a/view.go
+++ b/view.go
@@ -24,6 +24,7 @@ import (
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
+ "github.com/pkg/errors"
)
// View layout modes.
@@ -105,13 +106,13 @@ func (v *View) Open() error {
if err := func() error {
// Ensure the view's path exists.
if err := os.MkdirAll(v.path, 0777); err != nil {
- return err
+ return errors.Wrap(err, "creating view directory")
} else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil {
- return err
+ return errors.Wrap(err, "creating fragments directory")
}
if err := v.openFragments(); err != nil {
- return err
+ return errors.Wrap(err, "opening fragments")
}
return nil
@@ -129,13 +130,13 @@ func (v *View) openFragments() error {
if os.IsNotExist(err) {
return nil
} else if err != nil {
- return err
+ return errors.Wrap(err, "opening fragments directory")
}
defer file.Close()
fis, err := file.Readdir(0)
if err != nil {
- return err
+ return errors.Wrap(err, "reading fragments directory")
}
for _, fi := range fis {
@@ -168,7 +169,7 @@ func (v *View) Close() error {
// Close all fragments.
for _, frag := range v.fragments {
if err := frag.Close(); err != nil {
- return err
+ return errors.Wrap(err, "closing fragment")
}
}
v.fragments = make(map[uint64]*Fragment)
@@ -240,7 +241,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
// Initialize and open fragment.
frag := v.newFragment(v.FragmentPath(slice), slice)
if err := frag.Open(); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "opening fragment")
}
frag.RowAttrStore = v.RowAttrStore
@@ -256,7 +257,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
IsInverse: IsInverseView(v.name),
})
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "sending message")
}
}
@@ -286,12 +287,12 @@ func (v *View) DeleteFragment(slice uint64) error {
// Close data files before deletion.
if err := fragment.Close(); err != nil {
- return err
+ return errors.Wrap(err, "closing fragment")
}
// Delete fragment file.
if err := os.Remove(fragment.Path()); err != nil {
- return err
+ return errors.Wrap(err, "deleting fragment file")
}
// Delete fragment cache file.