From 1e816b401cc65b99b3d8bfa004a20256ad0e7c97 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 27 Jan 2017 13:16:27 -0600 Subject: [PATCH 01/61] Adds Todd's performance improvements: - Ingore asserts in fragment container. - Only log queries that take longer than 90 seconds. TODO: - address the TODOs that make the asserts configurable. --- fragment.go | 16 +++++++++++----- handler.go | 5 ++++- roaring/roaring.go | 24 ++++++++++++++---------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/fragment.go b/fragment.go index 1181f1739..aaef9323a 100644 --- a/fragment.go +++ b/fragment.go @@ -350,6 +350,11 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) return false, err } + // Don't update the cache if nothing changed. + if !changed { + return changed, nil + } + // Invalidate block checksum. delete(f.checksums, int(bitmapID/HashBlockSize)) @@ -389,6 +394,11 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (bool, error) { return false, err } + // Don't update the cache if nothing changed. + if !changed { + return changed, nil + } + // Invalidate block checksum. delete(f.checksums, int(bitmapID/HashBlockSize)) @@ -838,7 +848,6 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { // Process every bit. // If an error occurs then reopen the storage. lastID := uint64(0) - bmCounter := 0 if err := func() error { set := make(map[uint64]struct{}) for i := range bitmapIDs { @@ -851,7 +860,7 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { } // Write to storage. - changed, err := f.storage.Add(pos) + _, err = f.storage.Add(pos) if err != nil { return err } @@ -863,9 +872,6 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { lastID = bitmapID set[bitmapID] = struct{}{} } - if changed { - bmCounter += 1 - } // Invalidate block checksum. delete(f.checksums, int(bitmapID/HashBlockSize)) diff --git a/handler.go b/handler.go index 175161384..9a704e6c8 100644 --- a/handler.go +++ b/handler.go @@ -202,7 +202,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) } - h.logger().Printf("%s %s %.03fs", r.Method, r.URL.String(), time.Since(t).Seconds()) + dif := time.Since(t).Seconds() + if dif > 90 { + h.logger().Printf("%s %s %.03fs", r.Method, r.URL.String(), dif) + } } // handleGetSchema handles GET /schema requests. diff --git a/roaring/roaring.go b/roaring/roaring.go index 44b9c9341..efbf48374 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -460,8 +460,9 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { c := b.containers[i] // Verify container count before writing. - count := c.count() - assert(c.count() == c.n, "cannot write container count, mismatch: count=%d, n=%d", count, c.n) + // TODO: instead of commenting this out, we need to make it a configuration option + //count := c.count() + //assert(c.count() == c.n, "cannot write container count, mismatch: count=%d, n=%d", count, c.n) binary.LittleEndian.PutUint64(buf[headerSize+i*12:], uint64(key)) binary.LittleEndian.PutUint32(buf[headerSize+i*12+8:], uint32(c.n-1)) @@ -532,9 +533,10 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { c := b.containers[i] if c.n <= ArrayMaxSize { c.array = (*[0xFFFFFFF]uint32)(unsafe.Pointer(&data[offset]))[:c.n] - for _, v := range c.array { - assert(lowbits(uint64(v)) == v, "array value out of range: %d", v) - } + // TODO: instead of commenting this out, we need to make it a configuration option + //for _, v := range c.array { + // assert(lowbits(uint64(v)) == v, "array value out of range: %d", v) + //} opsOffset = int(offset) + len(c.array)*4 } else { c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN] @@ -542,8 +544,9 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { } // Verify container count on load. - count := c.count() - assert(c.count() == c.n, "container count mismatch: count=%d, n=%d", count, c.n) + // TODO: instead of commenting this out, we need to make it a configuration option + //count := c.count() + //assert(c.count() == c.n, "container count mismatch: count=%d, n=%d", count, c.n) } // Read ops log until the end of the file. @@ -1074,9 +1077,10 @@ func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) { } // Verify all elements are valid. - for _, v := range c.array { - assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v) - } + // TODO: instead of commenting this out, we need to make it a configuration option + //for _, v := range c.array { + // assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v) + //} nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:4*c.n]) return int64(nn), err From e3f4eab5ac05007fead5d4e7316fe75efd3d83e4 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 27 Jan 2017 18:42:47 -0500 Subject: [PATCH 02/61] removed maxslice from setbit path --- executor.go | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/executor.go b/executor.go index 3a8152ff5..f97c35c79 100644 --- a/executor.go +++ b/executor.go @@ -52,13 +52,15 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices // If slices aren't specified, then include all of them. if len(slices) == 0 { - // Round up the number of slices. - maxSlice := e.Index.DB(db).MaxSlice() + if needsSlices(q.Calls){ + // Round up the number of slices. +maxSlice := e.Index.DB(db).MaxSlice() - // Generate a slices of all slices. - slices = make([]uint64, maxSlice+1) - for i := range slices { - slices[i] = uint64(i) + // Generate a slices of all slices. + slices = make([]uint64, maxSlice+1) + for i := range slices { + slices[i] = uint64(i) + } } } @@ -876,3 +878,21 @@ func hasOnlySetBitmapAttrs(calls pql.Calls) bool { } return true } + +func needsSlices(calls pql.Calls) bool { + if len(calls) == 0 { + return false + } + + for _, call := range calls { + if _, ok := call.(pql.BitmapCall); !ok { + return true + }else if _, ok := call.(*pql.Count); !ok { + return true + }else if _, ok := call.(*pql.TopN); !ok { + return true + } + + } + return false +} From 7105c89398904a7a429fab1c9190eeceabb2e413 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 27 Jan 2017 17:48:44 -0600 Subject: [PATCH 03/61] run gofmt on the previous commit --- executor.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/executor.go b/executor.go index f97c35c79..6f213bafa 100644 --- a/executor.go +++ b/executor.go @@ -52,15 +52,15 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices // If slices aren't specified, then include all of them. if len(slices) == 0 { - if needsSlices(q.Calls){ + if needsSlices(q.Calls) { // Round up the number of slices. -maxSlice := e.Index.DB(db).MaxSlice() + maxSlice := e.Index.DB(db).MaxSlice() - // Generate a slices of all slices. - slices = make([]uint64, maxSlice+1) - for i := range slices { - slices[i] = uint64(i) - } + // Generate a slices of all slices. + slices = make([]uint64, maxSlice+1) + for i := range slices { + slices[i] = uint64(i) + } } } @@ -880,19 +880,19 @@ func hasOnlySetBitmapAttrs(calls pql.Calls) bool { } func needsSlices(calls pql.Calls) bool { - if len(calls) == 0 { - return false - } + if len(calls) == 0 { + return false + } for _, call := range calls { if _, ok := call.(pql.BitmapCall); !ok { return true - }else if _, ok := call.(*pql.Count); !ok { + } else if _, ok := call.(*pql.Count); !ok { return true - }else if _, ok := call.(*pql.TopN); !ok { + } else if _, ok := call.(*pql.TopN); !ok { return true } } - return false + return false } From 61da4a1001355875382817cb94df3cdd0bf882db Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 1 Feb 2017 13:40:12 -0500 Subject: [PATCH 04/61] limit the number of connections to single host --- cmd/pilosa/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go index 24d2a9fec..8513a5ce3 100644 --- a/cmd/pilosa/main.go +++ b/cmd/pilosa/main.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "math/rand" + "net/http" "os" "os/signal" "path/filepath" @@ -34,6 +35,7 @@ const ( ) func main() { + http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 64 m := NewMain() fmt.Fprintf(m.Stderr, "Pilosa %s\n", Build) From a35dd5e5861db6d9059dffa4963b7cd673692ebd Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 1 Feb 2017 12:57:06 -0600 Subject: [PATCH 05/61] add comments to the MaxIdleConnsPerHost tweak --- cmd/pilosa/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go index 8513a5ce3..4d19ef808 100644 --- a/cmd/pilosa/main.go +++ b/cmd/pilosa/main.go @@ -35,7 +35,10 @@ const ( ) func main() { + // Limit the number of connections that a server can make to a single node + // in order to prevent a cluster storm. http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 64 + m := NewMain() fmt.Fprintf(m.Stderr, "Pilosa %s\n", Build) From 631b3bc44c326cc39ea5aeb026e01dc68be39257 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 1 Feb 2017 16:29:48 -0500 Subject: [PATCH 06/61] added handling for float attribute --- attr.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/attr.go b/attr.go index 1425676cf..c85f77cad 100644 --- a/attr.go +++ b/attr.go @@ -265,6 +265,8 @@ func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string attr[k] = uint64(v) case uint: attr[k] = uint64(v) + case float64: + attr[k] = uint64(v) case int64: attr[k] = uint64(v) case string, uint64, bool: From e62bd5fad56f98a5b154b68b7c4839ec189b524b Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 16 Feb 2017 15:43:27 -0500 Subject: [PATCH 07/61] WIP snapshot optimization --- roaring/roaring.go | 48 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index efbf48374..d714577da 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -444,17 +444,30 @@ func (b *Bitmap) removeEmptyContainers() { i++ } } +func (b *Bitmap) countEmptyContainers() int { + result := 0 + for i := 0; i < len(b.containers); { + c := b.containers[i] + + if c.n == 0 { + result++ + } + i++ + } + return result +} // WriteTo writes b to w. func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { // Remove empty containers before persisting. - b.removeEmptyContainers() + //b.removeEmptyContainers() + containerCount := len(b.keys) - b.countEmptyContainers() // Build header before writing individual container blocks. - buf := make([]byte, headerSize+(len(b.keys)*(4+8+4))) + buf := make([]byte, headerSize+(containerCount*(4+8+4))) binary.LittleEndian.PutUint32(buf[0:], cookie) - binary.LittleEndian.PutUint32(buf[4:], uint32(len(b.keys))) - + binary.LittleEndian.PutUint32(buf[4:], uint32(containerCount)) + empty:=0 // Encode keys and cardinality. for i, key := range b.keys { c := b.containers[i] @@ -463,15 +476,24 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { // TODO: instead of commenting this out, we need to make it a configuration option //count := c.count() //assert(c.count() == c.n, "cannot write container count, mismatch: count=%d, n=%d", count, c.n) - - binary.LittleEndian.PutUint64(buf[headerSize+i*12:], uint64(key)) - binary.LittleEndian.PutUint32(buf[headerSize+i*12+8:], uint32(c.n-1)) + if c.n > 0 { + binary.LittleEndian.PutUint64(buf[headerSize+(i-empty)*12:], uint64(key)) + binary.LittleEndian.PutUint32(buf[headerSize+(i-empty)*12+8:], uint32(c.n-1)) + }else{ + empty++ + } } // Write the offset for each container block. offset := uint32(len(buf)) + empty=0 for i, c := range b.containers { - binary.LittleEndian.PutUint32(buf[headerSize+(len(b.keys)*12)+(i*4):], uint32(offset)) + + if c.n > 0 { + binary.LittleEndian.PutUint32(buf[headerSize+(containerCount*12)+((i-empty)*4):], uint32(offset)) + }else{ + empty++ + } offset += uint32(c.size()) } @@ -484,10 +506,12 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { // Write each container block. for _, c := range b.containers { - nn, err := c.WriteTo(w) - n += nn - if err != nil { - return n, err + if c.n > 0 { + nn, err := c.WriteTo(w) + n += nn + if err != nil { + return n, err + } } } From 9a506fa05477c4e227b739726ab6f731e63dfab7 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 16 Feb 2017 15:44:32 -0500 Subject: [PATCH 08/61] added max slice to schema --- db.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/db.go b/db.go index 2a4f1d6e0..60a018de4 100644 --- a/db.go +++ b/db.go @@ -426,8 +426,9 @@ func (p dbSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // DBInfo represents schema information for a database. type DBInfo struct { - Name string `json:"name"` - Frames []*FrameInfo `json:"frames"` + Name string `json:"name"` + Frames []*FrameInfo `json:"frames"` + MaxSlice uint64 } type dbInfoSlice []*DBInfo From 667d56eedf0b51e7c0e7541a17055ce50e3b7f7a Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 16 Feb 2017 15:54:47 -0500 Subject: [PATCH 09/61] added simple cache to fragment --- fragment.go | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/fragment.go b/fragment.go index aaef9323a..88b39663c 100644 --- a/fragment.go +++ b/fragment.go @@ -28,7 +28,8 @@ import ( const ( // SliceWidth is the number of profile IDs in a slice. - SliceWidth = 1048576 + //SliceWidth = 1048576 + SliceWidth = 262144 // SnapshotExt is the file extension used for an in-process snapshot. SnapshotExt = ".snapshotting" @@ -49,8 +50,25 @@ const ( const ( // DefaultFragmentMaxOpN is the default value for Fragment.MaxOpN. - DefaultFragmentMaxOpN = 1000 + //TODO CHANGING FOR TEST TO 10x + DefaultFragmentMaxOpN = 2000 ) +type BitmapCacher interface { + Fetch(id uint64)(*Bitmap,bool) + Add(id uint64, b *Bitmap) +} + + +type Simple struct{ + cache map[uint64]*Bitmap +} +func (s *Simple)Fetch(id uint64)(*Bitmap,bool){ + m,ok:=s.cache[id] + return m,ok +} +func (s *Simple)Add(id uint64,p*Bitmap){ + s.cache[id]=p +} // Fragment represents the intersection of a frame and slice in a database. type Fragment struct { @@ -87,6 +105,8 @@ type Fragment struct { BitmapAttrStore *AttrStore stats StatsClient + turbo BitmapCacher + } // NewFragment returns a new instance of Fragment. @@ -203,6 +223,7 @@ func (f *Fragment) openStorage() error { // Attach the file to the bitmap to act as a write-ahead log. f.storage.OpWriter = f.file + f.turbo = &Simple{make(map[uint64]*Bitmap)} return nil @@ -308,7 +329,12 @@ func (f *Fragment) Bitmap(bitmapID uint64) *Bitmap { return f.bitmap(bitmapID) } + func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { + r,ok:=f.turbo.Fetch(bitmapID) + if ok && r != nil{ + return r + } // Only use a subset of the containers. // NOTE: The start & end ranges must be divisible by data := f.storage.OffsetRange(f.slice*SliceWidth, bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) @@ -325,6 +351,7 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { // Update cache. f.cache.Add(bitmapID, bm.Count()) + f.turbo.Add(bitmapID, bm) return bm } @@ -918,10 +945,15 @@ func (f *Fragment) Snapshot() error { defer f.mu.Unlock() return f.snapshot() } +func track(start time.Time, name string, logger *log.Logger) { + elapsed := time.Since(start) + logger.Printf("%s took %s", name, elapsed) +} func (f *Fragment) snapshot() error { logger := f.logger() logger.Printf("fragment: snapshotting %s/%s/%d", f.db, f.frame, f.slice) + defer track(time.Now(), fmt.Sprintf("fragment: snapshot complete %s/%s/%d", f.db, f.frame, f.slice), logger) // Create a temporary file to snapshot to. snapshotPath := f.path + SnapshotExt From 028539f10da8d27e0aa81d27e79214322c357544 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 17 Feb 2017 15:18:38 -0500 Subject: [PATCH 10/61] top bug in cache refresh --- cache.go | 16 +++++++++++----- fragment.go | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cache.go b/cache.go index a1cba3dec..fb5f4defd 100644 --- a/cache.go +++ b/cache.go @@ -22,6 +22,7 @@ type Cache interface { // Updates the cache, if necessary. Invalidate() + Refresh() // Returns an ordered list of the top ranked bitmaps. Top() []BitmapPair @@ -86,6 +87,8 @@ func (c *LRUCache) Top() []BitmapPair { } func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) } +func (c *LRUCache) Refresh() { +} // Ensure LRUCache implements Cache. var _ Cache = &LRUCache{} @@ -117,17 +120,17 @@ func (c *RankCache) Add(bitmapID uint64, n uint64) { return } - // Add to cache. c.entries[bitmapID] = n + // If size is larger than the threshold then trim it. if len(c.entries) > c.ThresholdLength { c.update() - for id, n := range c.entries { - if n <= c.ThresholdValue { - delete(c.entries, id) + for id, n := range c.entries { + if n <= c.ThresholdValue { + delete(c.entries, id) + } } - } } } @@ -154,6 +157,9 @@ func (c *RankCache) Invalidate() { c.update() } } +func (c *RankCache) Refresh() { + c.update() +} // update reorders the entries by rank. func (c *RankCache) update() { diff --git a/fragment.go b/fragment.go index 88b39663c..51cb1dc61 100644 --- a/fragment.go +++ b/fragment.go @@ -909,7 +909,7 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { f.cache.Add(bitmapID, f.bitmap(bitmapID).Count()) } - f.cache.Invalidate() + f.cache.Refresh() return nil }(); err != nil { _ = f.closeStorage() From 2ecea00515e225c8e50c6a827dee797951605b2d Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Sun, 19 Feb 2017 21:09:12 +0300 Subject: [PATCH 11/61] updated README to have TopN(frame=bar) variant --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2079010ec..a3bfb35dc 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,11 @@ Range(id=10, frame="foo", start="1970-01-01T00:00", end="2000-01-02T03:04") --- #### TopN() +``` +TopN(frame="bar") +``` +Returns all Bitmaps in the cache from frame `bar` sorted by the count of bits. + ``` TopN(frame="bar", n=20) ``` @@ -206,11 +211,9 @@ TopN(Bitmap(id=10, frame="foo"), frame="bar", n=20) ``` Returns the top 20 Bitmaps from `bar` sorted by the count of bits in the intersection with `Bitmap(id=10)`. - ``` TopN(Bitmap(id=10, frame="foo"), frame="bar", n=20, field="category", [81,82]) ``` - Returns the top 20 Bitmaps from `bar`in attribute `category` with values `81 or 82` sorted by the count of bits in the intersection with `Bitmap(id=10)`. From d66e368843807d2307e90ac3e3c1ad17dba4f1fa Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 20 Feb 2017 18:02:24 -0600 Subject: [PATCH 12/61] handle `unionArrayBitmap` width of bitmaps correctly --- roaring/roaring.go | 1 + 1 file changed, 1 insertion(+) diff --git a/roaring/roaring.go b/roaring/roaring.go index efbf48374..c3e50a940 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1368,6 +1368,7 @@ func unionArrayBitmap(a, b *container) *container { break } else if i >= len(a.array) { output.add(vb) + continue } else if eof { output.add(a.array[i]) i++ From 43ede4d9c8efe07e05acbc557d1fa61d5abebb5c Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 21 Feb 2017 09:28:01 -0600 Subject: [PATCH 13/61] WIP increase caching and topn adjustments --- cache.go | 42 ++++++++++++++-------------- fragment.go | 69 ++++++++++++++++++++++++++++------------------ roaring/roaring.go | 13 +++++---- 3 files changed, 71 insertions(+), 53 deletions(-) diff --git a/cache.go b/cache.go index fb5f4defd..c6fc48ec6 100644 --- a/cache.go +++ b/cache.go @@ -14,6 +14,7 @@ import ( // Cache represents a cache for bitmap counts. type Cache interface { Add(bitmapID uint64, n uint64) + BulkAdd(bitmapID uint64, n uint64) Get(bitmapID uint64) uint64 Len() int @@ -22,7 +23,6 @@ type Cache interface { // Updates the cache, if necessary. Invalidate() - Refresh() // Returns an ordered list of the top ranked bitmaps. Top() []BitmapPair @@ -44,6 +44,10 @@ func NewLRUCache(maxEntries int) *LRUCache { return c } +func (c *LRUCache) BulkAdd(bitmapID, n uint64) { + c.Add(bitmapID, n) +} + // Add adds a bitmap to the cache. func (c *LRUCache) Add(bitmapID, n uint64) { c.cache.Add(bitmapID, n) @@ -87,7 +91,7 @@ func (c *LRUCache) Top() []BitmapPair { } func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) } -func (c *LRUCache) Refresh() { +func (c *LRUCache) Refresh() { } // Ensure LRUCache implements Cache. @@ -122,18 +126,26 @@ func (c *RankCache) Add(bitmapID uint64, n uint64) { c.entries[bitmapID] = n - + c.Invalidate() // If size is larger than the threshold then trim it. if len(c.entries) > c.ThresholdLength { - c.update() - for id, n := range c.entries { - if n <= c.ThresholdValue { - delete(c.entries, id) - } + for id, n := range c.entries { + if n <= c.ThresholdValue { + delete(c.entries, id) } + } } } +// Add adds a bitmap to the cache unsorted you should Invalidate after completion +func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) { + if n < c.ThresholdValue { + return + } + + c.entries[bitmapID] = n +} + // Get returns a bitmap with a given id. func (c *RankCache) Get(bitmapID uint64) uint64 { return c.entries[bitmapID] } @@ -150,19 +162,9 @@ func (c *RankCache) BitmapIDs() []uint64 { return a } -// Invalidate reorders the entries, if necessary. -func (c *RankCache) Invalidate() { - // Update if there aren't many items or it hasn't been updated recently. - if len(c.rankings) < 50 || (c.updateN > 0 && time.Since(c.updateTime) > 5*time.Minute) { - c.update() - } -} -func (c *RankCache) Refresh() { - c.update() -} - // update reorders the entries by rank. -func (c *RankCache) update() { +func (c *RankCache) Invalidate() { + //fmt.Println("RankCache Update") // Convert cache to a sorted list. rankings := make([]BitmapPair, 0, len(c.entries)) for id, n := range c.entries { diff --git a/fragment.go b/fragment.go index 51cb1dc61..f16bf6ddb 100644 --- a/fragment.go +++ b/fragment.go @@ -53,21 +53,22 @@ const ( //TODO CHANGING FOR TEST TO 10x DefaultFragmentMaxOpN = 2000 ) + type BitmapCacher interface { - Fetch(id uint64)(*Bitmap,bool) + Fetch(id uint64) (*Bitmap, bool) Add(id uint64, b *Bitmap) } +type Simple struct { + cache map[uint64]*Bitmap +} -type Simple struct{ - cache map[uint64]*Bitmap +func (s *Simple) Fetch(id uint64) (*Bitmap, bool) { + m, ok := s.cache[id] + return m, ok } -func (s *Simple)Fetch(id uint64)(*Bitmap,bool){ - m,ok:=s.cache[id] - return m,ok -} -func (s *Simple)Add(id uint64,p*Bitmap){ - s.cache[id]=p +func (s *Simple) Add(id uint64, p *Bitmap) { + s.cache[id] = p } // Fragment represents the intersection of a frame and slice in a database. @@ -105,8 +106,7 @@ type Fragment struct { BitmapAttrStore *AttrStore stats StatsClient - turbo BitmapCacher - + turbo BitmapCacher } // NewFragment returns a new instance of Fragment. @@ -173,6 +173,7 @@ func (f *Fragment) Open() error { // openStorage opens the storage bitmap. func (f *Fragment) openStorage() error { + //f.logger().Printf("Open Storage %s/%s/%d", f.db, f.frame, f.slice) // Create a roaring bitmap to serve as storage for the slice. f.storage = roaring.NewBitmap() @@ -260,9 +261,11 @@ func (f *Fragment) openCache() error { // Read in all bitmaps by ID. // This will cause them to be added to the cache. for _, bitmapID := range pb.BitmapIDs { - n := f.storage.CountRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) - f.cache.Add(bitmapID, n) + //n := f.storage.CountRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) + n := f.bitmap(bitmapID, false).Count() + f.cache.BulkAdd(bitmapID, n) } + f.cache.Invalidate() return nil } @@ -326,15 +329,14 @@ func (f *Fragment) logger() *log.Logger { return log.New(f.LogOutput, "", log.Ls func (f *Fragment) Bitmap(bitmapID uint64) *Bitmap { f.mu.Lock() defer f.mu.Unlock() - return f.bitmap(bitmapID) + return f.bitmap(bitmapID, false) } - -func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { - r,ok:=f.turbo.Fetch(bitmapID) - if ok && r != nil{ - return r - } +func (f *Fragment) bitmap(bitmapID uint64, updateCache bool) *Bitmap { + r, ok := f.turbo.Fetch(bitmapID) + if ok && r != nil { + return r + } // Only use a subset of the containers. // NOTE: The start & end ranges must be divisible by data := f.storage.OffsetRange(f.slice*SliceWidth, bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) @@ -349,9 +351,11 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { } bm.InvalidateCount() - // Update cache. - f.cache.Add(bitmapID, bm.Count()) - f.turbo.Add(bitmapID, bm) + if updateCache { + // Update cache. + f.cache.Add(bitmapID, bm.Count()) + f.turbo.Add(bitmapID, bm) + } return bm } @@ -391,7 +395,7 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) } // Update the cache. - if f.bitmap(bitmapID).SetBit(profileID) { + if f.bitmap(bitmapID, true).SetBit(profileID) { changed = true } @@ -435,7 +439,7 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (bool, error) { } // Update the cache. - if f.bitmap(bitmapID).ClearBit(profileID) { + if f.bitmap(bitmapID, true).ClearBit(profileID) { return true, nil } @@ -570,7 +574,16 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { return results, nil } +func debugDumpPairs(pairs []BitmapPair) { + fmt.Println("=====Start") + for i, pair := range pairs { + fmt.Println(i, pair.ID, pair.Count) + } + fmt.Println("=====Stop") +} + func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair { + //fmt.Println("DEBUG topBitmapPairs") // If no specific bitmaps are requested, retrieve top bitmaps. if len(bitmapIDs) == 0 { f.mu.Lock() @@ -597,6 +610,8 @@ func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair { Count: f.Bitmap(bitmapID).Count(), } } + sort.Sort(BitmapPairs(pairs)) + //debugDumpPairs(pairs) return pairs } @@ -906,10 +921,10 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { // Update cache counts for all bitmaps. for bitmapID := range set { - f.cache.Add(bitmapID, f.bitmap(bitmapID).Count()) + f.cache.BulkAdd(bitmapID, f.bitmap(bitmapID, false).Count()) } - f.cache.Refresh() + f.cache.Invalidate() return nil }(); err != nil { _ = f.closeStorage() diff --git a/roaring/roaring.go b/roaring/roaring.go index d714577da..58b4d1e19 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -467,7 +467,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { buf := make([]byte, headerSize+(containerCount*(4+8+4))) binary.LittleEndian.PutUint32(buf[0:], cookie) binary.LittleEndian.PutUint32(buf[4:], uint32(containerCount)) - empty:=0 + empty := 0 // Encode keys and cardinality. for i, key := range b.keys { c := b.containers[i] @@ -479,20 +479,20 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { if c.n > 0 { binary.LittleEndian.PutUint64(buf[headerSize+(i-empty)*12:], uint64(key)) binary.LittleEndian.PutUint32(buf[headerSize+(i-empty)*12+8:], uint32(c.n-1)) - }else{ + } else { empty++ } } // Write the offset for each container block. offset := uint32(len(buf)) - empty=0 + empty = 0 for i, c := range b.containers { if c.n > 0 { - binary.LittleEndian.PutUint32(buf[headerSize+(containerCount*12)+((i-empty)*4):], uint32(offset)) - }else{ - empty++ + binary.LittleEndian.PutUint32(buf[headerSize+(containerCount*12)+((i-empty)*4):], uint32(offset)) + } else { + empty++ } offset += uint32(c.size()) } @@ -1392,6 +1392,7 @@ func unionArrayBitmap(a, b *container) *container { break } else if i >= len(a.array) { output.add(vb) + continue } else if eof { output.add(a.array[i]) i++ From 7d30b72d432aa9578fee8dcfac0e8760dec2bef2 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 21 Feb 2017 10:09:49 -0600 Subject: [PATCH 14/61] add restriction database --- client.go | 10 ++++++++++ cmd/pilosactl/main.go | 2 ++ db.go | 5 ++++- fragment.go | 6 +++--- frame.go | 9 +++++++-- pilosa.go | 16 ++++++++++++++++ pilosactl/import.go | 10 ++++++++++ 7 files changed, 52 insertions(+), 6 deletions(-) diff --git a/client.go b/client.go index 5c24920ff..bc6338053 100644 --- a/client.go +++ b/client.go @@ -149,6 +149,11 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire return nil, ErrQueryRequired } + er := ValidateName(db) + if er != nil { + return nil, ErrName + } + // Encode query request. buf, err := proto.Marshal(&internal.QueryRequest{ DB: db, @@ -205,6 +210,11 @@ func (c *Client) ExecutePQL(ctx context.Context, db, query string) (interface{}, }.Encode(), } + er := ValidateName(db) + if er != nil { + return nil, ErrName + } + req, err := http.NewRequest("POST", u.String(), bytes.NewReader([]byte(query))) if err != nil { return nil, err diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 331459c31..7c25d62cb 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -290,6 +290,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) // Validate arguments. + fmt.Print("1213224") if cmd.Database == "" { return pilosa.ErrDatabaseRequired } else if cmd.Frame == "" { @@ -959,6 +960,7 @@ func (cmd *BenchCommand) Run(ctx context.Context) error { // runSetBit executes a benchmark of random SetBit() operations. func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) error { + fmt.Print("Nodb") if cmd.N == 0 { return errors.New("operation count required") } else if cmd.Database == "" { diff --git a/db.go b/db.go index 2a4f1d6e0..9d4c6532a 100644 --- a/db.go +++ b/db.go @@ -274,7 +274,10 @@ func (db *DB) createFrameIfNotExists(name string) (*Frame, error) { } func (db *DB) newFrame(path, name string) *Frame { - f := NewFrame(path, db.name, name) + f, err := NewFrame(path, db.name, name) + if err != nil { + return nil + } f.LogOutput = db.LogOutput f.stats = db.stats.WithTags(fmt.Sprintf("frame:%s", name)) return f diff --git a/fragment.go b/fragment.go index 1181f1739..9bc542f93 100644 --- a/fragment.go +++ b/fragment.go @@ -213,11 +213,11 @@ func (f *Fragment) openCache() error { // Determine cache type from frame name. if strings.HasSuffix(f.frame, FrameSuffixRank) { c := NewRankCache() - c.ThresholdLength = 50000 - c.ThresholdIndex = 45000 + c.ThresholdLength = 500000 + c.ThresholdIndex = 450000 f.cache = c } else { - f.cache = NewLRUCache(50000) + f.cache = NewLRUCache(500000) } // Read cache data from disk. diff --git a/frame.go b/frame.go index 9cdb4c707..1971900e4 100644 --- a/frame.go +++ b/frame.go @@ -38,7 +38,12 @@ type Frame struct { } // NewFrame returns a new instance of frame. -func NewFrame(path, db, name string) *Frame { +func NewFrame(path, db, name string) (*Frame, error) { + err := ValidateName(db) + if err != nil { + return nil, err + } + return &Frame{ path: path, db: db, @@ -50,7 +55,7 @@ func NewFrame(path, db, name string) *Frame { stats: NopStatsClient, LogOutput: ioutil.Discard, - } + }, nil } // Name returns the name the frame was initialized with. diff --git a/pilosa.go b/pilosa.go index dff78ebb1..1a8bba840 100644 --- a/pilosa.go +++ b/pilosa.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/pilosa/pilosa/internal" + "regexp" ) var ( @@ -16,6 +17,9 @@ var ( // ErrFrameRequired is returned when no frame is specified. ErrFrameRequired = errors.New("frame required") + // ErrFrameRequired is returned when no frame is specified. + ErrName = errors.New("name restricted to [a-z0-9_-.]") + // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") @@ -75,3 +79,15 @@ func decodeProfile(pb *internal.Profile) *Profile { // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" + + +// Restrict name using regex +func ValidateName(name string) error { + expr := regexp.MustCompile(`^([a-z0-9._-]{2,64}$)`) + validName := expr.FindStringSubmatchIndex(name) + + if len(validName) == 0 { + return ErrName + } + return nil +} \ No newline at end of file diff --git a/pilosactl/import.go b/pilosactl/import.go index 9d4ad63f4..40a735c61 100644 --- a/pilosactl/import.go +++ b/pilosactl/import.go @@ -15,6 +15,7 @@ import ( "time" "github.com/pilosa/pilosa" + "regexp" ) // ImportCommand represents a command for bulk importing data. @@ -104,7 +105,16 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } else if len(cmd.Paths) == 0 { return errors.New("path required") } + // Restrict frame name and database name with regex + dbError := pilosa.ValidateName(cmd.Database) + if dbError != nil { + return dbError + } + frameError := pilosa.ValidateName(cmd.Frame) + if frameError != nil { + return frameError + } // Create a client to the server. client, err := pilosa.NewClient(cmd.Host) if err != nil { From 774dc14412c4a7205f5448d9cd6a32d6c09dd746 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 22 Feb 2017 18:11:06 -0600 Subject: [PATCH 15/61] handle empty Union/Intersect --- executor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 6f213bafa..bfe1cccf1 100644 --- a/executor.go +++ b/executor.go @@ -281,7 +281,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Bit // executeIntersectSlice executes a intersect() call for a local slice. func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.Intersect, slice uint64) (*Bitmap, error) { - var other *Bitmap + other := &Bitmap{} for i, input := range c.Inputs { bm, err := e.executeBitmapCallSlice(ctx, db, input, slice) if err != nil { @@ -331,7 +331,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Rang // executeUnionSlice executes a union() call for a local slice. func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Union, slice uint64) (*Bitmap, error) { - var other *Bitmap + other := &Bitmap{} for i, input := range c.Inputs { bm, err := e.executeBitmapCallSlice(ctx, db, input, slice) if err != nil { From ff52ffa0473a02e68433d11c7f7ac038886d7a6f Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 23 Feb 2017 15:08:18 -0600 Subject: [PATCH 16/61] WIP TopN optimization --- cache.go | 18 ++++++++++++++++++ fragment.go | 47 +++++++++++++++++++++-------------------------- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/cache.go b/cache.go index c6fc48ec6..2f4761872 100644 --- a/cache.go +++ b/cache.go @@ -241,6 +241,24 @@ func (p Pairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p Pairs) Len() int { return len(p) } func (p Pairs) Less(i, j int) bool { return p[i].Count > p[j].Count } +type PairHeap struct { + Pairs +} + +func (h *Pairs) Push(x interface{}) { + // Push and Pop use pointer receivers because they modify the slice's length, + // not just its contents. + *h = append(*h, x.(Pair)) +} + +func (h *Pairs) Pop() interface{} { + old := *h + n := len(old) + x := old[n-1] + *h = old[0 : n-1] + return x +} + // Add merges other into p and returns a new slice. func (p Pairs) Add(other []Pair) []Pair { // Create lookup of key/counts. diff --git a/fragment.go b/fragment.go index f16bf6ddb..e7d6086d1 100644 --- a/fragment.go +++ b/fragment.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bufio" "bytes" + "container/heap" "context" "crypto/sha1" "encoding/binary" @@ -28,7 +29,7 @@ import ( const ( // SliceWidth is the number of profile IDs in a slice. - //SliceWidth = 1048576 + // SliceWidth = 1048576 SliceWidth = 262144 // SnapshotExt is the file extension used for an in-process snapshot. @@ -173,7 +174,6 @@ func (f *Fragment) Open() error { // openStorage opens the storage bitmap. func (f *Fragment) openStorage() error { - //f.logger().Printf("Open Storage %s/%s/%d", f.db, f.frame, f.slice) // Create a roaring bitmap to serve as storage for the slice. f.storage = roaring.NewBitmap() @@ -494,7 +494,8 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { } // Iterate over rankings and add to results until we have enough. - results := make([]Pair, 0, opt.N) + //results := make(PairHeap, 0, opt.N) + results := &PairHeap{} for _, pair := range pairs { bitmapID, n := pair.ID, pair.Count @@ -518,7 +519,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { } // The initial n pairs should simply be added to the results. - if opt.N == 0 || len(results) < opt.N { + if opt.N == 0 || results.Len() < opt.N { // Calculate count and append. count := n if opt.Src != nil { @@ -527,23 +528,26 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { if count == 0 { continue } - results = append(results, Pair{Key: bitmapID, Count: count}) + //results = append(results, Pair{Key: bitmapID, Count: count}) + heap.Push(results, Pair{Key: bitmapID, Count: count}) // If we reach the requested number of pairs and we are not computing // intersections then simply exit. If we are intersecting then sort // and then only keep pairs that are higher than the lowest count. - if opt.N > 0 && len(results) == opt.N { + if opt.N > 0 && results.Len() == opt.N { if opt.Src == nil { break } - sort.Sort(Pairs(results)) + // sort.Sort(Pairs(results)) } continue } // Retrieve the lowest count we have. // If it's too low then don't try finding anymore pairs. - threshold := results[len(results)-1].Count + //threshold := results[len(results)-1].Count + + threshold := results.Pairs[0].Count if threshold < MinThreshold { break } @@ -556,34 +560,25 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Calculate the intersecting bit count and skip if it's below our // last bitmap in our current result set. + count := opt.Src.IntersectionCount(f.Bitmap(bitmapID)) if count < threshold { continue } - // Swap out the last pair for this new count. - results[len(results)-1] = Pair{Key: bitmapID, Count: count} - - // If it's count is also higher than the second to last item then resort. - if len(results) >= 2 && count > results[len(results)-2].Count { - sort.Sort(Pairs(results)) - } + heap.Push(results, Pair{Key: bitmapID, Count: count}) } - - sort.Sort(Pairs(results)) - return results, nil -} - -func debugDumpPairs(pairs []BitmapPair) { - fmt.Println("=====Start") - for i, pair := range pairs { - fmt.Println(i, pair.ID, pair.Count) + r := make(Pairs, results.Len(), results.Len()) + x := results.Len() + i := 1 + for results.Len() > 0 { + r[x-i] = heap.Pop(results).(Pair) + i++ } - fmt.Println("=====Stop") + return r, nil } func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair { - //fmt.Println("DEBUG topBitmapPairs") // If no specific bitmaps are requested, retrieve top bitmaps. if len(bitmapIDs) == 0 { f.mu.Lock() From b4d6181285e8808511361694343f26082b72cdda Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 24 Feb 2017 12:57:41 -0600 Subject: [PATCH 17/61] no longer need to implement `Refresh()` since that was removed` --- cache.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cache.go b/cache.go index 2f4761872..af2284fc7 100644 --- a/cache.go +++ b/cache.go @@ -91,8 +91,6 @@ func (c *LRUCache) Top() []BitmapPair { } func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) } -func (c *LRUCache) Refresh() { -} // Ensure LRUCache implements Cache. var _ Cache = &LRUCache{} From 846b528c529565b33a81faddaf4d0f8a9d530df9 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 24 Feb 2017 13:18:29 -0600 Subject: [PATCH 18/61] change the names of `Simple`, `turbo` to `SimpleCache`, `bitmapCache` --- fragment.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/fragment.go b/fragment.go index e7d6086d1..d6eead246 100644 --- a/fragment.go +++ b/fragment.go @@ -29,8 +29,7 @@ import ( const ( // SliceWidth is the number of profile IDs in a slice. - // SliceWidth = 1048576 - SliceWidth = 262144 + SliceWidth = 1048576 // SnapshotExt is the file extension used for an in-process snapshot. SnapshotExt = ".snapshotting" @@ -51,24 +50,28 @@ const ( const ( // DefaultFragmentMaxOpN is the default value for Fragment.MaxOpN. - //TODO CHANGING FOR TEST TO 10x DefaultFragmentMaxOpN = 2000 ) +// BitmapCacher implements SimpleCache +// it is meant to be a short-lived cache for cases where writes are continuing to access +// the same bit withing a short time frame (i.e. good for write-heavy loads) +// A read-heavy use case would cause the cache to get bigger, potentially causing the +// node to run out of memory. type BitmapCacher interface { Fetch(id uint64) (*Bitmap, bool) Add(id uint64, b *Bitmap) } -type Simple struct { +type SimpleCache struct { cache map[uint64]*Bitmap } -func (s *Simple) Fetch(id uint64) (*Bitmap, bool) { +func (s *SimpleCache) Fetch(id uint64) (*Bitmap, bool) { m, ok := s.cache[id] return m, ok } -func (s *Simple) Add(id uint64, p *Bitmap) { +func (s *SimpleCache) Add(id uint64, p *Bitmap) { s.cache[id] = p } @@ -106,8 +109,8 @@ type Fragment struct { // This is set by the parent frame unless overridden for testing. BitmapAttrStore *AttrStore - stats StatsClient - turbo BitmapCacher + stats StatsClient + bitmapCache BitmapCacher } // NewFragment returns a new instance of Fragment. @@ -224,7 +227,7 @@ func (f *Fragment) openStorage() error { // Attach the file to the bitmap to act as a write-ahead log. f.storage.OpWriter = f.file - f.turbo = &Simple{make(map[uint64]*Bitmap)} + f.bitmapCache = &SimpleCache{make(map[uint64]*Bitmap)} return nil @@ -333,7 +336,7 @@ func (f *Fragment) Bitmap(bitmapID uint64) *Bitmap { } func (f *Fragment) bitmap(bitmapID uint64, updateCache bool) *Bitmap { - r, ok := f.turbo.Fetch(bitmapID) + r, ok := f.bitmapCache.Fetch(bitmapID) if ok && r != nil { return r } @@ -354,7 +357,7 @@ func (f *Fragment) bitmap(bitmapID uint64, updateCache bool) *Bitmap { if updateCache { // Update cache. f.cache.Add(bitmapID, bm.Count()) - f.turbo.Add(bitmapID, bm) + f.bitmapCache.Add(bitmapID, bm) } return bm From de1a779d41b042bdfe00c1d8896fa60ddd614de1 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 24 Feb 2017 13:20:04 -0600 Subject: [PATCH 19/61] remove unused `MaxSlice` from `DB` --- db.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/db.go b/db.go index 60a018de4..2a4f1d6e0 100644 --- a/db.go +++ b/db.go @@ -426,9 +426,8 @@ func (p dbSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // DBInfo represents schema information for a database. type DBInfo struct { - Name string `json:"name"` - Frames []*FrameInfo `json:"frames"` - MaxSlice uint64 + Name string `json:"name"` + Frames []*FrameInfo `json:"frames"` } type dbInfoSlice []*DBInfo From b6907c4e19558899654326fb5ea7e37c397fa7d3 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 24 Feb 2017 13:29:14 -0600 Subject: [PATCH 20/61] add TODO to make `MaxIdleConnsPerHost` configurable --- cmd/pilosa/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go index 6fff5db6b..82eb9d7a7 100644 --- a/cmd/pilosa/main.go +++ b/cmd/pilosa/main.go @@ -43,6 +43,7 @@ const ( func main() { // Limit the number of connections that a server can make to a single node // in order to prevent a cluster storm. + // TODO: make this configurable http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 64 m := NewMain() From 22e9c1cafaa85582bb040df77ecf5dbf2498da8c Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 27 Feb 2017 11:04:37 -0600 Subject: [PATCH 21/61] modify `needsSlices()` to use `call.Name` --- executor.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/executor.go b/executor.go index 59e5fab5e..5f6b766c9 100644 --- a/executor.go +++ b/executor.go @@ -995,20 +995,20 @@ func hasOnlySetBitmapAttrs(calls []*pql.Call) bool { return true } -func needsSlices(calls pql.Calls) bool { +func needsSlices(calls []*pql.Call) bool { if len(calls) == 0 { return false } - for _, call := range calls { - if _, ok := call.(pql.BitmapCall); !ok { + switch call.Name { + case "ClearBit", "Profile", "SetBit", "SetBitmapAttrs", "SetProfileAttrs": + continue + case "Count", "TopN": return true - } else if _, ok := call.(*pql.Count); !ok { - return true - } else if _, ok := call.(*pql.TopN); !ok { + // default catches Bitmap calls + default: return true } - } return false } From c36828863856b5b80161a1d4358dabfa04ef217d Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 27 Feb 2017 11:54:59 -0600 Subject: [PATCH 22/61] fix `BulkAdd` comment --- cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache.go b/cache.go index af2284fc7..86cab6b5a 100644 --- a/cache.go +++ b/cache.go @@ -135,7 +135,7 @@ func (c *RankCache) Add(bitmapID uint64, n uint64) { } } -// Add adds a bitmap to the cache unsorted you should Invalidate after completion +// BulkAdd adds a bitmap to the cache unsorted. You should Invalidate after completion. func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) { if n < c.ThresholdValue { return From ab52a803b849bc43efd13d3afc9304314a4eb64a Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 27 Feb 2017 12:42:36 -0600 Subject: [PATCH 23/61] rank cache update after count --- fragment.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/fragment.go b/fragment.go index d6eead246..8337dbb96 100644 --- a/fragment.go +++ b/fragment.go @@ -398,9 +398,10 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) } // Update the cache. - if f.bitmap(bitmapID, true).SetBit(profileID) { - changed = true - } + bm := f.bitmap(bitmapID, true) + bm.SetBit(profileID) + bm.InvalidateCount() //maybe a perf opportunity? + f.cache.Add(bitmapID, bm.Count()) f.stats.Count("setN", 1) @@ -442,9 +443,10 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (bool, error) { } // Update the cache. - if f.bitmap(bitmapID, true).ClearBit(profileID) { - return true, nil - } + bm := f.bitmap(bitmapID, true) + bm.ClearBit(profileID) + bm.InvalidateCount() //maybe a perf opportunity? + f.cache.Add(bitmapID, bm.Count()) f.stats.Count("clearN", 1) From fd800e130cc7c82cd603c8bfe0a21e33a7e13638 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 27 Feb 2017 12:43:31 -0600 Subject: [PATCH 24/61] heap sort order backwards --- cache.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cache.go b/cache.go index 86cab6b5a..8eab72368 100644 --- a/cache.go +++ b/cache.go @@ -243,6 +243,8 @@ type PairHeap struct { Pairs } +func (p PairHeap) Less(i, j int) bool { return p.Pairs[i].Count < p.Pairs[j].Count } + func (h *Pairs) Push(x interface{}) { // Push and Pop use pointer receivers because they modify the slice's length, // not just its contents. From 3b309d0d5140f12ada16a1a37bd8f39a2f35482b Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 27 Feb 2017 18:25:45 -0600 Subject: [PATCH 25/61] WIP TopN accuracy --- executor.go | 13 ++++++++++--- fragment.go | 25 ++++++++++++++----------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/executor.go b/executor.go index 5f6b766c9..c681495f7 100644 --- a/executor.go +++ b/executor.go @@ -168,6 +168,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql // requeries to retrieve the full counts for each of the top results. func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { bitmapIDs, _ := c.Args["ids"].([]uint64) + n := c.Args["n"].(uint64) // Execute original query. pairs, err := e.executeTopNSlices(ctx, db, c, slices, opt) @@ -180,16 +181,21 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic if len(pairs) == 0 || len(bitmapIDs) > 0 || opt.Remote { return pairs, nil } - // Only the original caller should refetch the full counts. other := c.Clone() - other.Args["n"] = 0 + //other.Args["n"] = 0 + other.Args["n"] = len(bitmapIDs) * 2 ids := Pairs(pairs).Keys() sort.Sort(uint64Slice(ids)) other.Args["ids"] = ids - return e.executeTopNSlices(ctx, db, other, slices, opt) + trimedlist, x := e.executeTopNSlices(ctx, db, other, slices, opt) + if x != nil { + return nil, x + } + trimedlist = trimedlist[0:n] + return trimedlist, nil } func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { @@ -896,6 +902,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod if n.Host == e.Host { resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn) } else if !opt.Remote { + results, err := e.exec(ctx, n, db, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) if len(results) > 0 { resp.result = results[0] diff --git a/fragment.go b/fragment.go index 8337dbb96..493640ef0 100644 --- a/fragment.go +++ b/fragment.go @@ -553,9 +553,9 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { //threshold := results[len(results)-1].Count threshold := results.Pairs[0].Count - if threshold < MinThreshold { - break - } + //if threshold < MinThreshold { + //break + //} // If the bitmap doesn't have enough bits set before the intersection // then we can assume that any remaing bitmaps also have a count too low. @@ -593,21 +593,24 @@ func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair { } // Otherwise retrieve specific bitmaps. - pairs := make([]BitmapPair, len(bitmapIDs)) - for i, bitmapID := range bitmapIDs { + pairs := make([]BitmapPair, 0, len(bitmapIDs)) + for _, bitmapID := range bitmapIDs { // Look up cache first, if available. if n := f.cache.Get(bitmapID); n > 0 { - pairs[i] = BitmapPair{ + pairs = append(pairs, BitmapPair{ ID: bitmapID, Count: n, - } + }) continue } - // Otherwise load from storage. - pairs[i] = BitmapPair{ - ID: bitmapID, - Count: f.Bitmap(bitmapID).Count(), + bm := f.Bitmap(bitmapID) + if bm.Count() > 0 { + // Otherwise load from storage. + pairs = append(pairs, BitmapPair{ + ID: bitmapID, + Count: bm.Count(), + }) } } sort.Sort(BitmapPairs(pairs)) From 5ef4d6fa6bd156a37576e228e3784e2b9153bb0f Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 27 Feb 2017 19:34:33 -0600 Subject: [PATCH 26/61] adjusted first phase topn to collect all slices id's --- executor.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/executor.go b/executor.go index c681495f7..72df37c80 100644 --- a/executor.go +++ b/executor.go @@ -199,7 +199,7 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic } func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { - n, _ := c.Args["n"].(uint64) + //n, _ := c.Args["n"].(uint64) // Execute calls in bulk on each remote node and merge. mapFn := func(slice uint64) (interface{}, error) { @@ -222,9 +222,9 @@ func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call sort.Sort(Pairs(results)) // Only keep the top n after sorting. - if n > 0 && len(results) > int(n) { - results = results[0:n] - } + // if n > 0 && len(results) > int(n) { + // results = results[0:n] + //} return results, nil } From 62a59f2bd1ba760885ed08e1339432fbb676aefe Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 28 Feb 2017 14:01:30 -0600 Subject: [PATCH 27/61] incorrect handling of large topns --- executor.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/executor.go b/executor.go index 72df37c80..f1f1adb3c 100644 --- a/executor.go +++ b/executor.go @@ -194,7 +194,10 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic if x != nil { return nil, x } - trimedlist = trimedlist[0:n] + + if int(n) < len(trimedlist) { + trimedlist = trimedlist[0:n] + } return trimedlist, nil } From 527ff5682f8175d0d46c92dbad8c3cfde5e78591 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 1 Mar 2017 15:14:00 -0600 Subject: [PATCH 28/61] cache performance enhancement --- bitmap.go | 23 ++++++++++++++++++++++- cache.go | 47 ++++++++++++++++++++++++++++++++++++----------- fragment.go | 22 +++++++++++++++++----- 3 files changed, 75 insertions(+), 17 deletions(-) diff --git a/bitmap.go b/bitmap.go index 55664b24b..d30a9192a 100644 --- a/bitmap.go +++ b/bitmap.go @@ -16,6 +16,8 @@ type Bitmap struct { // Attributes associated with the bitmap. Attrs map[string]interface{} + + cacheoveride uint64 } // NewBitmap returns a new instance of Bitmap. @@ -174,6 +176,26 @@ func (b *Bitmap) InvalidateCount() { } } +//increment the bitmap cached counter, note this is an optimization that assumes that the caller is aware the size increased +func (b *Bitmap) IncrementCount(i uint64) { + seg := b.segment(i / SliceWidth) + if seg != nil { + seg.n++ + } +} +func (b *Bitmap) DecrementCount(i uint64) { + seg := b.segment(i / SliceWidth) + if seg != nil { + if seg.n > 0 { + seg.n-- + } + } +} +func (b *Bitmap) SetCount(i uint64, count uint64) { + seg := b.segment(i / SliceWidth) + seg.n = count +} + // Count returns the number of set bits in the bitmap. func (b *Bitmap) Count() uint64 { var n uint64 @@ -312,7 +334,6 @@ func (s *BitmapSegment) Difference(other *BitmapSegment) *BitmapSegment { // SetBit sets the i-th bit of the bitmap. func (s *BitmapSegment) SetBit(i uint64) (changed bool) { s.ensureWritable() - changed, _ = s.data.Add(i) if changed { s.n++ diff --git a/cache.go b/cache.go index 8eab72368..4345c93fd 100644 --- a/cache.go +++ b/cache.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "sort" + "sync" "time" "github.com/golang/groupcache/lru" @@ -97,6 +98,7 @@ var _ Cache = &LRUCache{} // RankCache represents a cache with sorted entries. type RankCache struct { + mu sync.Mutex entries map[uint64]uint64 rankings []BitmapPair // cached, ordered list @@ -117,6 +119,8 @@ func NewRankCache() *RankCache { // Add adds a bitmap to the cache. func (c *RankCache) Add(bitmapID uint64, n uint64) { + c.mu.Lock() + defer c.mu.Unlock() // Ignore if the bit count on the bitmap is below the threshold. if n < c.ThresholdValue { return @@ -124,19 +128,13 @@ func (c *RankCache) Add(bitmapID uint64, n uint64) { c.entries[bitmapID] = n - c.Invalidate() - // If size is larger than the threshold then trim it. - if len(c.entries) > c.ThresholdLength { - for id, n := range c.entries { - if n <= c.ThresholdValue { - delete(c.entries, id) - } - } - } + c.invalidate() } // BulkAdd adds a bitmap to the cache unsorted. You should Invalidate after completion. func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) { + c.mu.Lock() + defer c.mu.Unlock() if n < c.ThresholdValue { return } @@ -145,13 +143,23 @@ func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) { } // Get returns a bitmap with a given id. -func (c *RankCache) Get(bitmapID uint64) uint64 { return c.entries[bitmapID] } +func (c *RankCache) Get(bitmapID uint64) uint64 { + c.mu.Lock() + defer c.mu.Unlock() + return c.entries[bitmapID] +} // Len returns the number of items in the cache. -func (c *RankCache) Len() int { return len(c.entries) } +func (c *RankCache) Len() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.entries) +} // BitmapIDs returns a list of all bitmap IDs in the cache. func (c *RankCache) BitmapIDs() []uint64 { + c.mu.Lock() + defer c.mu.Unlock() a := make([]uint64, 0, len(c.entries)) for id := range c.entries { a = append(a, id) @@ -162,6 +170,15 @@ func (c *RankCache) BitmapIDs() []uint64 { // update reorders the entries by rank. func (c *RankCache) Invalidate() { + c.mu.Lock() + defer c.mu.Unlock() + c.invalidate() + +} +func (c *RankCache) invalidate() { + if time.Now().Sub(c.updateTime).Seconds() < 10 { + return + } //fmt.Println("RankCache Update") // Convert cache to a sorted list. rankings := make([]BitmapPair, 0, len(c.entries)) @@ -183,6 +200,14 @@ func (c *RankCache) Invalidate() { // Reset counters. c.updateTime, c.updateN = time.Now(), 0 + // If size is larger than the threshold then trim it. + if len(c.entries) > c.ThresholdLength { + for id, n := range c.entries { + if n <= c.ThresholdValue { + delete(c.entries, id) + } + } + } } // Top returns an ordered list of bitmaps. diff --git a/fragment.go b/fragment.go index 493640ef0..5516e05ec 100644 --- a/fragment.go +++ b/fragment.go @@ -351,8 +351,10 @@ func (f *Fragment) bitmap(bitmapID uint64, updateCache bool) *Bitmap { slice: f.slice, writable: false, }}, + cacheoveride: 0, } bm.InvalidateCount() + bm.cacheoveride = bm.Count() if updateCache { // Update cache. @@ -380,6 +382,7 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) } // Write to storage. + if changed, err = f.storage.Add(pos); err != nil { return false, err } @@ -399,8 +402,13 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) // Update the cache. bm := f.bitmap(bitmapID, true) - bm.SetBit(profileID) - bm.InvalidateCount() //maybe a perf opportunity? + if bm.cacheoveride > 0 { + bm.SetCount(profileID, bm.cacheoveride) + bm.cacheoveride = 0 + } else { + bm.IncrementCount(profileID) + } + f.cache.Add(bitmapID, bm.Count()) f.stats.Count("setN", 1) @@ -443,9 +451,13 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (bool, error) { } // Update the cache. - bm := f.bitmap(bitmapID, true) - bm.ClearBit(profileID) - bm.InvalidateCount() //maybe a perf opportunity? + bm := f.bitmap(bitmapID, false) + if bm.cacheoveride > 0 { + bm.SetCount(profileID, bm.cacheoveride) + bm.cacheoveride = 0 + } else { + bm.DecrementCount(profileID) + } f.cache.Add(bitmapID, bm.Count()) f.stats.Count("clearN", 1) From 26041563fa6985d616c7f184e9afd343d8fb6da2 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 2 Mar 2017 10:36:35 -0600 Subject: [PATCH 29/61] fix for failed test TestMain_FrameRestore --- fragment.go | 1 + 1 file changed, 1 insertion(+) diff --git a/fragment.go b/fragment.go index 5516e05ec..ac4d63926 100644 --- a/fragment.go +++ b/fragment.go @@ -408,6 +408,7 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) } else { bm.IncrementCount(profileID) } + bm.SetBit(profileID) f.cache.Add(bitmapID, bm.Count()) From 9237e7b6995618af7f6aa818707e88ab4208038b Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 27 Feb 2017 12:42:36 -0600 Subject: [PATCH 30/61] rank cache update after count heap sort order backwards WIP TopN accuracy adjusted first phase topn to collect all slices id's incorrect handling of large topns cache performance enhancement fix for failed test TestMain_FrameRestore remove unused code and fix some variable names --- bitmap.go | 23 ++++++++++++++++++++++- cache.go | 49 ++++++++++++++++++++++++++++++++++++++----------- executor.go | 24 ++++++++++++++---------- fragment.go | 47 ++++++++++++++++++++++++++++++----------------- 4 files changed, 104 insertions(+), 39 deletions(-) diff --git a/bitmap.go b/bitmap.go index 55664b24b..d30a9192a 100644 --- a/bitmap.go +++ b/bitmap.go @@ -16,6 +16,8 @@ type Bitmap struct { // Attributes associated with the bitmap. Attrs map[string]interface{} + + cacheoveride uint64 } // NewBitmap returns a new instance of Bitmap. @@ -174,6 +176,26 @@ func (b *Bitmap) InvalidateCount() { } } +//increment the bitmap cached counter, note this is an optimization that assumes that the caller is aware the size increased +func (b *Bitmap) IncrementCount(i uint64) { + seg := b.segment(i / SliceWidth) + if seg != nil { + seg.n++ + } +} +func (b *Bitmap) DecrementCount(i uint64) { + seg := b.segment(i / SliceWidth) + if seg != nil { + if seg.n > 0 { + seg.n-- + } + } +} +func (b *Bitmap) SetCount(i uint64, count uint64) { + seg := b.segment(i / SliceWidth) + seg.n = count +} + // Count returns the number of set bits in the bitmap. func (b *Bitmap) Count() uint64 { var n uint64 @@ -312,7 +334,6 @@ func (s *BitmapSegment) Difference(other *BitmapSegment) *BitmapSegment { // SetBit sets the i-th bit of the bitmap. func (s *BitmapSegment) SetBit(i uint64) (changed bool) { s.ensureWritable() - changed, _ = s.data.Add(i) if changed { s.n++ diff --git a/cache.go b/cache.go index 86cab6b5a..4345c93fd 100644 --- a/cache.go +++ b/cache.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "sort" + "sync" "time" "github.com/golang/groupcache/lru" @@ -97,6 +98,7 @@ var _ Cache = &LRUCache{} // RankCache represents a cache with sorted entries. type RankCache struct { + mu sync.Mutex entries map[uint64]uint64 rankings []BitmapPair // cached, ordered list @@ -117,6 +119,8 @@ func NewRankCache() *RankCache { // Add adds a bitmap to the cache. func (c *RankCache) Add(bitmapID uint64, n uint64) { + c.mu.Lock() + defer c.mu.Unlock() // Ignore if the bit count on the bitmap is below the threshold. if n < c.ThresholdValue { return @@ -124,19 +128,13 @@ func (c *RankCache) Add(bitmapID uint64, n uint64) { c.entries[bitmapID] = n - c.Invalidate() - // If size is larger than the threshold then trim it. - if len(c.entries) > c.ThresholdLength { - for id, n := range c.entries { - if n <= c.ThresholdValue { - delete(c.entries, id) - } - } - } + c.invalidate() } // BulkAdd adds a bitmap to the cache unsorted. You should Invalidate after completion. func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) { + c.mu.Lock() + defer c.mu.Unlock() if n < c.ThresholdValue { return } @@ -145,13 +143,23 @@ func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) { } // Get returns a bitmap with a given id. -func (c *RankCache) Get(bitmapID uint64) uint64 { return c.entries[bitmapID] } +func (c *RankCache) Get(bitmapID uint64) uint64 { + c.mu.Lock() + defer c.mu.Unlock() + return c.entries[bitmapID] +} // Len returns the number of items in the cache. -func (c *RankCache) Len() int { return len(c.entries) } +func (c *RankCache) Len() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.entries) +} // BitmapIDs returns a list of all bitmap IDs in the cache. func (c *RankCache) BitmapIDs() []uint64 { + c.mu.Lock() + defer c.mu.Unlock() a := make([]uint64, 0, len(c.entries)) for id := range c.entries { a = append(a, id) @@ -162,6 +170,15 @@ func (c *RankCache) BitmapIDs() []uint64 { // update reorders the entries by rank. func (c *RankCache) Invalidate() { + c.mu.Lock() + defer c.mu.Unlock() + c.invalidate() + +} +func (c *RankCache) invalidate() { + if time.Now().Sub(c.updateTime).Seconds() < 10 { + return + } //fmt.Println("RankCache Update") // Convert cache to a sorted list. rankings := make([]BitmapPair, 0, len(c.entries)) @@ -183,6 +200,14 @@ func (c *RankCache) Invalidate() { // Reset counters. c.updateTime, c.updateN = time.Now(), 0 + // If size is larger than the threshold then trim it. + if len(c.entries) > c.ThresholdLength { + for id, n := range c.entries { + if n <= c.ThresholdValue { + delete(c.entries, id) + } + } + } } // Top returns an ordered list of bitmaps. @@ -243,6 +268,8 @@ type PairHeap struct { Pairs } +func (p PairHeap) Less(i, j int) bool { return p.Pairs[i].Count < p.Pairs[j].Count } + func (h *Pairs) Push(x interface{}) { // Push and Pop use pointer receivers because they modify the slice's length, // not just its contents. diff --git a/executor.go b/executor.go index 5f6b766c9..4a2d954c9 100644 --- a/executor.go +++ b/executor.go @@ -168,6 +168,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql // requeries to retrieve the full counts for each of the top results. func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { bitmapIDs, _ := c.Args["ids"].([]uint64) + n := c.Args["n"].(uint64) // Execute original query. pairs, err := e.executeTopNSlices(ctx, db, c, slices, opt) @@ -180,21 +181,28 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic if len(pairs) == 0 || len(bitmapIDs) > 0 || opt.Remote { return pairs, nil } - // Only the original caller should refetch the full counts. other := c.Clone() - other.Args["n"] = 0 + + // Double the size of n for other calls in order to... + other.Args["n"] = len(bitmapIDs) * 2 ids := Pairs(pairs).Keys() sort.Sort(uint64Slice(ids)) other.Args["ids"] = ids - return e.executeTopNSlices(ctx, db, other, slices, opt) + trimmedList, err := e.executeTopNSlices(ctx, db, other, slices, opt) + if err != nil { + return nil, err + } + + if int(n) < len(trimmedList) { + trimmedList = trimmedList[0:n] + } + return trimmedList, nil } func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { - n, _ := c.Args["n"].(uint64) - // Execute calls in bulk on each remote node and merge. mapFn := func(slice uint64) (interface{}, error) { return e.executeTopNSlice(ctx, db, c, slice) @@ -215,11 +223,6 @@ func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call // Sort final merged results. sort.Sort(Pairs(results)) - // Only keep the top n after sorting. - if n > 0 && len(results) > int(n) { - results = results[0:n] - } - return results, nil } @@ -896,6 +899,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod if n.Host == e.Host { resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn) } else if !opt.Remote { + results, err := e.exec(ctx, n, db, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) if len(results) > 0 { resp.result = results[0] diff --git a/fragment.go b/fragment.go index d6eead246..e203dd711 100644 --- a/fragment.go +++ b/fragment.go @@ -351,8 +351,10 @@ func (f *Fragment) bitmap(bitmapID uint64, updateCache bool) *Bitmap { slice: f.slice, writable: false, }}, + cacheoveride: 0, } bm.InvalidateCount() + bm.cacheoveride = bm.Count() if updateCache { // Update cache. @@ -380,6 +382,7 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) } // Write to storage. + if changed, err = f.storage.Add(pos); err != nil { return false, err } @@ -398,9 +401,16 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) } // Update the cache. - if f.bitmap(bitmapID, true).SetBit(profileID) { - changed = true + bm := f.bitmap(bitmapID, true) + if bm.cacheoveride > 0 { + bm.SetCount(profileID, bm.cacheoveride) + bm.cacheoveride = 0 + } else { + bm.IncrementCount(profileID) } + bm.SetBit(profileID) + + f.cache.Add(bitmapID, bm.Count()) f.stats.Count("setN", 1) @@ -442,9 +452,14 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (bool, error) { } // Update the cache. - if f.bitmap(bitmapID, true).ClearBit(profileID) { - return true, nil + bm := f.bitmap(bitmapID, false) + if bm.cacheoveride > 0 { + bm.SetCount(profileID, bm.cacheoveride) + bm.cacheoveride = 0 + } else { + bm.DecrementCount(profileID) } + f.cache.Add(bitmapID, bm.Count()) f.stats.Count("clearN", 1) @@ -548,12 +563,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Retrieve the lowest count we have. // If it's too low then don't try finding anymore pairs. - //threshold := results[len(results)-1].Count - threshold := results.Pairs[0].Count - if threshold < MinThreshold { - break - } // If the bitmap doesn't have enough bits set before the intersection // then we can assume that any remaing bitmaps also have a count too low. @@ -591,21 +601,24 @@ func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair { } // Otherwise retrieve specific bitmaps. - pairs := make([]BitmapPair, len(bitmapIDs)) - for i, bitmapID := range bitmapIDs { + pairs := make([]BitmapPair, 0, len(bitmapIDs)) + for _, bitmapID := range bitmapIDs { // Look up cache first, if available. if n := f.cache.Get(bitmapID); n > 0 { - pairs[i] = BitmapPair{ + pairs = append(pairs, BitmapPair{ ID: bitmapID, Count: n, - } + }) continue } - // Otherwise load from storage. - pairs[i] = BitmapPair{ - ID: bitmapID, - Count: f.Bitmap(bitmapID).Count(), + bm := f.Bitmap(bitmapID) + if bm.Count() > 0 { + // Otherwise load from storage. + pairs = append(pairs, BitmapPair{ + ID: bitmapID, + Count: bm.Count(), + }) } } sort.Sort(BitmapPairs(pairs)) From def03f3a45dead9baea89b69008b065961c7f7dc Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 2 Mar 2017 15:07:10 -0600 Subject: [PATCH 31/61] more TopN tests --- executor_test.go | 80 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/executor_test.go b/executor_test.go index 2cea37d3d..d6c9d0e91 100644 --- a/executor_test.go +++ b/executor_test.go @@ -215,6 +215,40 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { } } +// Ensure +func TestExecutor_Execute_TopN_fill_small(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 2).SetBit(0, 2*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 3).SetBit(0, 3*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 4).SetBit(0, 4*SliceWidth) + + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(1, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(1, 1) + + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(2, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(2, SliceWidth+1) + + idx.MustCreateFragmentIfNotExists("d", "f", 2).SetBit(3, 2*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 2).SetBit(3, 2*SliceWidth+1) + + idx.MustCreateFragmentIfNotExists("d", "f", 3).SetBit(4, 3*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 3).SetBit(4, 3*SliceWidth+1) + + // Execute query. + e := NewExecutor(idx.Index, NewCluster(1)) + if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + {Key: 0, Count: 5}, + }}) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } +} + // Ensure a TopN() query with a source bitmap can be executed. func TestExecutor_Execute_TopN_Src(t *testing.T) { idx := MustOpenIndex() @@ -248,6 +282,52 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { } } +//Ensure TopN handles Attribute filters +func TestExecutor_Execute_TopN_Attr(t *testing.T) { + // + idx := MustOpenIndex() + defer idx.Close() + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + + if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { + t.Fatal(err) + } + e := NewExecutor(idx.Index, NewCluster(1)) + if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + {Key: 10, Count: 1}, + }}) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + +} + +//Ensure TopN handles Attribute filters with source bitmap +func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { + // + idx := MustOpenIndex() + defer idx.Close() + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + + if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { + t.Fatal(err) + } + e := NewExecutor(idx.Index, NewCluster(1)) + if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(Bitmap(id=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + {Key: 10, Count: 1}, + }}) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + +} + // Ensure a range query can be executed. func TestExecutor_Execute_Range(t *testing.T) { idx := MustOpenIndex() From cbac9ec88bf80a864f95bb19115329b7ca469c39 Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 2 Mar 2017 15:17:21 -0600 Subject: [PATCH 32/61] move BitmapCache into cache.go. adjust some variable names for clarity --- bitmap.go | 2 +- cache.go | 35 +++++++++++++++++++++++++----- executor.go | 1 + fragment.go | 62 ++++++++++++++++------------------------------------- 4 files changed, 51 insertions(+), 49 deletions(-) diff --git a/bitmap.go b/bitmap.go index d30a9192a..651950e45 100644 --- a/bitmap.go +++ b/bitmap.go @@ -17,7 +17,7 @@ type Bitmap struct { // Attributes associated with the bitmap. Attrs map[string]interface{} - cacheoveride uint64 + adjustedCount uint64 } // NewBitmap returns a new instance of Bitmap. diff --git a/cache.go b/cache.go index 4345c93fd..184310361 100644 --- a/cache.go +++ b/cache.go @@ -176,16 +176,17 @@ func (c *RankCache) Invalidate() { } func (c *RankCache) invalidate() { + // Don't invalidate more than once every X seconds. + // TODO: consider making this configurable. if time.Now().Sub(c.updateTime).Seconds() < 10 { return } - //fmt.Println("RankCache Update") // Convert cache to a sorted list. rankings := make([]BitmapPair, 0, len(c.entries)) - for id, n := range c.entries { + for id, cnt := range c.entries { rankings = append(rankings, BitmapPair{ ID: id, - Count: n, + Count: cnt, }) } sort.Sort(BitmapPairs(rankings)) @@ -200,10 +201,11 @@ func (c *RankCache) invalidate() { // Reset counters. c.updateTime, c.updateN = time.Now(), 0 + // If size is larger than the threshold then trim it. if len(c.entries) > c.ThresholdLength { - for id, n := range c.entries { - if n <= c.ThresholdValue { + for id, cnt := range c.entries { + if cnt <= c.ThresholdValue { delete(c.entries, id) } } @@ -378,3 +380,26 @@ func (p uint64Slice) merge(other []uint64) []uint64 { return ret } + +// BitmapCache provides an interface for caching full bitmaps. +type BitmapCache interface { + Fetch(id uint64) (*Bitmap, bool) + Add(id uint64, b *Bitmap) +} + +// SimpleCache implements BitmapCache +// it is meant to be a short-lived cache for cases where writes are continuing to access +// the same bit within a short time frame (i.e. good for write-heavy loads) +// A read-heavy use case would cause the cache to get bigger, potentially causing the +// node to run out of memory. +type SimpleCache struct { + cache map[uint64]*Bitmap +} + +func (s *SimpleCache) Fetch(id uint64) (*Bitmap, bool) { + return s.cache[id] +} + +func (s *SimpleCache) Add(id uint64, b *Bitmap) { + s.cache[id] = b +} diff --git a/executor.go b/executor.go index 4a2d954c9..26b2f4747 100644 --- a/executor.go +++ b/executor.go @@ -185,6 +185,7 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic other := c.Clone() // Double the size of n for other calls in order to... + // TODO: travis review other.Args["n"] = len(bitmapIDs) * 2 ids := Pairs(pairs).Keys() diff --git a/fragment.go b/fragment.go index e203dd711..8f8698192 100644 --- a/fragment.go +++ b/fragment.go @@ -53,28 +53,6 @@ const ( DefaultFragmentMaxOpN = 2000 ) -// BitmapCacher implements SimpleCache -// it is meant to be a short-lived cache for cases where writes are continuing to access -// the same bit withing a short time frame (i.e. good for write-heavy loads) -// A read-heavy use case would cause the cache to get bigger, potentially causing the -// node to run out of memory. -type BitmapCacher interface { - Fetch(id uint64) (*Bitmap, bool) - Add(id uint64, b *Bitmap) -} - -type SimpleCache struct { - cache map[uint64]*Bitmap -} - -func (s *SimpleCache) Fetch(id uint64) (*Bitmap, bool) { - m, ok := s.cache[id] - return m, ok -} -func (s *SimpleCache) Add(id uint64, p *Bitmap) { - s.cache[id] = p -} - // Fragment represents the intersection of a frame and slice in a database. type Fragment struct { mu sync.Mutex @@ -91,9 +69,12 @@ type Fragment struct { storageData []byte opN int // number of ops since snapshot - // Bitmap cache. + // Cache for bitmap counts. cache Cache + // Cache containing full bitmaps (not just counts). + bitmapCache BitmapCache + // Cached checksums for each block. checksums map[int][]byte @@ -109,8 +90,7 @@ type Fragment struct { // This is set by the parent frame unless overridden for testing. BitmapAttrStore *AttrStore - stats StatsClient - bitmapCache BitmapCacher + stats StatsClient } // NewFragment returns a new instance of Fragment. @@ -351,10 +331,10 @@ func (f *Fragment) bitmap(bitmapID uint64, updateCache bool) *Bitmap { slice: f.slice, writable: false, }}, - cacheoveride: 0, + adjustedCount: 0, } bm.InvalidateCount() - bm.cacheoveride = bm.Count() + bm.adjustedCount = bm.Count() if updateCache { // Update cache. @@ -382,7 +362,6 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) } // Write to storage. - if changed, err = f.storage.Add(pos); err != nil { return false, err } @@ -400,18 +379,17 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) return false, err } - // Update the cache. + // If adjustedCount is set, then apply that value to bitmap.n instead. bm := f.bitmap(bitmapID, true) - if bm.cacheoveride > 0 { - bm.SetCount(profileID, bm.cacheoveride) - bm.cacheoveride = 0 + if bm.adjustedCount > 0 { + bm.SetCount(profileID, bm.adjustedCount) + bm.adjustedCount = 0 } else { bm.IncrementCount(profileID) + f.cache.Add(bitmapID, bm.Count()) } bm.SetBit(profileID) - f.cache.Add(bitmapID, bm.Count()) - f.stats.Count("setN", 1) return changed, nil @@ -451,15 +429,16 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (bool, error) { return false, err } - // Update the cache. - bm := f.bitmap(bitmapID, false) - if bm.cacheoveride > 0 { - bm.SetCount(profileID, bm.cacheoveride) - bm.cacheoveride = 0 + // If adjustedCount is set, then apply that value to bitmap.n instead. + bm := f.bitmap(bitmapID, true) + if bm.adjustedCount > 0 { + bm.SetCount(profileID, bm.adjustedCount) + bm.adjustedCount = 0 } else { bm.DecrementCount(profileID) + f.cache.Add(bitmapID, bm.Count()) } - f.cache.Add(bitmapID, bm.Count()) + bm.ClearBit(profileID) f.stats.Count("clearN", 1) @@ -546,7 +525,6 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { if count == 0 { continue } - //results = append(results, Pair{Key: bitmapID, Count: count}) heap.Push(results, Pair{Key: bitmapID, Count: count}) // If we reach the requested number of pairs and we are not computing @@ -556,7 +534,6 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { if opt.Src == nil { break } - // sort.Sort(Pairs(results)) } continue } @@ -622,7 +599,6 @@ func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair { } } sort.Sort(BitmapPairs(pairs)) - //debugDumpPairs(pairs) return pairs } From c013661880d0682521813f9374e16e8704636e7e Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 2 Mar 2017 16:00:34 -0600 Subject: [PATCH 33/61] bug fix in SimpleCache.Fetch (really just reverting my change) --- cache.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cache.go b/cache.go index 184310361..7c2e96f2d 100644 --- a/cache.go +++ b/cache.go @@ -397,7 +397,8 @@ type SimpleCache struct { } func (s *SimpleCache) Fetch(id uint64) (*Bitmap, bool) { - return s.cache[id] + m, ok := s.cache[id] + return m, ok } func (s *SimpleCache) Add(id uint64, b *Bitmap) { From 69cfdb0df9fd8f2394e46058e7fcda84cc3e85ea Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 2 Mar 2017 16:06:34 -0600 Subject: [PATCH 34/61] clean up --- cmd/pilosactl/main.go | 2 -- db_test.go | 2 +- fragment.go | 6 +++--- frame.go | 2 +- frame_test.go | 2 +- pilosa.go | 3 +-- 6 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 5491dbfca..bb22d14ac 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -303,7 +303,6 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) // Validate arguments. - fmt.Print("1213224") if cmd.Database == "" { return pilosa.ErrDatabaseRequired } else if cmd.Frame == "" { @@ -973,7 +972,6 @@ func (cmd *BenchCommand) Run(ctx context.Context) error { // runSetBit executes a benchmark of random SetBit() operations. func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) error { - fmt.Print("Nodb") if cmd.N == 0 { return errors.New("operation count required") } else if cmd.Database == "" { diff --git a/db_test.go b/db_test.go index f0bdc60e2..96120dc1e 100644 --- a/db_test.go +++ b/db_test.go @@ -149,4 +149,4 @@ func TestDB_InvalidName(t *testing.T) { if db != nil { t.Fatalf("unexpected db name %s", db) } -} \ No newline at end of file +} diff --git a/fragment.go b/fragment.go index 9bc542f93..1181f1739 100644 --- a/fragment.go +++ b/fragment.go @@ -213,11 +213,11 @@ func (f *Fragment) openCache() error { // Determine cache type from frame name. if strings.HasSuffix(f.frame, FrameSuffixRank) { c := NewRankCache() - c.ThresholdLength = 500000 - c.ThresholdIndex = 450000 + c.ThresholdLength = 50000 + c.ThresholdIndex = 45000 f.cache = c } else { - f.cache = NewLRUCache(500000) + f.cache = NewLRUCache(50000) } // Read cache data from disk. diff --git a/frame.go b/frame.go index 8ec419057..fa47517c5 100644 --- a/frame.go +++ b/frame.go @@ -46,7 +46,7 @@ type Frame struct { } // NewFrame returns a new instance of frame. -func NewFrame(path, db, name string) (*Frame, error){ +func NewFrame(path, db, name string) (*Frame, error) { err := ValidateName(name) if err != nil { return nil, err diff --git a/frame_test.go b/frame_test.go index 34f03ee15..042bd751e 100644 --- a/frame_test.go +++ b/frame_test.go @@ -96,7 +96,7 @@ func (f *Frame) Reopen() error { path, db, name := f.Path(), f.DB(), f.Name() f.Frame, err = pilosa.NewFrame(path, db, name) - if err != nil{ + if err != nil { return err } diff --git a/pilosa.go b/pilosa.go index f6efb0987..14e5ef203 100644 --- a/pilosa.go +++ b/pilosa.go @@ -80,7 +80,6 @@ func decodeProfile(pb *internal.Profile) *Profile { // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" - // Restrict name using regex func ValidateName(name string) error { expr := regexp.MustCompile(`^([a-z0-9._-]{1,64}$)`) @@ -89,4 +88,4 @@ func ValidateName(name string) error { return ErrName } return nil -} \ No newline at end of file +} From 02c3c6d3e69ccc8ccdc12e0101c5d267ecba4f3d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 13:03:57 -0600 Subject: [PATCH 35/61] use cobra/viper and move cmd/pilosa to server subcommand --- cmd/pilosa/main.go | 190 +---------------- cmd/root.go | 14 ++ cmd/server.go | 90 ++++++++ glide.lock | 50 ++++- glide.yaml | 2 + server/server.go | 197 ++++++++++++++++++ .../main_test.go => server/server_test.go | 10 +- 7 files changed, 360 insertions(+), 193 deletions(-) create mode 100644 cmd/root.go create mode 100644 cmd/server.go create mode 100644 server/server.go rename cmd/pilosa/main_test.go => server/server_test.go (99%) diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go index 4f7474996..255aef606 100644 --- a/cmd/pilosa/main.go +++ b/cmd/pilosa/main.go @@ -1,197 +1,15 @@ package main import ( - "errors" - "flag" "fmt" - "io" - "math/rand" "os" - "os/signal" - "path/filepath" - "runtime/pprof" - "strings" - "time" - "github.com/BurntSushi/toml" - "github.com/pilosa/pilosa" -) - -// Version and BuildTime hold the version/build time information passed in at compile time. -var ( - Version string - BuildTime string -) - -func init() { - if Version == "" { - Version = "v0.0.0" - } - if BuildTime == "" { - BuildTime = "not recorded" - } - - rand.Seed(time.Now().UTC().UnixNano()) -} - -const ( - // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" + "github.com/pilosa/pilosa/cmd" ) func main() { - m := NewMain() - m.Server.Handler.Version = Version - fmt.Fprintf(m.Stderr, "Pilosa %s, build time %s\n", Version, BuildTime) - - // Parse command line arguments. - if err := m.ParseFlags(os.Args[1:]); err != nil { - fmt.Fprintln(m.Stderr, err) - os.Exit(2) - } - - // Start CPU profiling. - if m.CPUProfile != "" { - f, err := os.Create(m.CPUProfile) - if err != nil { - fmt.Fprintf(m.Stderr, "create cpu profile: %v", err) - os.Exit(1) - } - defer f.Close() - - fmt.Fprintln(m.Stderr, "Starting cpu profile") - pprof.StartCPUProfile(f) - time.AfterFunc(m.CPUTime, func() { - fmt.Fprintln(m.Stderr, "Stopping cpu profile") - pprof.StopCPUProfile() - f.Close() - }) - } - - // Execute the program. - if err := m.Run(); err != nil { - fmt.Fprintln(m.Stderr, err) - fmt.Fprintln(m.Stderr, "stopping profile") - os.Exit(1) - } - - // First SIGKILL causes server to shut down gracefully. - c := make(chan os.Signal, 2) - signal.Notify(c, os.Interrupt) - sig := <-c - fmt.Fprintf(m.Stderr, "Received %s; gracefully shutting down...\n", sig.String()) - - // Second signal causes a hard shutdown. - go func() { <-c; os.Exit(1) }() - - if err := m.Close(); err != nil { - fmt.Fprintln(m.Stderr, err) - os.Exit(1) + if err := cmd.RootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(-1) } } - -// Main represents the main program execution. -type Main struct { - Server *pilosa.Server - - // Configuration options. - ConfigPath string - Config *pilosa.Config - - // Profiling options. - CPUProfile string - CPUTime time.Duration - - // Standard input/output - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewMain returns a new instance of Main. -func NewMain() *Main { - return &Main{ - Server: pilosa.NewServer(), - Config: pilosa.NewConfig(), - - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, - } -} - -// Run executes the main program execution. -func (m *Main) Run(args ...string) error { - // Notify user of config file. - if m.ConfigPath != "" { - fmt.Fprintf(m.Stdout, "Using config: %s\n", m.ConfigPath) - } - - // Setup logging output. - m.Server.LogOutput = m.Stderr - - // Configure index. - fmt.Fprintf(m.Stderr, "Using data from: %s\n", m.Config.DataDir) - m.Server.Index.Path = m.Config.DataDir - m.Server.Index.Stats = pilosa.NewExpvarStatsClient() - - // Build cluster from config file. - m.Server.Host = m.Config.Host - m.Server.Cluster = m.Config.PilosaCluster() - - // Set configuration options. - m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) - - // Initialize server. - if err := m.Server.Open(); err != nil { - return err - } - - fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host) - - return nil -} - -// Close shuts down the server. -func (m *Main) Close() error { - return m.Server.Close() -} - -// ParseFlags parses command line flags from args. -func (m *Main) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosa", flag.ContinueOnError) - fs.StringVar(&m.CPUProfile, "cpuprofile", "", "cpu profile") - fs.DurationVar(&m.CPUTime, "cputime", 30*time.Second, "cpu profile duration") - fs.StringVar(&m.ConfigPath, "config", "", "config path") - fs.SetOutput(m.Stderr) - if err := fs.Parse(args); err != nil { - return err - } - - // Load config, if specified. - if m.ConfigPath != "" { - if _, err := toml.DecodeFile(m.ConfigPath, &m.Config); err != nil { - return err - } - } - - // Use default data directory if one is not specified. - if m.Config.DataDir == "" { - m.Config.DataDir = DefaultDataDir - } - - // Expand home directory. - prefix := "~" + string(filepath.Separator) - if strings.HasPrefix(m.Config.DataDir, prefix) { - // u, err := user.Current() - HomeDir := os.Getenv("HOME") - /*if err != nil { - return err - } else*/if HomeDir == "" { - return errors.New("data directory not specified and no home dir available") - } - m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix)) - } - - return nil -} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 000000000..91d866a18 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,14 @@ +package cmd + +import "github.com/spf13/cobra" + +var RootCmd = &cobra.Command{ + Use: "pilosa", + Short: "pilosa - A Distributed In-memory Binary Bitmap Index", + Long: `Pilosa is a fast index to turbocharge your database. + +This binary contains Pilosa itself, as well as common +tools for administering pilosa, importing/exporting data, +backing up, and more. Complete documentation is available +at http://pilosa.com/docs`, // TODO - is documentation actually there? +} diff --git a/cmd/server.go b/cmd/server.go new file mode 100644 index 000000000..cc8bbec49 --- /dev/null +++ b/cmd/server.go @@ -0,0 +1,90 @@ +package cmd + +import ( + "fmt" + "log" + "os" + "os/signal" + "runtime/pprof" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/pilosa/pilosa/server" +) + +var serve = server.NewMain() + +var serveCmd = &cobra.Command{ + Use: "server", + Short: "server - run the pilosa server", + Long: `pilosa server runs Pilosa. + +It will load existing data from the configured +directory, and start listening client connections +on the configured port.`, + Run: func(cmd *cobra.Command, args []string) { + serve.Server.Handler.Version = server.Version + fmt.Fprintf(serve.Stderr, "Pilosa %s, build time %s\n", server.Version, server.BuildTime) + + // Parse command line arguments. + if err := serve.ParseFlags(os.Args[1:]); err != nil { + fmt.Fprintln(serve.Stderr, err) + os.Exit(2) + } + + // Start CPU profiling. + if serve.CPUProfile != "" { + f, err := os.Create(serve.CPUProfile) + if err != nil { + fmt.Fprintf(serve.Stderr, "create cpu profile: %v", err) + os.Exit(1) + } + defer f.Close() + + fmt.Fprintln(serve.Stderr, "Starting cpu profile") + pprof.StartCPUProfile(f) + time.AfterFunc(serve.CPUTime, func() { + fmt.Fprintln(serve.Stderr, "Stopping cpu profile") + pprof.StopCPUProfile() + f.Close() + }) + } + + // Execute the program. + if err := serve.Run(); err != nil { + fmt.Fprintln(serve.Stderr, err) + fmt.Fprintln(serve.Stderr, "stopping profile") + os.Exit(1) + } + + // First SIGKILL causes server to shut down gracefully. + c := make(chan os.Signal, 2) + signal.Notify(c, os.Interrupt) + sig := <-c + fmt.Fprintf(serve.Stderr, "Received %s; gracefully shutting down...\n", sig.String()) + + // Second signal causes a hard shutdown. + go func() { <-c; os.Exit(1) }() + + if err := serve.Close(); err != nil { + fmt.Fprintln(serve.Stderr, err) + os.Exit(1) + } + + }, +} + +func init() { + serveCmd.Flags().StringVarP(&serve.ConfigPath, "config", "c", "", "Configuration file to read from") + serveCmd.Flags().StringVarP(&serve.CPUProfile, "cpuprofile", "", "", "Where to store CPU profile") + serveCmd.Flags().DurationVarP(&serve.CPUTime, "cputime", "", 30*time.Second, "CPU profile duration") + + err := viper.BindPFlags(serveCmd.Flags()) + if err != nil { + log.Fatalf("Error binding server flags: %v", err) + } + + RootCmd.AddCommand(serveCmd) +} diff --git a/glide.lock b/glide.lock index af94d5502..001257bf1 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 469de49a1736f34a11e9b0e490f7c1da1d8cb0219fed4bf3ad9e71344ca7f58a -updated: 2017-02-09T17:03:01.816613507-06:00 +hash: 7de62dbaf3cc1dc4959f4f6d8213102cb182b4dd7a87b3ac29260ad6bc1b0cef +updated: 2017-03-03T12:25:48.088390296-06:00 imports: - name: github.com/boltdb/bolt version: 4b1ebc1869ad66568b313d0dc410e2be72670dda @@ -13,6 +13,8 @@ imports: version: 346938d642f2ec3594ed81d874461961cd0faa76 subpackages: - spew +- name: github.com/fsnotify/fsnotify + version: 7d7316ed6e1ed2de075aab8dfc76de5d158d66e1 - name: github.com/gogo/protobuf version: a9cd0c35b97daf74d0ebf3514c5254814b2703b4 subpackages: @@ -23,10 +25,54 @@ imports: - lru - name: github.com/golang/protobuf version: 888eb0692c857ec880338addf316bd662d5e630e + subpackages: + - proto +- name: github.com/hashicorp/hcl + version: 630949a3c5fa3c613328e1b8256052cbc2327c9b + subpackages: + - hcl/ast + - hcl/parser + - hcl/scanner + - hcl/strconv + - hcl/token + - json/parser + - json/scanner + - json/token +- name: github.com/inconshreveable/mousetrap + version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 +- name: github.com/magiconair/properties + version: b3b15ef068fd0b17ddf408a23669f20811d194d2 +- name: github.com/mitchellh/mapstructure + version: db1efb556f84b25a0a13a04aad883943538ad2e0 +- name: github.com/pelletier/go-buffruneio + version: c37440a7cf42ac63b919c752ca73a85067e05992 +- name: github.com/pelletier/go-toml + version: 13d49d4606eb801b8f01ae542b4afc4c6ee3d84a - name: github.com/satori/go.uuid version: 879c5887cd475cd7864858769793b2ceb0d44feb +- name: github.com/spf13/afero + version: 9be650865eab0c12963d8753212f4f9c66cdcf12 + subpackages: + - mem +- name: github.com/spf13/cast + version: 4f1683a2242a92e62d6ff705a30e435cbf2b50a3 +- name: github.com/spf13/cobra + version: fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c +- name: github.com/spf13/jwalterweatherman + version: fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66 +- name: github.com/spf13/pflag + version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 +- name: github.com/spf13/viper + version: 7538d73b4eb9511d85a9f1dfef202eeb8ac260f4 - name: golang.org/x/sys version: c200b10b5d5e122be351b67af224adc6128af5bf subpackages: - unix +- name: golang.org/x/text + version: 5a42fa2464759cbb7ee0af9de00b54d69f09a29c + subpackages: + - transform + - unicode/norm +- name: gopkg.in/yaml.v2 + version: a3f3340b5840cee44f372bddb5880fcbc419b46a testImports: [] diff --git a/glide.yaml b/glide.yaml index 10c8509d0..1c4f35dae 100644 --- a/glide.yaml +++ b/glide.yaml @@ -27,3 +27,5 @@ import: - package: github.com/golang/protobuf - package: github.com/satori/go.uuid version: ^1.1.0 +- package: github.com/spf13/cobra +- package: github.com/spf13/viper diff --git a/server/server.go b/server/server.go new file mode 100644 index 000000000..bb151a37d --- /dev/null +++ b/server/server.go @@ -0,0 +1,197 @@ +package server + +import ( + "errors" + "flag" + "fmt" + "io" + "math/rand" + "os" + "os/signal" + "path/filepath" + "runtime/pprof" + "strings" + "time" + + "github.com/BurntSushi/toml" + "github.com/pilosa/pilosa" +) + +// Version and BuildTime hold the version/build time information passed in at compile time. +var ( + Version string + BuildTime string +) + +func init() { + if Version == "" { + Version = "v0.0.0" + } + if BuildTime == "" { + BuildTime = "not recorded" + } + + rand.Seed(time.Now().UTC().UnixNano()) +} + +const ( + // DefaultDataDir is the default data directory. + DefaultDataDir = "~/.pilosa" +) + +func mainz() { + serve := NewMain() + serve.Server.Handler.Version = Version + fmt.Fprintf(serve.Stderr, "Pilosa %s, build time %s\n", Version, BuildTime) + + // Parse command line arguments. + if err := serve.ParseFlags(os.Args[1:]); err != nil { + fmt.Fprintln(serve.Stderr, err) + os.Exit(2) + } + + // Start CPU profiling. + if serve.CPUProfile != "" { + f, err := os.Create(serve.CPUProfile) + if err != nil { + fmt.Fprintf(serve.Stderr, "create cpu profile: %v", err) + os.Exit(1) + } + defer f.Close() + + fmt.Fprintln(serve.Stderr, "Starting cpu profile") + pprof.StartCPUProfile(f) + time.AfterFunc(serve.CPUTime, func() { + fmt.Fprintln(serve.Stderr, "Stopping cpu profile") + pprof.StopCPUProfile() + f.Close() + }) + } + + // Execute the program. + if err := serve.Run(); err != nil { + fmt.Fprintln(serve.Stderr, err) + fmt.Fprintln(serve.Stderr, "stopping profile") + os.Exit(1) + } + + // First SIGKILL causes server to shut down gracefully. + c := make(chan os.Signal, 2) + signal.Notify(c, os.Interrupt) + sig := <-c + fmt.Fprintf(serve.Stderr, "Received %s; gracefully shutting down...\n", sig.String()) + + // Second signal causes a hard shutdown. + go func() { <-c; os.Exit(1) }() + + if err := serve.Close(); err != nil { + fmt.Fprintln(serve.Stderr, err) + os.Exit(1) + } +} + +// Main represents the main program execution. +type Main struct { + Server *pilosa.Server + + // Configuration options. + ConfigPath string + Config *pilosa.Config + + // Profiling options. + CPUProfile string + CPUTime time.Duration + + // Standard input/output + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewMain returns a new instance of Main. +func NewMain() *Main { + return &Main{ + Server: pilosa.NewServer(), + Config: pilosa.NewConfig(), + + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + } +} + +// Run executes the main program execution. +func (m *Main) Run(args ...string) error { + // Notify user of config file. + if m.ConfigPath != "" { + fmt.Fprintf(m.Stdout, "Using config: %s\n", m.ConfigPath) + } + + // Setup logging output. + m.Server.LogOutput = m.Stderr + + // Configure index. + fmt.Fprintf(m.Stderr, "Using data from: %s\n", m.Config.DataDir) + m.Server.Index.Path = m.Config.DataDir + m.Server.Index.Stats = pilosa.NewExpvarStatsClient() + + // Build cluster from config file. + m.Server.Host = m.Config.Host + m.Server.Cluster = m.Config.PilosaCluster() + + // Set configuration options. + m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) + + // Initialize server. + if err := m.Server.Open(); err != nil { + return err + } + + fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host) + + return nil +} + +// Close shuts down the server. +func (m *Main) Close() error { + return m.Server.Close() +} + +// ParseFlags parses command line flags from args. +func (m *Main) ParseFlags(args []string) error { + fs := flag.NewFlagSet("pilosa", flag.ContinueOnError) + fs.StringVar(&m.CPUProfile, "cpuprofile", "", "cpu profile") + fs.DurationVar(&m.CPUTime, "cputime", 30*time.Second, "cpu profile duration") + fs.StringVar(&m.ConfigPath, "config", "", "config path") + fs.SetOutput(m.Stderr) + if err := fs.Parse(args); err != nil { + return err + } + + // Load config, if specified. + if m.ConfigPath != "" { + if _, err := toml.DecodeFile(m.ConfigPath, &m.Config); err != nil { + return err + } + } + + // Use default data directory if one is not specified. + if m.Config.DataDir == "" { + m.Config.DataDir = DefaultDataDir + } + + // Expand home directory. + prefix := "~" + string(filepath.Separator) + if strings.HasPrefix(m.Config.DataDir, prefix) { + // u, err := user.Current() + HomeDir := os.Getenv("HOME") + /*if err != nil { + return err + } else*/if HomeDir == "" { + return errors.New("data directory not specified and no home dir available") + } + m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix)) + } + + return nil +} diff --git a/cmd/pilosa/main_test.go b/server/server_test.go similarity index 99% rename from cmd/pilosa/main_test.go rename to server/server_test.go index 184f5c334..bb772ab45 100644 --- a/cmd/pilosa/main_test.go +++ b/server/server_test.go @@ -1,4 +1,4 @@ -package main_test +package server_test import ( "bytes" @@ -18,7 +18,7 @@ import ( "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" - main "github.com/pilosa/pilosa/cmd/pilosa" + "github.com/pilosa/pilosa/server" ) // Ensure program can process queries and maintain consistency. @@ -304,7 +304,7 @@ path = "/path/to/plugins" // Main represents a test wrapper for main.Main. type Main struct { - *main.Main + *server.Main Stdin bytes.Buffer Stdout bytes.Buffer @@ -318,7 +318,7 @@ func NewMain() *Main { panic(err) } - m := &Main{Main: main.NewMain()} + m := &Main{Main: server.NewMain()} m.Config.DataDir = path m.Config.Host = "localhost:0" m.Main.Stdin = &m.Stdin @@ -356,7 +356,7 @@ func (m *Main) Reopen() error { // Create new main with the same config. config := m.Config - m.Main = main.NewMain() + m.Main = server.NewMain() m.Config = config // Run new program. From ed43eca004396b001124cf8955267f9db7c13a71 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 13:35:06 -0600 Subject: [PATCH 36/61] move "config" to subcommand --- cmd/config.go | 29 ++++++++++++++++++++++ cmd/pilosactl/main.go | 57 ------------------------------------------- ctl/config.go | 43 ++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 57 deletions(-) create mode 100644 cmd/config.go create mode 100644 ctl/config.go diff --git a/cmd/config.go b/cmd/config.go new file mode 100644 index 000000000..72202ffe3 --- /dev/null +++ b/cmd/config.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "context" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/pilosa/pilosa/ctl" +) + +var conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr) + +var confCmd = &cobra.Command{ + Use: "config", + Short: "config - prints the default configuration", + Long: `config prints the default configuration to stdout +`, + Run: func(cmd *cobra.Command, args []string) { + if err := conf.Run(context.Background()); err != nil { + fmt.Println(err) + } + }, +} + +func init() { + RootCmd.AddCommand(confCmd) +} diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 88fa6fea5..8327839fe 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -96,7 +96,6 @@ Usage: The commands are: - config prints the default configuration import imports data from a CSV file export exports data to a CSV file sort sorts a data file for optimal import speed @@ -126,8 +125,6 @@ func (m *Main) ParseFlags(args []string) error { fmt.Fprintln(m.Stderr, m.Usage()) fmt.Fprintln(m.Stderr, "") return flag.ErrHelp - case "config": - m.Cmd = NewConfigCommand(m.Stdin, m.Stdout, m.Stderr) case "import": m.Cmd = pilosactl.NewImportCommand(m.Stdin, m.Stdout, m.Stderr) case "export": @@ -167,60 +164,6 @@ type Command interface { Run(context.Context) error } -// ConfigCommand represents a command for printing a default config. -type ConfigCommand struct { - // Standard input/output - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewConfigCommand returns a new instance of ConfigCommand. -func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *ConfigCommand { - return &ConfigCommand{ - Stdin: stdin, - Stdout: stdout, - Stderr: stderr, - } -} - -// ParseFlags parses command line flags from args. -func (cmd *ConfigCommand) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) - fs.SetOutput(ioutil.Discard) - if err := fs.Parse(args); err != nil { - return err - } - return nil -} - -// Usage returns the usage message to be printed. -func (cmd *ConfigCommand) Usage() string { - return strings.TrimSpace(` -usage: pilosactl config - -Prints the default configuration file to standard out. -`) -} - -// Run executes the main program execution. -func (cmd *ConfigCommand) Run(ctx context.Context) error { - fmt.Fprintln(cmd.Stdout, strings.TrimSpace(` -data-dir = "~/.pilosa" -host = "localhost:15000" - -[cluster] -replicas = 1 - -[[cluster.node]] -host = "localhost:15000" - -[plugins] -path = "" -`)+"\n") - return nil -} - // ExportCommand represents a command for bulk exporting data from a server. type ExportCommand struct { // Remote host and port. diff --git a/ctl/config.go b/ctl/config.go new file mode 100644 index 000000000..516069e2d --- /dev/null +++ b/ctl/config.go @@ -0,0 +1,43 @@ +package ctl + +import ( + "context" + "fmt" + "io" + "strings" +) + +// ConfigCommand represents a command for printing a default config. +type ConfigCommand struct { + // Standard input/output + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewConfigCommand returns a new instance of ConfigCommand. +func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *ConfigCommand { + return &ConfigCommand{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + } +} + +// Run executes the main program execution. +func (cmd *ConfigCommand) Run(ctx context.Context) error { + fmt.Fprintln(cmd.Stdout, strings.TrimSpace(` +data-dir = "~/.pilosa" +host = "localhost:15000" + +[cluster] +replicas = 1 + +[[cluster.node]] +host = "localhost:15000" + +[plugins] +path = "" +`)+"\n") + return nil +} From e349dca06ac1bae7561f13f2a8407e215fdaa284 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 14:11:12 -0600 Subject: [PATCH 37/61] move import to subcommand --- cmd/import.go | 50 ++++++++++++++++++++++++++++++++++++ cmd/pilosactl/main.go | 3 --- {pilosactl => ctl}/import.go | 40 +---------------------------- 3 files changed, 51 insertions(+), 42 deletions(-) create mode 100644 cmd/import.go rename {pilosactl => ctl}/import.go (78%) diff --git a/cmd/import.go b/cmd/import.go new file mode 100644 index 000000000..af1a7fcd5 --- /dev/null +++ b/cmd/import.go @@ -0,0 +1,50 @@ +package cmd + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/pilosa/pilosa/ctl" +) + +var importer = ctl.NewImportCommand(os.Stdin, os.Stdout, os.Stderr) + +var importCmd = &cobra.Command{ + Use: "import", + Short: "import - import data to pilosa", + Long: `Bulk imports one or more CSV files to a host's database and frame. The bits +of the CSV file are grouped by slice for the most efficient import. + +The format of the CSV file is: + + BITMAPID,PROFILEID,[TIME] + +The file should contain no headers. The TIME column is optional and can be +omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. +`, + Run: func(cmd *cobra.Command, args []string) { + importer.Paths = args + if err := importer.Run(context.Background()); err != nil { + fmt.Println(err) + } + }, +} + +func init() { + importCmd.Flags().StringVarP(&importer.Host, "host", "", "localhost:15000", "host:port of Pilosa.") + importCmd.Flags().StringVarP(&importer.Database, "database", "d", "", "Pilosa database to import into.") + importCmd.Flags().StringVarP(&importer.Frame, "frame", "f", "", "Frame to import into.") + importCmd.Flags().IntVarP(&importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") + + err := viper.BindPFlags(importCmd.Flags()) + if err != nil { + log.Fatalf("Error binding import flags: %v", err) + } + + RootCmd.AddCommand(importCmd) +} diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 8327839fe..8b0980815 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -22,7 +22,6 @@ import ( "unsafe" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/pilosactl" "github.com/pilosa/pilosa/roaring" ) @@ -125,8 +124,6 @@ func (m *Main) ParseFlags(args []string) error { fmt.Fprintln(m.Stderr, m.Usage()) fmt.Fprintln(m.Stderr, "") return flag.ErrHelp - case "import": - m.Cmd = pilosactl.NewImportCommand(m.Stdin, m.Stdout, m.Stderr) case "export": m.Cmd = NewExportCommand(m.Stdin, m.Stdout, m.Stderr) case "sort": diff --git a/pilosactl/import.go b/ctl/import.go similarity index 78% rename from pilosactl/import.go rename to ctl/import.go index 9d4ad63f4..39e46eb3d 100644 --- a/pilosactl/import.go +++ b/ctl/import.go @@ -1,17 +1,14 @@ -package pilosactl +package ctl import ( "context" "encoding/csv" "errors" - "flag" "fmt" "io" - "io/ioutil" "log" "os" "strconv" - "strings" "time" "github.com/pilosa/pilosa" @@ -56,41 +53,6 @@ func (cmd *ImportCommand) String() string { return fmt.Sprint(*cmd) } -// ParseFlags parses command line flags from args. -func (cmd *ImportCommand) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) - fs.SetOutput(ioutil.Discard) - fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port") - fs.StringVar(&cmd.Database, "d", "", "database") - fs.StringVar(&cmd.Frame, "f", "", "frame") - fs.IntVar(&cmd.BufferSize, "buffer-size", cmd.BufferSize, "buffer size") - if err := fs.Parse(args); err != nil { - return err - } - - // Extract the import paths. - cmd.Paths = fs.Args() - - return nil -} - -// Usage returns the usage message to be printed. -func (cmd *ImportCommand) Usage() string { - return strings.TrimSpace(` -usage: pilosactl import -host HOST -d database -f frame paths - -Bulk imports one or more CSV files to a host's database and frame. The bits -of the CSV file are grouped by slice for the most efficient import. - -The format of the CSV file is: - - BITMAPID,PROFILEID,[TIME] - -The file should contain no headers. The TIME column is optional and can be -omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. -`) -} - // Run executes the main program execution. func (cmd *ImportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) From 362539fe2dd4ecb906248e6375f85d87573148fc Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 14:18:52 -0600 Subject: [PATCH 38/61] make export a subcommand --- cmd/export.go | 49 ++++++++++++++++++ cmd/pilosactl/main.go | 116 ------------------------------------------ ctl/export.go | 91 +++++++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 116 deletions(-) create mode 100644 cmd/export.go create mode 100644 ctl/export.go diff --git a/cmd/export.go b/cmd/export.go new file mode 100644 index 000000000..16afd2475 --- /dev/null +++ b/cmd/export.go @@ -0,0 +1,49 @@ +package cmd + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/pilosa/pilosa/ctl" +) + +var exporter = ctl.NewExportCommand(os.Stdin, os.Stdout, os.Stderr) + +var exportCmd = &cobra.Command{ + Use: "export", + Short: "export - export data from pilosa", + Long: ` +Bulk exports a fragment to a CSV file. If the OUTFILE is not specified then +the output is written to STDOUT. + +The format of the CSV file is: + + BITMAPID,PROFILEID + +The file does not contain any headers. +`, + Run: func(cmd *cobra.Command, args []string) { + if err := exporter.Run(context.Background()); err != nil { + fmt.Println(err) + } + }, +} + +func init() { + exportCmd.Flags().StringVarP(&exporter.Host, "host", "", "localhost:15000", "host:port of Pilosa.") + exportCmd.Flags().StringVarP(&exporter.Database, "database", "d", "", "Pilosa database to export into.") + exportCmd.Flags().StringVarP(&exporter.Frame, "frame", "f", "", "Frame to export into.") + exportCmd.Flags().StringVarP(&exporter.Frame, "output-file", "o", "", "File to write export to - default stdout") + + err := viper.BindPFlags(exportCmd.Flags()) + if err != nil { + log.Fatalf("Error binding export flags: %v", err) + } + + RootCmd.AddCommand(exportCmd) +} diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 8b0980815..a1524bc35 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -95,8 +95,6 @@ Usage: The commands are: - import imports data from a CSV file - export exports data to a CSV file sort sorts a data file for optimal import speed backup backs up a frame to an archive file restore restores a frame from an archive file @@ -124,8 +122,6 @@ func (m *Main) ParseFlags(args []string) error { fmt.Fprintln(m.Stderr, m.Usage()) fmt.Fprintln(m.Stderr, "") return flag.ErrHelp - case "export": - m.Cmd = NewExportCommand(m.Stdin, m.Stdout, m.Stderr) case "sort": m.Cmd = NewSortCommand(m.Stdin, m.Stdout, m.Stderr) case "backup": @@ -161,118 +157,6 @@ type Command interface { Run(context.Context) error } -// ExportCommand represents a command for bulk exporting data from a server. -type ExportCommand struct { - // Remote host and port. - Host string - - // Name of the database & frame to export from. - Database string - Frame string - - // Filename to export to. - Path string - - // Standard input/output - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewExportCommand returns a new instance of ExportCommand. -func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *ExportCommand { - return &ExportCommand{ - Stdin: stdin, - Stdout: stdout, - Stderr: stderr, - } -} - -// ParseFlags parses command line flags from args. -func (cmd *ExportCommand) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) - fs.SetOutput(ioutil.Discard) - fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port") - fs.StringVar(&cmd.Database, "d", "", "database") - fs.StringVar(&cmd.Frame, "f", "", "frame") - fs.StringVar(&cmd.Path, "o", "", "output file") - if err := fs.Parse(args); err != nil { - return err - } - - return nil -} - -// Usage returns the usage message to be printed. -func (cmd *ExportCommand) Usage() string { - return strings.TrimSpace(` -usage: pilosactl export -host HOST -d database -f frame -o OUTFILE - -Bulk exports a fragment to a CSV file. If the OUTFILE is not specified then -the output is written to STDOUT. - -The format of the CSV file is: - - BITMAPID,PROFILEID - -The file does not contain any headers. -`) -} - -// Run executes the main program execution. -func (cmd *ExportCommand) Run(ctx context.Context) error { - logger := log.New(cmd.Stderr, "", log.LstdFlags) - - // Validate arguments. - if cmd.Database == "" { - return pilosa.ErrDatabaseRequired - } else if cmd.Frame == "" { - return pilosa.ErrFrameRequired - } - - // Use output file, if specified. - // Otherwise use STDOUT. - var w io.Writer = cmd.Stdout - if cmd.Path != "" { - f, err := os.Create(cmd.Path) - if err != nil { - return err - } - defer f.Close() - - w = f - } - - // Create a client to the server. - client, err := pilosa.NewClient(cmd.Host) - if err != nil { - return err - } - - // Determine slice count. - maxSlices, err := client.MaxSliceByDatabase(ctx) - if err != nil { - return err - } - - // Export each slice. - for slice := uint64(0); slice <= maxSlices[cmd.Database]; slice++ { - logger.Printf("exporting slice: %d", slice) - if err := client.ExportCSV(ctx, cmd.Database, cmd.Frame, slice, w); err != nil { - return err - } - } - - // Close writer, if applicable. - if w, ok := w.(io.Closer); ok { - if err := w.Close(); err != nil { - return err - } - } - - return nil -} - // SortCommand represents a command for sorting import data. type SortCommand struct { // Filename to sort diff --git a/ctl/export.go b/ctl/export.go new file mode 100644 index 000000000..f19c7123f --- /dev/null +++ b/ctl/export.go @@ -0,0 +1,91 @@ +package ctl + +import ( + "context" + "io" + "log" + "os" + + "github.com/pilosa/pilosa" +) + +// ExportCommand represents a command for bulk exporting data from a server. +type ExportCommand struct { + // Remote host and port. + Host string + + // Name of the database & frame to export from. + Database string + Frame string + + // Filename to export to. + Path string + + // Standard input/output + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewExportCommand returns a new instance of ExportCommand. +func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *ExportCommand { + return &ExportCommand{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + } +} + +// Run executes the main program execution. +func (cmd *ExportCommand) Run(ctx context.Context) error { + logger := log.New(cmd.Stderr, "", log.LstdFlags) + + // Validate arguments. + if cmd.Database == "" { + return pilosa.ErrDatabaseRequired + } else if cmd.Frame == "" { + return pilosa.ErrFrameRequired + } + + // Use output file, if specified. + // Otherwise use STDOUT. + var w io.Writer = cmd.Stdout + if cmd.Path != "" { + f, err := os.Create(cmd.Path) + if err != nil { + return err + } + defer f.Close() + + w = f + } + + // Create a client to the server. + client, err := pilosa.NewClient(cmd.Host) + if err != nil { + return err + } + + // Determine slice count. + maxSlices, err := client.MaxSliceByDatabase(ctx) + if err != nil { + return err + } + + // Export each slice. + for slice := uint64(0); slice <= maxSlices[cmd.Database]; slice++ { + logger.Printf("exporting slice: %d", slice) + if err := client.ExportCSV(ctx, cmd.Database, cmd.Frame, slice, w); err != nil { + return err + } + } + + // Close writer, if applicable. + if w, ok := w.(io.Closer); ok { + if err := w.Close(); err != nil { + return err + } + } + + return nil +} From 312a9a533430eafefddd79bcc58a919f0e09d2fe Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 14:38:44 -0600 Subject: [PATCH 39/61] move sort to subcommand --- cmd/pilosactl/main.go | 164 ------------------------------------------ cmd/sort.go | 45 ++++++++++++ ctl/sort.go | 138 +++++++++++++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 164 deletions(-) create mode 100644 cmd/sort.go create mode 100644 ctl/sort.go diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index a1524bc35..e3052ac12 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -1,20 +1,15 @@ package main import ( - "bufio" "context" - "encoding/csv" "errors" "flag" "fmt" "io" "io/ioutil" - "log" "math/rand" "os" "path/filepath" - "sort" - "strconv" "strings" "syscall" "text/tabwriter" @@ -95,7 +90,6 @@ Usage: The commands are: - sort sorts a data file for optimal import speed backup backs up a frame to an archive file restore restores a frame from an archive file inspect inspects fragment data files @@ -122,8 +116,6 @@ func (m *Main) ParseFlags(args []string) error { fmt.Fprintln(m.Stderr, m.Usage()) fmt.Fprintln(m.Stderr, "") return flag.ErrHelp - case "sort": - m.Cmd = NewSortCommand(m.Stdin, m.Stdout, m.Stderr) case "backup": m.Cmd = NewBackupCommand(m.Stdin, m.Stdout, m.Stderr) case "restore": @@ -157,120 +149,6 @@ type Command interface { Run(context.Context) error } -// SortCommand represents a command for sorting import data. -type SortCommand struct { - // Filename to sort - Path string - - // Standard input/output - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewSortCommand returns a new instance of SortCommand. -func NewSortCommand(stdin io.Reader, stdout, stderr io.Writer) *SortCommand { - return &SortCommand{ - Stdin: stdin, - Stdout: stdout, - Stderr: stderr, - } -} - -// ParseFlags parses command line flags from args. -func (cmd *SortCommand) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) - fs.SetOutput(ioutil.Discard) - if err := fs.Parse(args); err != nil { - return err - } - - // Extract the data path. - if fs.NArg() == 0 { - return errors.New("path required") - } else if fs.NArg() > 1 { - return errors.New("only one path allowed") - } - cmd.Path = fs.Arg(0) - - return nil -} - -// Usage returns the usage message to be printed. -func (cmd *SortCommand) Usage() string { - return strings.TrimSpace(` -usage: pilosactl sort PATH - -Sorts the import data at PATH into the optimal sort order for importing. - -The format of the CSV file is: - - BITMAPID,PROFILEID - -The file should contain no headers. -`) -} - -// Run executes the main program execution. -func (cmd *SortCommand) Run(ctx context.Context) error { - // Open file for reading. - f, err := os.Open(cmd.Path) - if err != nil { - return err - } - defer f.Close() - - // Read rows as bits. - r := csv.NewReader(f) - r.FieldsPerRecord = -1 - a := make([]pilosa.Bit, 0, 1000000) - for { - bitmapID, profileID, timestamp, err := readCSVRow(r) - if err == io.EOF { - break - } else if err == errBlank { - continue - } else if err != nil { - return err - } - a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID, Timestamp: timestamp}) - } - - // Sort bits by position. - sort.Sort(pilosa.BitsByPos(a)) - - // Rewrite to STDOUT. - w := bufio.NewWriter(cmd.Stdout) - buf := make([]byte, 0, 1024) - for _, bit := range a { - // Write CSV to buffer. - buf = buf[:0] - buf = strconv.AppendUint(buf, bit.BitmapID, 10) - - buf = append(buf, ',') - buf = strconv.AppendUint(buf, bit.ProfileID, 10) - - if bit.Timestamp != 0 { - buf = append(buf, ',') - buf = append(buf, time.Unix(0, bit.Timestamp).UTC().Format(pilosa.TimeFormat)...) - } - - buf = append(buf, '\n') - - // Write to output. - if _, err := w.Write(buf); err != nil { - return err - } - } - - // Ensure buffer is flushed before exiting. - if err := w.Flush(); err != nil { - return err - } - - return nil -} - // BackupCommand represents a command for backing up a frame. type BackupCommand struct { // Destination host and port. @@ -808,45 +686,3 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) e return nil } - -// readCSVRow reads a bitmap/profile pair from a CSV row. -func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err error) { - // Read CSV row. - record, err := r.Read() - if err != nil { - return 0, 0, 0, err - } - - // Ignore blank rows. - if record[0] == "" { - return 0, 0, 0, errBlank - } else if len(record) < 2 { - return 0, 0, 0, fmt.Errorf("bad column count: %d", len(record)) - } - - // Parse bitmap id. - bitmapID, err = strconv.ParseUint(record[0], 10, 64) - if err != nil { - return 0, 0, 0, fmt.Errorf("invalid bitmap id: %q", record[0]) - } - - // Parse bitmap id. - profileID, err = strconv.ParseUint(record[1], 10, 64) - if err != nil { - return 0, 0, 0, fmt.Errorf("invalid profile id: %q", record[1]) - } - - // Parse timestamp, if available. - if len(record) > 2 && record[2] != "" { - t, err := time.Parse(pilosa.TimeFormat, record[2]) - if err != nil { - return 0, 0, 0, fmt.Errorf("invalid timestamp: %q", record[2]) - } - timestamp = t.UnixNano() - } - - return bitmapID, profileID, timestamp, nil -} - -// errBlank indicates a blank row in a CSV file. -var errBlank = errors.New("blank row") diff --git a/cmd/sort.go b/cmd/sort.go new file mode 100644 index 000000000..813624ddc --- /dev/null +++ b/cmd/sort.go @@ -0,0 +1,45 @@ +package cmd + +import ( + "context" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/pilosa/pilosa/ctl" +) + +var sorter = ctl.NewSortCommand(os.Stdin, os.Stdout, os.Stderr) + +var sortCmd = &cobra.Command{ + Use: "sort ", + Short: "sort - sort import data for optimal import performance", + Long: ` +Sorts the import data at PATH into the optimal sort order for importing. + +The format of the CSV file is: + + BITMAPID,PROFILEID + +The file should contain no headers. +`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(cmd.Flags()) + if len(args) == 0 { + fmt.Println("path required") + return + } else if len(args) > 1 { + fmt.Println("only one path supported") + return + } + sorter.Path = args[0] + if err := sorter.Run(context.Background()); err != nil { + fmt.Println(err) + } + }, +} + +func init() { + RootCmd.AddCommand(sortCmd) +} diff --git a/ctl/sort.go b/ctl/sort.go new file mode 100644 index 000000000..eac68b7ab --- /dev/null +++ b/ctl/sort.go @@ -0,0 +1,138 @@ +package ctl + +import ( + "bufio" + "context" + "encoding/csv" + "errors" + "fmt" + "io" + "os" + "sort" + "strconv" + "time" + + "github.com/pilosa/pilosa" +) + +// SortCommand represents a command for sorting import data. +type SortCommand struct { + // Filename to sort + Path string + + // Standard input/output + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewSortCommand returns a new instance of SortCommand. +func NewSortCommand(stdin io.Reader, stdout, stderr io.Writer) *SortCommand { + return &SortCommand{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + } +} + +// Run executes the main program execution. +func (cmd *SortCommand) Run(ctx context.Context) error { + // Open file for reading. + f, err := os.Open(cmd.Path) + if err != nil { + return err + } + defer f.Close() + + // Read rows as bits. + r := csv.NewReader(f) + r.FieldsPerRecord = -1 + a := make([]pilosa.Bit, 0, 1000000) + for { + bitmapID, profileID, timestamp, err := readCSVRow(r) + if err == io.EOF { + break + } else if err == errBlank { + continue + } else if err != nil { + return err + } + a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID, Timestamp: timestamp}) + } + + // Sort bits by position. + sort.Sort(pilosa.BitsByPos(a)) + + // Rewrite to STDOUT. + w := bufio.NewWriter(cmd.Stdout) + buf := make([]byte, 0, 1024) + for _, bit := range a { + // Write CSV to buffer. + buf = buf[:0] + buf = strconv.AppendUint(buf, bit.BitmapID, 10) + + buf = append(buf, ',') + buf = strconv.AppendUint(buf, bit.ProfileID, 10) + + if bit.Timestamp != 0 { + buf = append(buf, ',') + buf = append(buf, time.Unix(0, bit.Timestamp).UTC().Format(pilosa.TimeFormat)...) + } + + buf = append(buf, '\n') + + // Write to output. + if _, err := w.Write(buf); err != nil { + return err + } + } + + // Ensure buffer is flushed before exiting. + if err := w.Flush(); err != nil { + return err + } + + return nil +} + +// readCSVRow reads a bitmap/profile pair from a CSV row. +func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err error) { + // Read CSV row. + record, err := r.Read() + if err != nil { + return 0, 0, 0, err + } + + // Ignore blank rows. + if record[0] == "" { + return 0, 0, 0, errBlank + } else if len(record) < 2 { + return 0, 0, 0, fmt.Errorf("bad column count: %d", len(record)) + } + + // Parse bitmap id. + bitmapID, err = strconv.ParseUint(record[0], 10, 64) + if err != nil { + return 0, 0, 0, fmt.Errorf("invalid bitmap id: %q", record[0]) + } + + // Parse bitmap id. + profileID, err = strconv.ParseUint(record[1], 10, 64) + if err != nil { + return 0, 0, 0, fmt.Errorf("invalid profile id: %q", record[1]) + } + + // Parse timestamp, if available. + if len(record) > 2 && record[2] != "" { + t, err := time.Parse(pilosa.TimeFormat, record[2]) + if err != nil { + return 0, 0, 0, fmt.Errorf("invalid timestamp: %q", record[2]) + } + timestamp = t.UnixNano() + } + + return bitmapID, profileID, timestamp, nil +} + +// errBlank indicates a blank row in a CSV file. +var errBlank = errors.New("blank row") From f20e6c193d01207ead2191ddd318d28a1d0ef155 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 14:46:45 -0600 Subject: [PATCH 40/61] move backup to subcommand --- cmd/backup.go | 42 ++++++++++++++++++++ cmd/pilosactl/main.go | 89 ------------------------------------------- ctl/backup.go | 72 ++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 89 deletions(-) create mode 100644 cmd/backup.go create mode 100644 ctl/backup.go diff --git a/cmd/backup.go b/cmd/backup.go new file mode 100644 index 000000000..883ce7be8 --- /dev/null +++ b/cmd/backup.go @@ -0,0 +1,42 @@ +package cmd + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/pilosa/pilosa/ctl" +) + +var backuper = ctl.NewBackupCommand(os.Stdin, os.Stdout, os.Stderr) + +var backupCmd = &cobra.Command{ + Use: "backup", + Short: "backup - backup data from pilosa", + Long: ` +Backs up the database and frame from across the cluster into a single file. +`, + Run: func(cmd *cobra.Command, args []string) { + if err := backuper.Run(context.Background()); err != nil { + fmt.Println(err) + } + }, +} + +func init() { + backupCmd.Flags().StringVarP(&backuper.Host, "host", "", "localhost:15000", "host:port of Pilosa.") + backupCmd.Flags().StringVarP(&backuper.Database, "database", "d", "", "Pilosa database to backup into.") + backupCmd.Flags().StringVarP(&backuper.Frame, "frame", "f", "", "Frame to backup into.") + backupCmd.Flags().StringVarP(&backuper.Path, "output-file", "o", "", "File to write backup to - default stdout") + + err := viper.BindPFlags(backupCmd.Flags()) + if err != nil { + log.Fatalf("Error binding backup flags: %v", err) + } + + RootCmd.AddCommand(backupCmd) +} diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index e3052ac12..72399e9b2 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -90,7 +90,6 @@ Usage: The commands are: - backup backs up a frame to an archive file restore restores a frame from an archive file inspect inspects fragment data files check performs a consistency check of data files @@ -116,8 +115,6 @@ func (m *Main) ParseFlags(args []string) error { fmt.Fprintln(m.Stderr, m.Usage()) fmt.Fprintln(m.Stderr, "") return flag.ErrHelp - case "backup": - m.Cmd = NewBackupCommand(m.Stdin, m.Stdout, m.Stderr) case "restore": m.Cmd = NewRestoreCommand(m.Stdin, m.Stdout, m.Stderr) case "inspect": @@ -149,92 +146,6 @@ type Command interface { Run(context.Context) error } -// BackupCommand represents a command for backing up a frame. -type BackupCommand struct { - // Destination host and port. - Host string - - // Name of the database & frame to backup. - Database string - Frame string - - // Output file to write to. - Path string - - // Standard input/output - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewBackupCommand returns a new instance of BackupCommand. -func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand { - return &BackupCommand{ - Stdin: stdin, - Stdout: stdout, - Stderr: stderr, - } -} - -// ParseFlags parses command line flags from args. -func (cmd *BackupCommand) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) - fs.SetOutput(ioutil.Discard) - fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port") - fs.StringVar(&cmd.Database, "d", "", "database") - fs.StringVar(&cmd.Frame, "f", "", "frame") - fs.StringVar(&cmd.Path, "o", "", "output file") - if err := fs.Parse(args); err != nil { - return err - } - - return nil -} - -// Usage returns the usage message to be printed. -func (cmd *BackupCommand) Usage() string { - return strings.TrimSpace(` -usage: pilosactl backup -host HOST -d database -f frame -o PATH - -Backs up the database and frame from across the cluster into a single file. -`) -} - -// Run executes the main program execution. -func (cmd *BackupCommand) Run(ctx context.Context) error { - // Validate arguments. - if cmd.Path == "" { - return errors.New("output file required") - } - - // Create a client to the server. - client, err := pilosa.NewClient(cmd.Host) - if err != nil { - return err - } - - // Open output file. - f, err := os.Create(cmd.Path) - if err != nil { - return err - } - defer f.Close() - - // Begin streaming backup. - if err := client.BackupTo(ctx, f, cmd.Database, cmd.Frame); err != nil { - return err - } - - // Sync & close file to ensure durability. - if err := f.Sync(); err != nil { - return err - } else if err = f.Close(); err != nil { - return err - } - - return nil -} - // RestoreCommand represents a command for restoring a frame from a backup. type RestoreCommand struct { // Destination host and port. diff --git a/ctl/backup.go b/ctl/backup.go new file mode 100644 index 000000000..23b34cd1f --- /dev/null +++ b/ctl/backup.go @@ -0,0 +1,72 @@ +package ctl + +import ( + "context" + "errors" + "io" + "os" + + "github.com/pilosa/pilosa" +) + +// BackupCommand represents a command for backing up a frame. +type BackupCommand struct { + // Destination host and port. + Host string + + // Name of the database & frame to backup. + Database string + Frame string + + // Output file to write to. + Path string + + // Standard input/output + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewBackupCommand returns a new instance of BackupCommand. +func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand { + return &BackupCommand{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + } +} + +// Run executes the main program execution. +func (cmd *BackupCommand) Run(ctx context.Context) error { + // Validate arguments. + if cmd.Path == "" { + return errors.New("output file required") + } + + // Create a client to the server. + client, err := pilosa.NewClient(cmd.Host) + if err != nil { + return err + } + + // Open output file. + f, err := os.Create(cmd.Path) + if err != nil { + return err + } + defer f.Close() + + // Begin streaming backup. + if err := client.BackupTo(ctx, f, cmd.Database, cmd.Frame); err != nil { + return err + } + + // Sync & close file to ensure durability. + if err := f.Sync(); err != nil { + return err + } else if err = f.Close(); err != nil { + return err + } + + return nil +} From 6dc164da69ed49543997245b06b06edc22aee1e6 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 14:47:47 -0600 Subject: [PATCH 41/61] fix copy/paste bug in export subcommand --- cmd/export.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/export.go b/cmd/export.go index 16afd2475..a5cd0f62d 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -38,7 +38,7 @@ func init() { exportCmd.Flags().StringVarP(&exporter.Host, "host", "", "localhost:15000", "host:port of Pilosa.") exportCmd.Flags().StringVarP(&exporter.Database, "database", "d", "", "Pilosa database to export into.") exportCmd.Flags().StringVarP(&exporter.Frame, "frame", "f", "", "Frame to export into.") - exportCmd.Flags().StringVarP(&exporter.Frame, "output-file", "o", "", "File to write export to - default stdout") + exportCmd.Flags().StringVarP(&exporter.Path, "output-file", "o", "", "File to write export to - default stdout") err := viper.BindPFlags(exportCmd.Flags()) if err != nil { From 21478cb85270052e4817d9f26a1c5431757c377a Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 14:53:35 -0600 Subject: [PATCH 42/61] move restore to subcommand --- cmd/pilosactl/main.go | 89 ------------------------------------------- cmd/restore.go | 42 ++++++++++++++++++++ ctl/restore.go | 65 +++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 89 deletions(-) create mode 100644 cmd/restore.go create mode 100644 ctl/restore.go diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 72399e9b2..ee77ba2a8 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -90,7 +90,6 @@ Usage: The commands are: - restore restores a frame from an archive file inspect inspects fragment data files check performs a consistency check of data files bench benchmarks operations @@ -115,8 +114,6 @@ func (m *Main) ParseFlags(args []string) error { fmt.Fprintln(m.Stderr, m.Usage()) fmt.Fprintln(m.Stderr, "") return flag.ErrHelp - case "restore": - m.Cmd = NewRestoreCommand(m.Stdin, m.Stdout, m.Stderr) case "inspect": m.Cmd = NewInspectCommand(m.Stdin, m.Stdout, m.Stderr) case "check": @@ -146,92 +143,6 @@ type Command interface { Run(context.Context) error } -// RestoreCommand represents a command for restoring a frame from a backup. -type RestoreCommand struct { - // Destination host and port. - Host string - - // Name of the database & frame to backup. - Database string - Frame string - - // Import file to read from. - Path string - - // Standard input/output - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewRestoreCommand returns a new instance of RestoreCommand. -func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand { - return &RestoreCommand{ - Stdin: stdin, - Stdout: stdout, - Stderr: stderr, - } -} - -// ParseFlags parses command line flags from args. -func (cmd *RestoreCommand) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) - fs.SetOutput(ioutil.Discard) - fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port") - fs.StringVar(&cmd.Database, "d", "", "database") - fs.StringVar(&cmd.Frame, "f", "", "frame") - if err := fs.Parse(args); err != nil { - return err - } - - // Read input path from the args. - if fs.NArg() == 0 { - return errors.New("path required") - } else if fs.NArg() > 1 { - return errors.New("too many paths specified") - } - cmd.Path = fs.Arg(0) - - return nil -} - -// Usage returns the usage message to be printed. -func (cmd *RestoreCommand) Usage() string { - return strings.TrimSpace(` -usage: pilosactl restore -host HOST -d database -f frame PATH - -Restores a frame to the cluster from a backup file. -`) -} - -// Run executes the main program execution. -func (cmd *RestoreCommand) Run(ctx context.Context) error { - // Validate arguments. - if cmd.Path == "" { - return errors.New("backup file required") - } - - // Create a client to the server. - client, err := pilosa.NewClient(cmd.Host) - if err != nil { - return err - } - - // Open backup file. - f, err := os.Open(cmd.Path) - if err != nil { - return err - } - defer f.Close() - - // Restore backup file to the cluster. - if err := client.RestoreFrom(ctx, f, cmd.Database, cmd.Frame); err != nil { - return err - } - - return nil -} - // InspectCommand represents a command for inspecting fragment data files. type InspectCommand struct { // Path to data file diff --git a/cmd/restore.go b/cmd/restore.go new file mode 100644 index 000000000..420cd9c5f --- /dev/null +++ b/cmd/restore.go @@ -0,0 +1,42 @@ +package cmd + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/pilosa/pilosa/ctl" +) + +var restorer = ctl.NewRestoreCommand(os.Stdin, os.Stdout, os.Stderr) + +var restoreCmd = &cobra.Command{ + Use: "restore", + Short: "restore - restore data to pilosa", + Long: ` +Restores a frame to the cluster from a backup file. +`, + Run: func(cmd *cobra.Command, args []string) { + if err := restorer.Run(context.Background()); err != nil { + fmt.Println(err) + } + }, +} + +func init() { + restoreCmd.Flags().StringVarP(&restorer.Host, "host", "", "localhost:15000", "host:port of Pilosa.") + restoreCmd.Flags().StringVarP(&restorer.Database, "database", "d", "", "Pilosa database to restore into.") + restoreCmd.Flags().StringVarP(&restorer.Frame, "frame", "f", "", "Frame to restore into.") + restoreCmd.Flags().StringVarP(&restorer.Path, "input-file", "i", "", "File to write restore from") + + err := viper.BindPFlags(restoreCmd.Flags()) + if err != nil { + log.Fatalf("Error binding restore flags: %v", err) + } + + RootCmd.AddCommand(restoreCmd) +} diff --git a/ctl/restore.go b/ctl/restore.go new file mode 100644 index 000000000..b9a573704 --- /dev/null +++ b/ctl/restore.go @@ -0,0 +1,65 @@ +package ctl + +import ( + "context" + "errors" + "io" + "os" + + "github.com/pilosa/pilosa" +) + +// RestoreCommand represents a command for restoring a frame from a backup. +type RestoreCommand struct { + // Destination host and port. + Host string + + // Name of the database & frame to backup. + Database string + Frame string + + // Import file to read from. + Path string + + // Standard input/output + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewRestoreCommand returns a new instance of RestoreCommand. +func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand { + return &RestoreCommand{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + } +} + +// Run executes the main program execution. +func (cmd *RestoreCommand) Run(ctx context.Context) error { + // Validate arguments. + if cmd.Path == "" { + return errors.New("backup file required") + } + + // Create a client to the server. + client, err := pilosa.NewClient(cmd.Host) + if err != nil { + return err + } + + // Open backup file. + f, err := os.Open(cmd.Path) + if err != nil { + return err + } + defer f.Close() + + // Restore backup file to the cluster. + if err := client.RestoreFrom(ctx, f, cmd.Database, cmd.Frame); err != nil { + return err + } + + return nil +} From 2c1929a063f7560c9ea95ba644c89ceeeff5768d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 15:00:54 -0600 Subject: [PATCH 43/61] move inspect to subcommand --- cmd/inspect.go | 39 +++++++++++++ cmd/pilosactl/main.go | 114 ------------------------------------- ctl/inspect.go | 127 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 114 deletions(-) create mode 100644 cmd/inspect.go create mode 100644 ctl/inspect.go diff --git a/cmd/inspect.go b/cmd/inspect.go new file mode 100644 index 000000000..48bbf8308 --- /dev/null +++ b/cmd/inspect.go @@ -0,0 +1,39 @@ +package cmd + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/pilosa/pilosa/ctl" +) + +var inspecter = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr) + +var inspectCmd = &cobra.Command{ + Use: "inspect", + Short: "inspect - inspect a pilosa data file", + Long: ` +Inspects a data file and provides stats. +`, + Run: func(cmd *cobra.Command, args []string) { + if err := inspecter.Run(context.Background()); err != nil { + fmt.Println(err) + } + }, +} + +func init() { + inspectCmd.Flags().StringVarP(&inspecter.Path, "file", "i", "", "File to inspect") + + err := viper.BindPFlags(inspectCmd.Flags()) + if err != nil { + log.Fatalf("Error binding inspect flags: %v", err) + } + + RootCmd.AddCommand(inspectCmd) +} diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index ee77ba2a8..a04be0b02 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -12,9 +12,7 @@ import ( "path/filepath" "strings" "syscall" - "text/tabwriter" "time" - "unsafe" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/roaring" @@ -90,7 +88,6 @@ Usage: The commands are: - inspect inspects fragment data files check performs a consistency check of data files bench benchmarks operations @@ -114,8 +111,6 @@ func (m *Main) ParseFlags(args []string) error { fmt.Fprintln(m.Stderr, m.Usage()) fmt.Fprintln(m.Stderr, "") return flag.ErrHelp - case "inspect": - m.Cmd = NewInspectCommand(m.Stdin, m.Stdout, m.Stderr) case "check": m.Cmd = NewCheckCommand(m.Stdin, m.Stdout, m.Stderr) case "bench": @@ -143,115 +138,6 @@ type Command interface { Run(context.Context) error } -// InspectCommand represents a command for inspecting fragment data files. -type InspectCommand struct { - // Path to data file - Path string - - // Standard input/output - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewInspectCommand returns a new instance of InspectCommand. -func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectCommand { - return &InspectCommand{ - Stdin: stdin, - Stdout: stdout, - Stderr: stderr, - } -} - -// ParseFlags parses command line flags from args. -func (cmd *InspectCommand) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) - fs.SetOutput(ioutil.Discard) - if err := fs.Parse(args); err != nil { - return err - } - - // Parse path. - if fs.NArg() == 0 { - return errors.New("path required") - } else if fs.NArg() > 1 { - return errors.New("only one path allowed") - } - cmd.Path = fs.Arg(0) - - return nil -} - -// Usage returns the usage message to be printed. -func (cmd *InspectCommand) Usage() string { - return strings.TrimSpace(` -usage: pilosactl inspect PATH - -Inspects a data file and provides stats. - -`) -} - -// Run executes the main program execution. -func (cmd *InspectCommand) Run(ctx context.Context) error { - // Open file handle. - f, err := os.Open(cmd.Path) - if err != nil { - return err - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - return err - } - - // Memory map the file. - data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) - if err != nil { - return err - } - defer syscall.Munmap(data) - - // Attach the mmap file to the bitmap. - t := time.Now() - fmt.Fprintf(cmd.Stderr, "unmarshaling bitmap...") - bm := roaring.NewBitmap() - if err := bm.UnmarshalBinary(data); err != nil { - return err - } - fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t)) - - // Retrieve stats. - t = time.Now() - fmt.Fprintf(cmd.Stderr, "calculating stats...") - info := bm.Info() - fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t)) - - // Print top-level info. - fmt.Fprintf(cmd.Stdout, "== Bitmap Info ==\n") - fmt.Fprintf(cmd.Stdout, "Containers: %d\n", len(info.Containers)) - fmt.Fprintf(cmd.Stdout, "Operations: %d\n", info.OpN) - fmt.Fprintln(cmd.Stdout, "") - - // Print info for each container. - fmt.Fprintln(cmd.Stdout, "== Containers ==") - tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0) - fmt.Fprintf(tw, "%s\t%s\t% 8s \t% 8s\t%s\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET") - for _, ci := range info.Containers { - fmt.Fprintf(tw, "%d\t%s\t% 8d \t% 8d \t0x%08x\n", - ci.Key, - ci.Type, - ci.N, - ci.Alloc, - uintptr(ci.Pointer)-uintptr(unsafe.Pointer(&data[0])), - ) - } - tw.Flush() - - return nil -} - // CheckCommand represents a command for performing consistency checks on data files. type CheckCommand struct { // Data file paths. diff --git a/ctl/inspect.go b/ctl/inspect.go new file mode 100644 index 000000000..1b00dbcee --- /dev/null +++ b/ctl/inspect.go @@ -0,0 +1,127 @@ +package ctl + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "syscall" + "text/tabwriter" + "time" + "unsafe" + + "github.com/pilosa/pilosa/roaring" +) + +// InspectCommand represents a command for inspecting fragment data files. +type InspectCommand struct { + // Path to data file + Path string + + // Standard input/output + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewInspectCommand returns a new instance of InspectCommand. +func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectCommand { + return &InspectCommand{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + } +} + +// ParseFlags parses command line flags from args. +func (cmd *InspectCommand) ParseFlags(args []string) error { + fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) + fs.SetOutput(ioutil.Discard) + if err := fs.Parse(args); err != nil { + return err + } + + // Parse path. + if fs.NArg() == 0 { + return errors.New("path required") + } else if fs.NArg() > 1 { + return errors.New("only one path allowed") + } + cmd.Path = fs.Arg(0) + + return nil +} + +// Usage returns the usage message to be printed. +func (cmd *InspectCommand) Usage() string { + return strings.TrimSpace(` +usage: pilosactl inspect PATH + +Inspects a data file and provides stats. + +`) +} + +// Run executes the main program execution. +func (cmd *InspectCommand) Run(ctx context.Context) error { + // Open file handle. + f, err := os.Open(cmd.Path) + if err != nil { + return err + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return err + } + + // Memory map the file. + data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) + if err != nil { + return err + } + defer syscall.Munmap(data) + + // Attach the mmap file to the bitmap. + t := time.Now() + fmt.Fprintf(cmd.Stderr, "unmarshaling bitmap...") + bm := roaring.NewBitmap() + if err := bm.UnmarshalBinary(data); err != nil { + return err + } + fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t)) + + // Retrieve stats. + t = time.Now() + fmt.Fprintf(cmd.Stderr, "calculating stats...") + info := bm.Info() + fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t)) + + // Print top-level info. + fmt.Fprintf(cmd.Stdout, "== Bitmap Info ==\n") + fmt.Fprintf(cmd.Stdout, "Containers: %d\n", len(info.Containers)) + fmt.Fprintf(cmd.Stdout, "Operations: %d\n", info.OpN) + fmt.Fprintln(cmd.Stdout, "") + + // Print info for each container. + fmt.Fprintln(cmd.Stdout, "== Containers ==") + tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0) + fmt.Fprintf(tw, "%s\t%s\t% 8s \t% 8s\t%s\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET") + for _, ci := range info.Containers { + fmt.Fprintf(tw, "%d\t%s\t% 8d \t% 8d \t0x%08x\n", + ci.Key, + ci.Type, + ci.N, + ci.Alloc, + uintptr(ci.Pointer)-uintptr(unsafe.Pointer(&data[0])), + ) + } + tw.Flush() + + return nil +} From ff532f2064e5ae0c7cefc9dabefcca582ee14436 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 15:01:29 -0600 Subject: [PATCH 44/61] fix help text in restore command --- cmd/restore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restore.go b/cmd/restore.go index 420cd9c5f..44899e4ca 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -31,7 +31,7 @@ func init() { restoreCmd.Flags().StringVarP(&restorer.Host, "host", "", "localhost:15000", "host:port of Pilosa.") restoreCmd.Flags().StringVarP(&restorer.Database, "database", "d", "", "Pilosa database to restore into.") restoreCmd.Flags().StringVarP(&restorer.Frame, "frame", "f", "", "Frame to restore into.") - restoreCmd.Flags().StringVarP(&restorer.Path, "input-file", "i", "", "File to write restore from") + restoreCmd.Flags().StringVarP(&restorer.Path, "input-file", "i", "", "File to restore from.") err := viper.BindPFlags(restoreCmd.Flags()) if err != nil { From 01020d06a84ae4bf18fb308d6fddf1d114bb6e47 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 15:05:59 -0600 Subject: [PATCH 45/61] mvoe check to subcommand --- cmd/check.go | 35 ++++++++++ cmd/pilosactl/main.go | 135 --------------------------------------- ctl/check.go | 145 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 135 deletions(-) create mode 100644 cmd/check.go create mode 100644 ctl/check.go diff --git a/cmd/check.go b/cmd/check.go new file mode 100644 index 000000000..541f33cb2 --- /dev/null +++ b/cmd/check.go @@ -0,0 +1,35 @@ +package cmd + +import ( + "context" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/pilosa/pilosa/ctl" +) + +var checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr) + +var checkCmd = &cobra.Command{ + Use: "check [path2]...", + Short: "check - check a pilosa data file", + Long: ` +Performs a consistency check on data files. +`, + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + fmt.Println("path required") + return + } + checker.Paths = args + if err := checker.Run(context.Background()); err != nil { + fmt.Println(err) + } + }, +} + +func init() { + RootCmd.AddCommand(checkCmd) +} diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index a04be0b02..401d1cff5 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -9,13 +9,10 @@ import ( "io/ioutil" "math/rand" "os" - "path/filepath" "strings" - "syscall" "time" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/roaring" ) var ( @@ -88,7 +85,6 @@ Usage: The commands are: - check performs a consistency check of data files bench benchmarks operations Use the "-h" flag with any command for more information. @@ -111,8 +107,6 @@ func (m *Main) ParseFlags(args []string) error { fmt.Fprintln(m.Stderr, m.Usage()) fmt.Fprintln(m.Stderr, "") return flag.ErrHelp - case "check": - m.Cmd = NewCheckCommand(m.Stdin, m.Stdout, m.Stderr) case "bench": m.Cmd = NewBenchCommand(m.Stdin, m.Stdout, m.Stderr) default: @@ -138,135 +132,6 @@ type Command interface { Run(context.Context) error } -// CheckCommand represents a command for performing consistency checks on data files. -type CheckCommand struct { - // Data file paths. - Paths []string - - // Standard input/output - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewCheckCommand returns a new instance of CheckCommand. -func NewCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *CheckCommand { - return &CheckCommand{ - Stdin: stdin, - Stdout: stdout, - Stderr: stderr, - } -} - -// ParseFlags parses command line flags from args. -func (cmd *CheckCommand) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) - fs.SetOutput(ioutil.Discard) - if err := fs.Parse(args); err != nil { - return err - } - - // Parse path. - if fs.NArg() == 0 { - return errors.New("path required") - } - cmd.Paths = fs.Args() - - return nil -} - -// Usage returns the usage message to be printed. -func (cmd *CheckCommand) Usage() string { - return strings.TrimSpace(` -usage: pilosactl check PATHS... - -Performs a consistency check on data files. - -`) -} - -// Run executes the main program execution. -func (cmd *CheckCommand) Run(ctx context.Context) error { - for _, path := range cmd.Paths { - switch filepath.Ext(path) { - case "": - if err := cmd.checkBitmapFile(path); err != nil { - return err - } - - case ".cache": - if err := cmd.checkCacheFile(path); err != nil { - return err - } - - case ".snapshotting": - if err := cmd.checkSnapshotFile(path); err != nil { - return err - } - } - } - - return nil -} - -// checkBitmapFile performs a consistency check on path for a roaring bitmap file. -func (cmd *CheckCommand) checkBitmapFile(path string) error { - // Open file handle. - f, err := os.Open(path) - if err != nil { - return err - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - return err - } - - // Memory map the file. - data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) - if err != nil { - return err - } - defer syscall.Munmap(data) - - // Attach the mmap file to the bitmap. - bm := roaring.NewBitmap() - if err := bm.UnmarshalBinary(data); err != nil { - return err - } - - // Perform consistency check. - if err := bm.Check(); err != nil { - // Print returned errors. - switch err := err.(type) { - case roaring.ErrorList: - for i := range err { - fmt.Fprintf(cmd.Stdout, "%s: %s\n", path, err[i].Error()) - } - default: - fmt.Fprintf(cmd.Stdout, "%s: %s\n", path, err.Error()) - } - } - - // Print success message if no errors were found. - fmt.Fprintf(cmd.Stdout, "%s: ok\n", path) - - return nil -} - -// checkCacheFile performs a consistency check on path for a cache file. -func (cmd *CheckCommand) checkCacheFile(path string) error { - fmt.Fprintf(cmd.Stderr, "%s: ignoring cache file\n", path) - return nil -} - -// checkSnapshotFile performs a consistency check on path for a snapshot file. -func (cmd *CheckCommand) checkSnapshotFile(path string) error { - fmt.Fprintf(cmd.Stderr, "%s: ignoring snapshot file\n", path) - return nil -} - // BenchCommand represents a command for benchmarking database operations. type BenchCommand struct { // Destination host and port. diff --git a/ctl/check.go b/ctl/check.go new file mode 100644 index 000000000..616e189bf --- /dev/null +++ b/ctl/check.go @@ -0,0 +1,145 @@ +package ctl + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/pilosa/pilosa/roaring" +) + +// CheckCommand represents a command for performing consistency checks on data files. +type CheckCommand struct { + // Data file paths. + Paths []string + + // Standard input/output + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewCheckCommand returns a new instance of CheckCommand. +func NewCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *CheckCommand { + return &CheckCommand{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + } +} + +// ParseFlags parses command line flags from args. +func (cmd *CheckCommand) ParseFlags(args []string) error { + fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) + fs.SetOutput(ioutil.Discard) + if err := fs.Parse(args); err != nil { + return err + } + + // Parse path. + if fs.NArg() == 0 { + return errors.New("path required") + } + cmd.Paths = fs.Args() + + return nil +} + +// Usage returns the usage message to be printed. +func (cmd *CheckCommand) Usage() string { + return strings.TrimSpace(` +usage: pilosactl check PATHS... + +Performs a consistency check on data files. + +`) +} + +// Run executes the main program execution. +func (cmd *CheckCommand) Run(ctx context.Context) error { + for _, path := range cmd.Paths { + switch filepath.Ext(path) { + case "": + if err := cmd.checkBitmapFile(path); err != nil { + return err + } + + case ".cache": + if err := cmd.checkCacheFile(path); err != nil { + return err + } + + case ".snapshotting": + if err := cmd.checkSnapshotFile(path); err != nil { + return err + } + } + } + + return nil +} + +// checkBitmapFile performs a consistency check on path for a roaring bitmap file. +func (cmd *CheckCommand) checkBitmapFile(path string) error { + // Open file handle. + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return err + } + + // Memory map the file. + data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) + if err != nil { + return err + } + defer syscall.Munmap(data) + + // Attach the mmap file to the bitmap. + bm := roaring.NewBitmap() + if err := bm.UnmarshalBinary(data); err != nil { + return err + } + + // Perform consistency check. + if err := bm.Check(); err != nil { + // Print returned errors. + switch err := err.(type) { + case roaring.ErrorList: + for i := range err { + fmt.Fprintf(cmd.Stdout, "%s: %s\n", path, err[i].Error()) + } + default: + fmt.Fprintf(cmd.Stdout, "%s: %s\n", path, err.Error()) + } + } + + // Print success message if no errors were found. + fmt.Fprintf(cmd.Stdout, "%s: ok\n", path) + + return nil +} + +// checkCacheFile performs a consistency check on path for a cache file. +func (cmd *CheckCommand) checkCacheFile(path string) error { + fmt.Fprintf(cmd.Stderr, "%s: ignoring cache file\n", path) + return nil +} + +// checkSnapshotFile performs a consistency check on path for a snapshot file. +func (cmd *CheckCommand) checkSnapshotFile(path string) error { + fmt.Fprintf(cmd.Stderr, "%s: ignoring snapshot file\n", path) + return nil +} From 64f007547e88be2690bb0e4da9a5be0687ee1bd6 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 15:14:02 -0600 Subject: [PATCH 46/61] move bench to subcommand --- cmd/bench.go | 43 +++++++++++++ cmd/pilosactl/main.go | 3 - ctl/bench.go | 143 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 cmd/bench.go create mode 100644 ctl/bench.go diff --git a/cmd/bench.go b/cmd/bench.go new file mode 100644 index 000000000..9b3e85399 --- /dev/null +++ b/cmd/bench.go @@ -0,0 +1,43 @@ +package cmd + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/pilosa/pilosa/ctl" +) + +var bencher = ctl.NewBenchCommand(os.Stdin, os.Stdout, os.Stderr) + +var benchCmd = &cobra.Command{ + Use: "bench", + Short: "bench - benchmark operations", + Long: ` +Executes a benchmark for a given operation against the database. +`, + Run: func(cmd *cobra.Command, args []string) { + if err := bencher.Run(context.Background()); err != nil { + fmt.Println(err) + } + }, +} + +func init() { + benchCmd.Flags().StringVarP(&bencher.Host, "host", "", "localhost:15000", "host:port of Pilosa.") + benchCmd.Flags().StringVarP(&bencher.Database, "database", "d", "", "Pilosa database to benchmark.") + benchCmd.Flags().StringVarP(&bencher.Frame, "frame", "f", "", "Frame to benchmark.") + benchCmd.Flags().StringVarP(&bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]") + benchCmd.Flags().IntVarP(&bencher.N, "num", "n", 0, "Number of operations to perform.") + + err := viper.BindPFlags(benchCmd.Flags()) + if err != nil { + log.Fatalf("Error binding bench flags: %v", err) + } + + RootCmd.AddCommand(benchCmd) +} diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 401d1cff5..417ebaead 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -85,7 +85,6 @@ Usage: The commands are: - bench benchmarks operations Use the "-h" flag with any command for more information. `) @@ -107,8 +106,6 @@ func (m *Main) ParseFlags(args []string) error { fmt.Fprintln(m.Stderr, m.Usage()) fmt.Fprintln(m.Stderr, "") return flag.ErrHelp - case "bench": - m.Cmd = NewBenchCommand(m.Stdin, m.Stdout, m.Stderr) default: return ErrUnknownCommand } diff --git a/ctl/bench.go b/ctl/bench.go new file mode 100644 index 000000000..7e789b5cd --- /dev/null +++ b/ctl/bench.go @@ -0,0 +1,143 @@ +package ctl + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "io/ioutil" + "math/rand" + "strings" + "time" + + "github.com/pilosa/pilosa" +) + +// BenchCommand represents a command for benchmarking database operations. +type BenchCommand struct { + // Destination host and port. + Host string + + // Name of the database & frame to execute against. + Database string + Frame string + + // Type of operation and number to execute. + Op string + N int + + // Standard input/output + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewBenchCommand returns a new instance of BenchCommand. +func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *BenchCommand { + return &BenchCommand{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + } +} + +// ParseFlags parses command line flags from args. +func (cmd *BenchCommand) ParseFlags(args []string) error { + fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) + fs.SetOutput(ioutil.Discard) + fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port") + fs.StringVar(&cmd.Database, "d", "", "database") + fs.StringVar(&cmd.Frame, "f", "", "frame") + fs.StringVar(&cmd.Op, "op", "", "operation") + fs.IntVar(&cmd.N, "n", 0, "op count") + + if err := fs.Parse(args); err != nil { + return err + } + return nil +} + +// Usage returns the usage message to be printed. +func (cmd *BenchCommand) Usage() string { + return strings.TrimSpace(` +usage: pilosactl bench [args] + +Executes a benchmark for a given operation against the database. + +The following flags are allowed: + + -host HOSTPORT + hostname and port of running pilosa server + + -d DATABASE + database to execute operation against + + -f FRAME + frame to execute operation against + + -op OP + name of operation to execute + + -n COUNT + number of iterations to execute + +The following operations are available: + + set-bit + Sets a single random bit on the frame + +`) +} + +// Run executes the main program execution. +func (cmd *BenchCommand) Run(ctx context.Context) error { + // Create a client to the server. + client, err := pilosa.NewClient(cmd.Host) + if err != nil { + return err + } + + switch cmd.Op { + case "set-bit": + return cmd.runSetBit(ctx, client) + case "": + return errors.New("op required") + default: + return fmt.Errorf("unknown bench op: %q", cmd.Op) + } +} + +// runSetBit executes a benchmark of random SetBit() operations. +func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) error { + if cmd.N == 0 { + return errors.New("operation count required") + } else if cmd.Database == "" { + return pilosa.ErrDatabaseRequired + } else if cmd.Frame == "" { + return pilosa.ErrFrameRequired + } + + const maxBitmapID = 1000 + const maxProfileID = 100000 + + startTime := time.Now() + + // Execute operation continuously. + for i := 0; i < cmd.N; i++ { + bitmapID := rand.Intn(maxBitmapID) + profileID := rand.Intn(maxProfileID) + + q := fmt.Sprintf(`SetBit(id=%d, frame="%s", profileID=%d)`, bitmapID, cmd.Frame, profileID) + + if _, err := client.ExecuteQuery(ctx, cmd.Database, q, true); err != nil { + return err + } + } + + // Print results. + elapsed := time.Since(startTime) + fmt.Fprintf(cmd.Stdout, "Executed %d operations in %s (%0.3f op/sec)\n", cmd.N, elapsed, float64(cmd.N)/elapsed.Seconds()) + + return nil +} From a47f93329ea7aa55d22904df6f6875f7e2e08c49 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 15:24:39 -0600 Subject: [PATCH 47/61] remove pilosactl and add version/build to root cmd --- cmd/pilosactl/main.go | 258 ------------------------------------- cmd/pilosactl/main_test.go | 1 - cmd/root.go | 21 ++- 3 files changed, 20 insertions(+), 260 deletions(-) delete mode 100644 cmd/pilosactl/main.go delete mode 100644 cmd/pilosactl/main_test.go diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go deleted file mode 100644 index 417ebaead..000000000 --- a/cmd/pilosactl/main.go +++ /dev/null @@ -1,258 +0,0 @@ -package main - -import ( - "context" - "errors" - "flag" - "fmt" - "io" - "io/ioutil" - "math/rand" - "os" - "strings" - "time" - - "github.com/pilosa/pilosa" -) - -var ( - // ErrUnknownCommand is returned when specifying an unknown command. - ErrUnknownCommand = errors.New("unknown command") - - // ErrPathRequired is returned when executing a command without a required path. - ErrPathRequired = errors.New("path required") - Version string - BuildTime string -) - -func init() { - if Version == "" { - Version = "v0.0.0" - } - if BuildTime == "" { - BuildTime = "not recorded" - } -} - -func main() { - m := NewMain() - - fmt.Fprintf(m.Stderr, "Pilosactl %s, build time %s\n", Version, BuildTime) - - // Parse command line arguments. - if err := m.ParseFlags(os.Args[1:]); err == flag.ErrHelp { - os.Exit(2) - } else if err != nil { - fmt.Fprintln(m.Stderr, err) - os.Exit(2) - } - - // Execute the program. - if err := m.Run(); err != nil { - fmt.Fprintln(m.Stderr, err) - os.Exit(1) - } -} - -// Main represents the main program execution. -type Main struct { - // Subcommand to execute. - Cmd Command - - // Standard input/output - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewMain returns a new instance of Main. -func NewMain() *Main { - return &Main{ - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, - } -} - -// Usage returns the usage message to be printed. -func (m *Main) Usage() string { - return strings.TrimSpace(` -Pilosactl is a tool for interacting with a pilosa server. - -Usage: - - pilosactl command [arguments] - -The commands are: - - -Use the "-h" flag with any command for more information. -`) -} - -// Run executes the main program execution. -func (m *Main) Run() error { return m.Cmd.Run(context.Background()) } - -// ParseFlags parses command line flags from args. -func (m *Main) ParseFlags(args []string) error { - var command string - if len(args) > 0 { - command = args[0] - args = args[1:] - } - - switch command { - case "", "help", "-h": - fmt.Fprintln(m.Stderr, m.Usage()) - fmt.Fprintln(m.Stderr, "") - return flag.ErrHelp - default: - return ErrUnknownCommand - } - - // Parse command's flags. - if err := m.Cmd.ParseFlags(args); err == flag.ErrHelp { - fmt.Fprintln(m.Stderr, m.Cmd.Usage()) - fmt.Fprintln(m.Stderr, "") - return err - } else if err != nil { - return err - } - - return nil -} - -// Command represents an executable subcommand. -type Command interface { - Usage() string - ParseFlags(args []string) error - Run(context.Context) error -} - -// BenchCommand represents a command for benchmarking database operations. -type BenchCommand struct { - // Destination host and port. - Host string - - // Name of the database & frame to execute against. - Database string - Frame string - - // Type of operation and number to execute. - Op string - N int - - // Standard input/output - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewBenchCommand returns a new instance of BenchCommand. -func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *BenchCommand { - return &BenchCommand{ - Stdin: stdin, - Stdout: stdout, - Stderr: stderr, - } -} - -// ParseFlags parses command line flags from args. -func (cmd *BenchCommand) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) - fs.SetOutput(ioutil.Discard) - fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port") - fs.StringVar(&cmd.Database, "d", "", "database") - fs.StringVar(&cmd.Frame, "f", "", "frame") - fs.StringVar(&cmd.Op, "op", "", "operation") - fs.IntVar(&cmd.N, "n", 0, "op count") - - if err := fs.Parse(args); err != nil { - return err - } - return nil -} - -// Usage returns the usage message to be printed. -func (cmd *BenchCommand) Usage() string { - return strings.TrimSpace(` -usage: pilosactl bench [args] - -Executes a benchmark for a given operation against the database. - -The following flags are allowed: - - -host HOSTPORT - hostname and port of running pilosa server - - -d DATABASE - database to execute operation against - - -f FRAME - frame to execute operation against - - -op OP - name of operation to execute - - -n COUNT - number of iterations to execute - -The following operations are available: - - set-bit - Sets a single random bit on the frame - -`) -} - -// Run executes the main program execution. -func (cmd *BenchCommand) Run(ctx context.Context) error { - // Create a client to the server. - client, err := pilosa.NewClient(cmd.Host) - if err != nil { - return err - } - - switch cmd.Op { - case "set-bit": - return cmd.runSetBit(ctx, client) - case "": - return errors.New("op required") - default: - return fmt.Errorf("unknown bench op: %q", cmd.Op) - } -} - -// runSetBit executes a benchmark of random SetBit() operations. -func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) error { - if cmd.N == 0 { - return errors.New("operation count required") - } else if cmd.Database == "" { - return pilosa.ErrDatabaseRequired - } else if cmd.Frame == "" { - return pilosa.ErrFrameRequired - } - - const maxBitmapID = 1000 - const maxProfileID = 100000 - - startTime := time.Now() - - // Execute operation continuously. - for i := 0; i < cmd.N; i++ { - bitmapID := rand.Intn(maxBitmapID) - profileID := rand.Intn(maxProfileID) - - q := fmt.Sprintf(`SetBit(id=%d, frame="%s", profileID=%d)`, bitmapID, cmd.Frame, profileID) - - if _, err := client.ExecuteQuery(ctx, cmd.Database, q, true); err != nil { - return err - } - } - - // Print results. - elapsed := time.Since(startTime) - fmt.Fprintf(cmd.Stdout, "Executed %d operations in %s (%0.3f op/sec)\n", cmd.N, elapsed, float64(cmd.N)/elapsed.Seconds()) - - return nil -} diff --git a/cmd/pilosactl/main_test.go b/cmd/pilosactl/main_test.go deleted file mode 100644 index 0fee6f5dc..000000000 --- a/cmd/pilosactl/main_test.go +++ /dev/null @@ -1 +0,0 @@ -package main_test diff --git a/cmd/root.go b/cmd/root.go index 91d866a18..f7517dc73 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,13 +2,32 @@ package cmd import "github.com/spf13/cobra" +var ( + Version string + BuildTime string +) + var RootCmd = &cobra.Command{ Use: "pilosa", Short: "pilosa - A Distributed In-memory Binary Bitmap Index", + // TODO - is documentation actually there? Long: `Pilosa is a fast index to turbocharge your database. This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available -at http://pilosa.com/docs`, // TODO - is documentation actually there? +at http://pilosa.com/docs + +`, +} + +func init() { + if Version == "" { + Version = "v0.0.0" + } + if BuildTime == "" { + BuildTime = "not recorded" + } + + RootCmd.Long = RootCmd.Long + "Version: " + Version + "\nBuild Time: " + BuildTime + "\n" } From 756c44c17d0e775bc795ab91af195315eed2b083 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 6 Mar 2017 10:19:50 -0600 Subject: [PATCH 48/61] fix bugs with pilosa server -config and remove dead code --- cmd/server.go | 2 +- server/server.go | 65 +----------------------------------------------- 2 files changed, 2 insertions(+), 65 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index cc8bbec49..d31806eea 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -29,7 +29,7 @@ on the configured port.`, fmt.Fprintf(serve.Stderr, "Pilosa %s, build time %s\n", server.Version, server.BuildTime) // Parse command line arguments. - if err := serve.ParseFlags(os.Args[1:]); err != nil { + if err := serve.SetupConfig(args); err != nil { fmt.Fprintln(serve.Stderr, err) os.Exit(2) } diff --git a/server/server.go b/server/server.go index bb151a37d..247d0efa9 100644 --- a/server/server.go +++ b/server/server.go @@ -2,14 +2,11 @@ package server import ( "errors" - "flag" "fmt" "io" "math/rand" "os" - "os/signal" "path/filepath" - "runtime/pprof" "strings" "time" @@ -39,57 +36,6 @@ const ( DefaultDataDir = "~/.pilosa" ) -func mainz() { - serve := NewMain() - serve.Server.Handler.Version = Version - fmt.Fprintf(serve.Stderr, "Pilosa %s, build time %s\n", Version, BuildTime) - - // Parse command line arguments. - if err := serve.ParseFlags(os.Args[1:]); err != nil { - fmt.Fprintln(serve.Stderr, err) - os.Exit(2) - } - - // Start CPU profiling. - if serve.CPUProfile != "" { - f, err := os.Create(serve.CPUProfile) - if err != nil { - fmt.Fprintf(serve.Stderr, "create cpu profile: %v", err) - os.Exit(1) - } - defer f.Close() - - fmt.Fprintln(serve.Stderr, "Starting cpu profile") - pprof.StartCPUProfile(f) - time.AfterFunc(serve.CPUTime, func() { - fmt.Fprintln(serve.Stderr, "Stopping cpu profile") - pprof.StopCPUProfile() - f.Close() - }) - } - - // Execute the program. - if err := serve.Run(); err != nil { - fmt.Fprintln(serve.Stderr, err) - fmt.Fprintln(serve.Stderr, "stopping profile") - os.Exit(1) - } - - // First SIGKILL causes server to shut down gracefully. - c := make(chan os.Signal, 2) - signal.Notify(c, os.Interrupt) - sig := <-c - fmt.Fprintf(serve.Stderr, "Received %s; gracefully shutting down...\n", sig.String()) - - // Second signal causes a hard shutdown. - go func() { <-c; os.Exit(1) }() - - if err := serve.Close(); err != nil { - fmt.Fprintln(serve.Stderr, err) - os.Exit(1) - } -} - // Main represents the main program execution. type Main struct { Server *pilosa.Server @@ -158,16 +104,7 @@ func (m *Main) Close() error { } // ParseFlags parses command line flags from args. -func (m *Main) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosa", flag.ContinueOnError) - fs.StringVar(&m.CPUProfile, "cpuprofile", "", "cpu profile") - fs.DurationVar(&m.CPUTime, "cputime", 30*time.Second, "cpu profile duration") - fs.StringVar(&m.ConfigPath, "config", "", "config path") - fs.SetOutput(m.Stderr) - if err := fs.Parse(args); err != nil { - return err - } - +func (m *Main) SetupConfig(args []string) error { // Load config, if specified. if m.ConfigPath != "" { if _, err := toml.DecodeFile(m.ConfigPath, &m.Config); err != nil { From c6a502efed6f089306f821b4452c6d8ea31efd79 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 6 Mar 2017 11:03:17 -0600 Subject: [PATCH 49/61] fix review --- client.go | 10 ---------- db.go | 6 +++--- frame.go | 6 +++--- pilosa.go | 15 ++++----------- 4 files changed, 10 insertions(+), 27 deletions(-) diff --git a/client.go b/client.go index bfe4dc57e..e7d0eaf85 100644 --- a/client.go +++ b/client.go @@ -194,11 +194,6 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire return nil, ErrQueryRequired } - er := ValidateName(db) - if er != nil { - return nil, ErrName - } - // Encode query request. buf, err := proto.Marshal(&internal.QueryRequest{ DB: db, @@ -255,11 +250,6 @@ func (c *Client) ExecutePQL(ctx context.Context, db, query string) (interface{}, }.Encode(), } - er := ValidateName(db) - if er != nil { - return nil, ErrName - } - req, err := http.NewRequest("POST", u.String(), bytes.NewReader([]byte(query))) if err != nil { return nil, err diff --git a/db.go b/db.go index 9348c97b6..6ad81e16b 100644 --- a/db.go +++ b/db.go @@ -49,9 +49,9 @@ type DB struct { // NewDB returns a new instance of DB. func NewDB(path, name string) (*DB, error) { - err := ValidateName(name) - if err != nil { - return nil, err + validName := Exp.FindStringSubmatchIndex(name) + if len(validName) == 0 { + return nil, ErrName } return &DB{ diff --git a/frame.go b/frame.go index fa47517c5..368cde821 100644 --- a/frame.go +++ b/frame.go @@ -47,9 +47,9 @@ type Frame struct { // NewFrame returns a new instance of frame. func NewFrame(path, db, name string) (*Frame, error) { - err := ValidateName(name) - if err != nil { - return nil, err + validName := Exp.FindStringSubmatchIndex(name) + if len(validName) == 0 { + return nil, ErrName } return &Frame{ diff --git a/pilosa.go b/pilosa.go index 14e5ef203..b3cee7450 100644 --- a/pilosa.go +++ b/pilosa.go @@ -20,13 +20,16 @@ var ( ErrFrameNotFound = errors.New("frame not found") // ErrFrameRequired is returned when no frame is specified. - ErrName = errors.New("name restricted to [a-z0-9_-.]") + ErrName = errors.New("name restricted to [a-z0-9_-]") // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") ErrQueryRequired = errors.New("query required") ) +// Regular expression to valuate db and frame's name +var Exp = regexp.MustCompile(`^([a-z0-9._-]{1,64}$)`) + // Profile represents vertical column in a database. // A profile can have a set of attributes attached to it. type Profile struct { @@ -79,13 +82,3 @@ func decodeProfile(pb *internal.Profile) *Profile { // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" - -// Restrict name using regex -func ValidateName(name string) error { - expr := regexp.MustCompile(`^([a-z0-9._-]{1,64}$)`) - validName := expr.FindStringSubmatchIndex(name) - if len(validName) == 0 { - return ErrName - } - return nil -} From c3ed12c20ebe09c12efc431bfcd9666f1c588305 Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 6 Mar 2017 11:06:22 -0600 Subject: [PATCH 50/61] Refactor fragment.bitmap() so that it leverages `bitmapCache` and so that it's no longer reponsible for updating the count cache. This commit also helps SetBit/ClearBit performance by allowing them to work against data from `bitmapCache` instead of loading bitmaps from fragment.storage every time. --- bitmap.go | 2 -- fragment.go | 72 ++++++++++++++++++++++------------------------ roaring/roaring.go | 2 +- 3 files changed, 35 insertions(+), 41 deletions(-) diff --git a/bitmap.go b/bitmap.go index 651950e45..3a2e4c96a 100644 --- a/bitmap.go +++ b/bitmap.go @@ -16,8 +16,6 @@ type Bitmap struct { // Attributes associated with the bitmap. Attrs map[string]interface{} - - adjustedCount uint64 } // NewBitmap returns a new instance of Bitmap. diff --git a/fragment.go b/fragment.go index 8f8698192..51259585a 100644 --- a/fragment.go +++ b/fragment.go @@ -245,7 +245,7 @@ func (f *Fragment) openCache() error { // This will cause them to be added to the cache. for _, bitmapID := range pb.BitmapIDs { //n := f.storage.CountRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) - n := f.bitmap(bitmapID, false).Count() + n := f.bitmap(bitmapID, true, false).Count() f.cache.BulkAdd(bitmapID, n) } f.cache.Invalidate() @@ -312,33 +312,35 @@ func (f *Fragment) logger() *log.Logger { return log.New(f.LogOutput, "", log.Ls func (f *Fragment) Bitmap(bitmapID uint64) *Bitmap { f.mu.Lock() defer f.mu.Unlock() - return f.bitmap(bitmapID, false) + return f.bitmap(bitmapID, true, true) } -func (f *Fragment) bitmap(bitmapID uint64, updateCache bool) *Bitmap { - r, ok := f.bitmapCache.Fetch(bitmapID) - if ok && r != nil { - return r +func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCache bool) *Bitmap { + + if checkBitmapCache { + r, ok := f.bitmapCache.Fetch(bitmapID) + if ok && r != nil { + return r + } } + // Only use a subset of the containers. // NOTE: The start & end ranges must be divisible by data := f.storage.OffsetRange(f.slice*SliceWidth, bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) // Reference bitmap subrange in storage. + // We Clone() data because otherwise bm will contains pointers to containers in storage. + // This causes unexpected results when we cache the bitmap and try to use it later. bm := &Bitmap{ segments: []BitmapSegment{{ - data: *data, + data: *data.Clone(), slice: f.slice, writable: false, }}, - adjustedCount: 0, } bm.InvalidateCount() - bm.adjustedCount = bm.Count() - if updateCache { - // Update cache. - f.cache.Add(bitmapID, bm.Count()) + if updateBitmapCache { f.bitmapCache.Add(bitmapID, bm) } @@ -353,9 +355,9 @@ func (f *Fragment) SetBit(bitmapID, profileID uint64) (changed bool, err error) return f.setBit(bitmapID, profileID) } -func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) { - // Determine the position of the bit in the storage. +func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, err error) { changed = false + // Determine the position of the bit in the storage. pos, err := f.pos(bitmapID, profileID) if err != nil { return false, err @@ -374,22 +376,18 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) // Invalidate block checksum. delete(f.checksums, int(bitmapID/HashBlockSize)) - // If the number of operations exceeds the limit then snapshot. + // Increment number of operations until snapshot is required. if err := f.incrementOpN(); err != nil { return false, err } - // If adjustedCount is set, then apply that value to bitmap.n instead. - bm := f.bitmap(bitmapID, true) - if bm.adjustedCount > 0 { - bm.SetCount(profileID, bm.adjustedCount) - bm.adjustedCount = 0 - } else { - bm.IncrementCount(profileID) - f.cache.Add(bitmapID, bm.Count()) - } + // Get the bitmap from bitmapCache or fragment.storage. + bm := f.bitmap(bitmapID, true, true) bm.SetBit(profileID) + // Update the cache. + f.cache.Add(bitmapID, bm.Count()) + f.stats.Count("setN", 1) return changed, nil @@ -403,7 +401,8 @@ func (f *Fragment) ClearBit(bitmapID, profileID uint64) (bool, error) { return f.clearBit(bitmapID, profileID) } -func (f *Fragment) clearBit(bitmapID, profileID uint64) (bool, error) { +func (f *Fragment) clearBit(bitmapID, profileID uint64) (changed bool, err error) { + changed = false // Determine the position of the bit in the storage. pos, err := f.pos(bitmapID, profileID) if err != nil { @@ -411,8 +410,7 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (bool, error) { } // Write to storage. - changed, err := f.storage.Remove(pos) - if err != nil { + if changed, err = f.storage.Remove(pos); err != nil { return false, err } @@ -429,17 +427,13 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (bool, error) { return false, err } - // If adjustedCount is set, then apply that value to bitmap.n instead. - bm := f.bitmap(bitmapID, true) - if bm.adjustedCount > 0 { - bm.SetCount(profileID, bm.adjustedCount) - bm.adjustedCount = 0 - } else { - bm.DecrementCount(profileID) - f.cache.Add(bitmapID, bm.Count()) - } + // Get the bitmap from bitmapCache or fragment.storage. + bm := f.bitmap(bitmapID, true, true) bm.ClearBit(profileID) + // Update the cache. + f.cache.Add(bitmapID, bm.Count()) + f.stats.Count("clearN", 1) return changed, nil @@ -908,7 +902,10 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { // Update cache counts for all bitmaps. for bitmapID := range set { - f.cache.BulkAdd(bitmapID, f.bitmap(bitmapID, false).Count()) + // Import should ALWAYS have bitmap() load a new bm from fragment.storage + // because the bitmap that's in bitmapCache hasn't been updated with + // this import's data. + f.cache.BulkAdd(bitmapID, f.bitmap(bitmapID, false, false).Count()) } f.cache.Invalidate() @@ -1256,7 +1253,6 @@ func (s *FragmentSyncer) SyncFragment() error { // Determine replica set. nodes := s.Cluster.FragmentNodes(s.Fragment.DB(), s.Fragment.Slice()) if len(nodes) == 1 { - //fmt.Println("no place to replicate", s.Fragment.DB(), s.Fragment.Frame(), s.Fragment.Slice()) return nil } diff --git a/roaring/roaring.go b/roaring/roaring.go index 58b4d1e19..e74739311 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1084,7 +1084,7 @@ func (c *container) clone() *container { copy(other.bitmap, c.bitmap) } - return c + return other } // WriteTo writes c to w. From 28580fd3540f3f793c71124ec26c09d479c0940e Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 6 Mar 2017 11:10:23 -0600 Subject: [PATCH 51/61] remove validate name from client --- pilosactl/import.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pilosactl/import.go b/pilosactl/import.go index 80e9a2973..313c5da99 100644 --- a/pilosactl/import.go +++ b/pilosactl/import.go @@ -104,16 +104,6 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } else if len(cmd.Paths) == 0 { return errors.New("path required") } - // Restrict frame name and database name with regex - dbError := pilosa.ValidateName(cmd.Database) - if dbError != nil { - return dbError - } - - frameError := pilosa.ValidateName(cmd.Frame) - if frameError != nil { - return frameError - } // Create a client to the server. client, err := pilosa.NewClient(cmd.Host) if err != nil { From c419c082da0a13ac0ed285853fe316f58f42baa0 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 6 Mar 2017 11:18:16 -0600 Subject: [PATCH 52/61] update readme to reflect subcommands and pilosactl gone --- README.md | 92 +++++-------------------------------------------------- 1 file changed, 8 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index bd6aacbe2..148a7a139 100644 --- a/README.md +++ b/README.md @@ -23,18 +23,16 @@ $ go install github.com/pilosa/pilosa/cmd/... Now run a single pilosa node with the default configuration: ```sh -pilosa +pilosa server ``` -If you would like to quickly create a multi-node pilosa cluster, see the `pilosactl create` documentation. - ## Configuration You can specify a configuration by setting the `-config` flag when running `pilosa`. ```sh -pilosa -config custom-config-file.cfg +pilosa server --config custom-config-file.cfg ``` The config file uses the [TOML](https://github.com/toml-lang/toml) configuration file format, @@ -54,6 +52,12 @@ host = "127.0.0.1:15000" host = "127.0.0.1:15001" ``` +You can generate a template config file with default values with: + +```sh +pilosa config +``` + The first two configuration options will be unique to each node in the cluster: `data-dir`: directory in which data is stored to disk @@ -242,83 +246,3 @@ $ go install --ldflags="-X main.Version=1.0.0" ``` [Glide]: http://glide.sh/ - -## Pilosactl - -Pilosactl contains a suite of tools for interacting with pilosa. Run `pilosactl` for an overview of commands, and `pilosactl -h` for specific information on that command. - -### Create - -`pilosactl create` is used to create pilosa clusters. It has a number of options for controlling how the cluster is configured, what hosts it is on, and even the ability to build the pilosa binary locally and copy it to each cluster node automatically. To start pilosa on remote hosts, you only need `ssh` access to those hosts. See `pilosactl create -h` for a full list of options. - -Examples: - -Create a 5 node cluster locally (using 5 different ports), with a replication factor of 2. -``` -pilosactl create \ - -serverN 5 \ - -replicaN 2 -``` - -Create a cluster on 3 remote hosts - all logs will come to local stderr, pilosa binary must be available on remote hosts. The ssh user on the remote hosts needs to be the same as your local user. Otherwise use the `ssh-user` option. -``` -pilosactl create \ - -hosts="node1.example.com:15000,node2.example.com:15000,node3.example.com:15000" -``` - -Create a cluster on 3 remote hosts running OSX, but build the binary locally and copy it up. Stream the stderr of each node to a separate local log file. -``` -pilosactl create \ - -hosts="mac1.example.com:15000,mac2.example.com:15000,mac3.example.com:15000" \ - -copy-binary \ - -goos=darwin \ - -goarch=amd64 \ - -log-file-prefix=clusterlogs -``` - -### Bagent - -`pilosactl bagent` is what you want if you just want to run a simple benchmark against an existing cluster. Running it with no arguments will print some help, including the set of subcommands that it may be passed. Calling a subcommand with `-h'` will print the options for that subcommand. The `agent-num` flag can be passed an integer which can change the behavior the benchmarks that are run. This is useful when multiple invocations of the same benchmark are made by the `bspawn` command - they can each (for example) set different bits even though they all have the same arguments. - -E.G. -``` -pilosactl bagent \ - -hosts="localhost:15000,localhost:15001" \ - import -h -``` - -Multiple subcommands and their arguments may be concatenated at the command line and they will be run serially. This is useful (i.e.) for importing a bunch of data, and then executing queries against it. - -This will generate and import a bunch of data, and then execute random queries against it. - -``` -pilosactl bagent \ - -hosts="localhost:15000,localhost:15001" \ - import -max-bits-per-map=10000 \ - random-query -iterations 100 -``` - -### Bspawn -`pilosactl bspawn` allows you to automate the creation of clusters and the running of complex benchmarks which span multiple benchmark agents against them. It has a number of options which are described by `pilosactl bspawn` with no arguments, and also takes a config file which describes the Benchmark itself - this file is described below. - -#### Configuration Format - -The configuration file is a json object with the top level key `benchmarks`. This contains a list of objects each of which represents a `bagent` command (the `args` key) that will be run some number of times concurrently (the `num` key), and a `name` which should describe the overall effect that command. An example is below. -```json -{ - "benchmarks": [ - { - "num": 3, - "name": "set-diags", - "args": ["diagonal-set-bits", "-iterations", "30000", "-client-type", "round_robin"] - }, - { - "num": 2, - "name": "rand-plus-zipf", - "args": ["random-set-bits", "-iterations", "20000", "zipf", "-iterations", "100"] - } - ] -} -``` - -All of the benchmarks, and agents are run concurrently. Each agent will be passed an `agent-num` which can modify the behavior in a way that is benchmark specific. See the documentation for each benchmark to see how `agent-num` changes its behavior. From 4e31d02979d9ece4099adcfaf37f0d510473e7b0 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 6 Mar 2017 12:31:51 -0600 Subject: [PATCH 53/61] wrap regexp in function --- db.go | 6 +++--- frame.go | 6 +++--- pilosa.go | 13 ++++++++++++- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/db.go b/db.go index 6ad81e16b..9348c97b6 100644 --- a/db.go +++ b/db.go @@ -49,9 +49,9 @@ type DB struct { // NewDB returns a new instance of DB. func NewDB(path, name string) (*DB, error) { - validName := Exp.FindStringSubmatchIndex(name) - if len(validName) == 0 { - return nil, ErrName + err := ValidateName(name) + if err != nil { + return nil, err } return &DB{ diff --git a/frame.go b/frame.go index 368cde821..fa47517c5 100644 --- a/frame.go +++ b/frame.go @@ -47,9 +47,9 @@ type Frame struct { // NewFrame returns a new instance of frame. func NewFrame(path, db, name string) (*Frame, error) { - validName := Exp.FindStringSubmatchIndex(name) - if len(validName) == 0 { - return nil, ErrName + err := ValidateName(name) + if err != nil { + return nil, err } return &Frame{ diff --git a/pilosa.go b/pilosa.go index b3cee7450..b3620a900 100644 --- a/pilosa.go +++ b/pilosa.go @@ -28,7 +28,8 @@ var ( ) // Regular expression to valuate db and frame's name -var Exp = regexp.MustCompile(`^([a-z0-9._-]{1,64}$)`) +// Todo: remove . when frame doesn't require . for topN +var nameRegexp = regexp.MustCompile(`^([a-z0-9._-]{1,64}$)`) // Profile represents vertical column in a database. // A profile can have a set of attributes attached to it. @@ -82,3 +83,13 @@ func decodeProfile(pb *internal.Profile) *Profile { // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" + + +// Restrict name using regex +func ValidateName(name string) error { + validName := nameRegexp.Match([]byte(name)) + if validName == false{ + return ErrName + } + return nil +} \ No newline at end of file From 6216c4fb8a03a450277dfce9976bb2f4b475a91b Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 6 Mar 2017 15:34:57 -0600 Subject: [PATCH 54/61] don't panic if 'n' not supplied to TopN --- executor.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 26b2f4747..728e22cee 100644 --- a/executor.go +++ b/executor.go @@ -168,7 +168,10 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql // requeries to retrieve the full counts for each of the top results. func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { bitmapIDs, _ := c.Args["ids"].([]uint64) - n := c.Args["n"].(uint64) + var n uint64 + if nval, ok := c.Args["n"]; ok { + n = nval.(uint64) + } // Execute original query. pairs, err := e.executeTopNSlices(ctx, db, c, slices, opt) @@ -197,7 +200,7 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic return nil, err } - if int(n) < len(trimmedList) { + if n != 0 && int(n) < len(trimmedList) { trimmedList = trimmedList[0:n] } return trimmedList, nil From f463ab9d9b19177e5c8e9bc4df32d0b68d1fbdab Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 6 Mar 2017 16:37:41 -0600 Subject: [PATCH 55/61] make inspect behave like pilosactl version --- cmd/inspect.go | 19 +++++++++---------- cmd/sort.go | 1 - 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/cmd/inspect.go b/cmd/inspect.go index 48bbf8308..56ded3bb7 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -3,11 +3,9 @@ package cmd import ( "context" "fmt" - "log" "os" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/pilosa/pilosa/ctl" ) @@ -16,11 +14,19 @@ var inspecter = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr) var inspectCmd = &cobra.Command{ Use: "inspect", - Short: "inspect - inspect a pilosa data file", + Short: "get stats on pilosa data file", Long: ` Inspects a data file and provides stats. `, Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + fmt.Println("path required") + return + } else if len(args) > 1 { + fmt.Println("only one path allowed") + return + } + inspecter.Path = args[0] if err := inspecter.Run(context.Background()); err != nil { fmt.Println(err) } @@ -28,12 +34,5 @@ Inspects a data file and provides stats. } func init() { - inspectCmd.Flags().StringVarP(&inspecter.Path, "file", "i", "", "File to inspect") - - err := viper.BindPFlags(inspectCmd.Flags()) - if err != nil { - log.Fatalf("Error binding inspect flags: %v", err) - } - RootCmd.AddCommand(inspectCmd) } diff --git a/cmd/sort.go b/cmd/sort.go index 813624ddc..4df585629 100644 --- a/cmd/sort.go +++ b/cmd/sort.go @@ -25,7 +25,6 @@ The format of the CSV file is: The file should contain no headers. `, Run: func(cmd *cobra.Command, args []string) { - fmt.Println(cmd.Flags()) if len(args) == 0 { fmt.Println("path required") return From c7caea6b30dacd853d7c6482ef295e983ab1a408 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 6 Mar 2017 16:45:37 -0600 Subject: [PATCH 56/61] fix short help strings on commands --- cmd/backup.go | 2 +- cmd/bench.go | 2 +- cmd/check.go | 2 +- cmd/config.go | 2 +- cmd/export.go | 2 +- cmd/import.go | 2 +- cmd/inspect.go | 2 +- cmd/restore.go | 2 +- cmd/root.go | 2 +- cmd/server.go | 2 +- cmd/sort.go | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/backup.go b/cmd/backup.go index 883ce7be8..418f6a720 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -16,7 +16,7 @@ var backuper = ctl.NewBackupCommand(os.Stdin, os.Stdout, os.Stderr) var backupCmd = &cobra.Command{ Use: "backup", - Short: "backup - backup data from pilosa", + Short: "Backup data from pilosa.", Long: ` Backs up the database and frame from across the cluster into a single file. `, diff --git a/cmd/bench.go b/cmd/bench.go index 9b3e85399..3e093d68e 100644 --- a/cmd/bench.go +++ b/cmd/bench.go @@ -16,7 +16,7 @@ var bencher = ctl.NewBenchCommand(os.Stdin, os.Stdout, os.Stderr) var benchCmd = &cobra.Command{ Use: "bench", - Short: "bench - benchmark operations", + Short: "Benchmark operations.", Long: ` Executes a benchmark for a given operation against the database. `, diff --git a/cmd/check.go b/cmd/check.go index 541f33cb2..560130187 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -14,7 +14,7 @@ var checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr) var checkCmd = &cobra.Command{ Use: "check [path2]...", - Short: "check - check a pilosa data file", + Short: "Do a consistency check on a pilosa data file.", Long: ` Performs a consistency check on data files. `, diff --git a/cmd/config.go b/cmd/config.go index 72202ffe3..5fccbfc7d 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -14,7 +14,7 @@ var conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr) var confCmd = &cobra.Command{ Use: "config", - Short: "config - prints the default configuration", + Short: "Print the default configuration.", Long: `config prints the default configuration to stdout `, Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/export.go b/cmd/export.go index a5cd0f62d..06a76118d 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -16,7 +16,7 @@ var exporter = ctl.NewExportCommand(os.Stdin, os.Stdout, os.Stderr) var exportCmd = &cobra.Command{ Use: "export", - Short: "export - export data from pilosa", + Short: "Export data from pilosa.", Long: ` Bulk exports a fragment to a CSV file. If the OUTFILE is not specified then the output is written to STDOUT. diff --git a/cmd/import.go b/cmd/import.go index af1a7fcd5..5f878c431 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -16,7 +16,7 @@ var importer = ctl.NewImportCommand(os.Stdin, os.Stdout, os.Stderr) var importCmd = &cobra.Command{ Use: "import", - Short: "import - import data to pilosa", + Short: "Bulk load data into pilosa.", Long: `Bulk imports one or more CSV files to a host's database and frame. The bits of the CSV file are grouped by slice for the most efficient import. diff --git a/cmd/inspect.go b/cmd/inspect.go index 56ded3bb7..666c59d5c 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -14,7 +14,7 @@ var inspecter = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr) var inspectCmd = &cobra.Command{ Use: "inspect", - Short: "get stats on pilosa data file", + Short: "Get stats on a pilosa data file.", Long: ` Inspects a data file and provides stats. `, diff --git a/cmd/restore.go b/cmd/restore.go index 44899e4ca..399122ff3 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -16,7 +16,7 @@ var restorer = ctl.NewRestoreCommand(os.Stdin, os.Stdout, os.Stderr) var restoreCmd = &cobra.Command{ Use: "restore", - Short: "restore - restore data to pilosa", + Short: "Restore data to pilosa from a backup file.", Long: ` Restores a frame to the cluster from a backup file. `, diff --git a/cmd/root.go b/cmd/root.go index f7517dc73..139214302 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -9,7 +9,7 @@ var ( var RootCmd = &cobra.Command{ Use: "pilosa", - Short: "pilosa - A Distributed In-memory Binary Bitmap Index", + Short: "Pilosa - A Distributed In-memory Binary Bitmap Index.", // TODO - is documentation actually there? Long: `Pilosa is a fast index to turbocharge your database. diff --git a/cmd/server.go b/cmd/server.go index d31806eea..6603a0989 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -18,7 +18,7 @@ var serve = server.NewMain() var serveCmd = &cobra.Command{ Use: "server", - Short: "server - run the pilosa server", + Short: "Run Pilosa.", Long: `pilosa server runs Pilosa. It will load existing data from the configured diff --git a/cmd/sort.go b/cmd/sort.go index 4df585629..f9a1d682e 100644 --- a/cmd/sort.go +++ b/cmd/sort.go @@ -14,7 +14,7 @@ var sorter = ctl.NewSortCommand(os.Stdin, os.Stdout, os.Stderr) var sortCmd = &cobra.Command{ Use: "sort ", - Short: "sort - sort import data for optimal import performance", + Short: "Sort import data for optimal import performance.", Long: ` Sorts the import data at PATH into the optimal sort order for importing. From 03e1f4f09bb54f0262b333cddf5365a7b96680f1 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 7 Mar 2017 08:57:03 -0600 Subject: [PATCH 57/61] turn on bitmapCache support in openCache() --- fragment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fragment.go b/fragment.go index fd73ff343..18306bc5d 100644 --- a/fragment.go +++ b/fragment.go @@ -245,7 +245,7 @@ func (f *Fragment) openCache() error { // This will cause them to be added to the cache. for _, bitmapID := range pb.BitmapIDs { //n := f.storage.CountRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) - n := f.bitmap(bitmapID, true, false).Count() + n := f.bitmap(bitmapID, true, true).Count() f.cache.BulkAdd(bitmapID, n) } f.cache.Invalidate() From 555a514e37526ec6624a4b1e632f927f0c96ecb7 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 7 Mar 2017 11:30:31 -0600 Subject: [PATCH 58/61] code review tweaks --- cmd/pilosa/main.go | 2 +- cmd/server.go | 2 +- ctl/backup.go | 2 +- ctl/config.go | 2 +- ctl/export.go | 2 +- ctl/sort.go | 2 +- server/server.go | 21 +++++++++------------ server/server_test.go | 20 ++++++++++---------- 8 files changed, 25 insertions(+), 28 deletions(-) diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go index 255aef606..c3bef0622 100644 --- a/cmd/pilosa/main.go +++ b/cmd/pilosa/main.go @@ -10,6 +10,6 @@ import ( func main() { if err := cmd.RootCmd.Execute(); err != nil { fmt.Println(err) - os.Exit(-1) + os.Exit(1) } } diff --git a/cmd/server.go b/cmd/server.go index 6603a0989..64e45eed4 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -14,7 +14,7 @@ import ( "github.com/pilosa/pilosa/server" ) -var serve = server.NewMain() +var serve = server.NewCommand() var serveCmd = &cobra.Command{ Use: "server", diff --git a/ctl/backup.go b/ctl/backup.go index 23b34cd1f..f0adf5766 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -36,7 +36,7 @@ func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand } } -// Run executes the main program execution. +// Run executes the backup. func (cmd *BackupCommand) Run(ctx context.Context) error { // Validate arguments. if cmd.Path == "" { diff --git a/ctl/config.go b/ctl/config.go index 516069e2d..998c17b47 100644 --- a/ctl/config.go +++ b/ctl/config.go @@ -24,7 +24,7 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *ConfigCommand } } -// Run executes the main program execution. +// Run prints out the default config. func (cmd *ConfigCommand) Run(ctx context.Context) error { fmt.Fprintln(cmd.Stdout, strings.TrimSpace(` data-dir = "~/.pilosa" diff --git a/ctl/export.go b/ctl/export.go index f19c7123f..4eead3239 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -36,7 +36,7 @@ func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *ExportCommand } } -// Run executes the main program execution. +// Run executes the export. func (cmd *ExportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) diff --git a/ctl/sort.go b/ctl/sort.go index eac68b7ab..66bd16372 100644 --- a/ctl/sort.go +++ b/ctl/sort.go @@ -35,7 +35,7 @@ func NewSortCommand(stdin io.Reader, stdout, stderr io.Writer) *SortCommand { } } -// Run executes the main program execution. +// Run executes the sort command. func (cmd *SortCommand) Run(ctx context.Context) error { // Open file for reading. f, err := os.Open(cmd.Path) diff --git a/server/server.go b/server/server.go index 247d0efa9..b7b8ca2f9 100644 --- a/server/server.go +++ b/server/server.go @@ -36,8 +36,8 @@ const ( DefaultDataDir = "~/.pilosa" ) -// Main represents the main program execution. -type Main struct { +// Command represents the state of the pilosa server command. +type Command struct { Server *pilosa.Server // Configuration options. @@ -55,8 +55,8 @@ type Main struct { } // NewMain returns a new instance of Main. -func NewMain() *Main { - return &Main{ +func NewCommand() *Command { + return &Command{ Server: pilosa.NewServer(), Config: pilosa.NewConfig(), @@ -66,8 +66,8 @@ func NewMain() *Main { } } -// Run executes the main program execution. -func (m *Main) Run(args ...string) error { +// Run executes the pilosa server. +func (m *Command) Run(args ...string) error { // Notify user of config file. if m.ConfigPath != "" { fmt.Fprintf(m.Stdout, "Using config: %s\n", m.ConfigPath) @@ -99,12 +99,12 @@ func (m *Main) Run(args ...string) error { } // Close shuts down the server. -func (m *Main) Close() error { +func (m *Command) Close() error { return m.Server.Close() } // ParseFlags parses command line flags from args. -func (m *Main) SetupConfig(args []string) error { +func (m *Command) SetupConfig(args []string) error { // Load config, if specified. if m.ConfigPath != "" { if _, err := toml.DecodeFile(m.ConfigPath, &m.Config); err != nil { @@ -120,11 +120,8 @@ func (m *Main) SetupConfig(args []string) error { // Expand home directory. prefix := "~" + string(filepath.Separator) if strings.HasPrefix(m.Config.DataDir, prefix) { - // u, err := user.Current() HomeDir := os.Getenv("HOME") - /*if err != nil { - return err - } else*/if HomeDir == "" { + if HomeDir == "" { return errors.New("data directory not specified and no home dir available") } m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix)) diff --git a/server/server_test.go b/server/server_test.go index bb772ab45..9e907522a 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -304,7 +304,7 @@ path = "/path/to/plugins" // Main represents a test wrapper for main.Main. type Main struct { - *server.Main + *server.Command Stdin bytes.Buffer Stdout bytes.Buffer @@ -318,16 +318,16 @@ func NewMain() *Main { panic(err) } - m := &Main{Main: server.NewMain()} + m := &Main{Command: server.NewCommand()} m.Config.DataDir = path m.Config.Host = "localhost:0" - m.Main.Stdin = &m.Stdin - m.Main.Stdout = &m.Stdout - m.Main.Stderr = &m.Stderr + m.Command.Stdin = &m.Stdin + m.Command.Stdout = &m.Stdout + m.Command.Stderr = &m.Stderr if testing.Verbose() { - m.Main.Stdout = io.MultiWriter(os.Stdout, m.Main.Stdout) - m.Main.Stderr = io.MultiWriter(os.Stderr, m.Main.Stderr) + m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout) + m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr) } return m @@ -345,18 +345,18 @@ func MustRunMain() *Main { // Close closes the program and removes the underlying data directory. func (m *Main) Close() error { defer os.RemoveAll(m.Config.DataDir) - return m.Main.Close() + return m.Command.Close() } // Reopen closes the program and reopens it. func (m *Main) Reopen() error { - if err := m.Main.Close(); err != nil { + if err := m.Command.Close(); err != nil { return err } // Create new main with the same config. config := m.Config - m.Main = server.NewMain() + m.Command = server.NewCommand() m.Config = config // Run new program. From 320110f0116c94a6194cb8a8fa276fbfde6afe9e Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 7 Mar 2017 11:35:58 -0600 Subject: [PATCH 59/61] tweak comments and remove dead code --- ctl/bench.go | 53 +----------------------------------------------- ctl/check.go | 33 +----------------------------- ctl/inspect.go | 35 +------------------------------- ctl/restore.go | 2 +- server/server.go | 2 +- 5 files changed, 5 insertions(+), 120 deletions(-) diff --git a/ctl/bench.go b/ctl/bench.go index 7e789b5cd..2afe7bb27 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -3,12 +3,9 @@ package ctl import ( "context" "errors" - "flag" "fmt" "io" - "io/ioutil" "math/rand" - "strings" "time" "github.com/pilosa/pilosa" @@ -42,55 +39,7 @@ func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *BenchCommand { } } -// ParseFlags parses command line flags from args. -func (cmd *BenchCommand) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) - fs.SetOutput(ioutil.Discard) - fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port") - fs.StringVar(&cmd.Database, "d", "", "database") - fs.StringVar(&cmd.Frame, "f", "", "frame") - fs.StringVar(&cmd.Op, "op", "", "operation") - fs.IntVar(&cmd.N, "n", 0, "op count") - - if err := fs.Parse(args); err != nil { - return err - } - return nil -} - -// Usage returns the usage message to be printed. -func (cmd *BenchCommand) Usage() string { - return strings.TrimSpace(` -usage: pilosactl bench [args] - -Executes a benchmark for a given operation against the database. - -The following flags are allowed: - - -host HOSTPORT - hostname and port of running pilosa server - - -d DATABASE - database to execute operation against - - -f FRAME - frame to execute operation against - - -op OP - name of operation to execute - - -n COUNT - number of iterations to execute - -The following operations are available: - - set-bit - Sets a single random bit on the frame - -`) -} - -// Run executes the main program execution. +// Run executes the bench command. func (cmd *BenchCommand) Run(ctx context.Context) error { // Create a client to the server. client, err := pilosa.NewClient(cmd.Host) diff --git a/ctl/check.go b/ctl/check.go index 616e189bf..0893790f2 100644 --- a/ctl/check.go +++ b/ctl/check.go @@ -2,14 +2,10 @@ package ctl import ( "context" - "errors" - "flag" "fmt" "io" - "io/ioutil" "os" "path/filepath" - "strings" "syscall" "github.com/pilosa/pilosa/roaring" @@ -35,34 +31,7 @@ func NewCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *CheckCommand { } } -// ParseFlags parses command line flags from args. -func (cmd *CheckCommand) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) - fs.SetOutput(ioutil.Discard) - if err := fs.Parse(args); err != nil { - return err - } - - // Parse path. - if fs.NArg() == 0 { - return errors.New("path required") - } - cmd.Paths = fs.Args() - - return nil -} - -// Usage returns the usage message to be printed. -func (cmd *CheckCommand) Usage() string { - return strings.TrimSpace(` -usage: pilosactl check PATHS... - -Performs a consistency check on data files. - -`) -} - -// Run executes the main program execution. +// Run executes the check command. func (cmd *CheckCommand) Run(ctx context.Context) error { for _, path := range cmd.Paths { switch filepath.Ext(path) { diff --git a/ctl/inspect.go b/ctl/inspect.go index 1b00dbcee..86434b131 100644 --- a/ctl/inspect.go +++ b/ctl/inspect.go @@ -2,13 +2,9 @@ package ctl import ( "context" - "errors" - "flag" "fmt" "io" - "io/ioutil" "os" - "strings" "syscall" "text/tabwriter" "time" @@ -37,36 +33,7 @@ func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectComman } } -// ParseFlags parses command line flags from args. -func (cmd *InspectCommand) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError) - fs.SetOutput(ioutil.Discard) - if err := fs.Parse(args); err != nil { - return err - } - - // Parse path. - if fs.NArg() == 0 { - return errors.New("path required") - } else if fs.NArg() > 1 { - return errors.New("only one path allowed") - } - cmd.Path = fs.Arg(0) - - return nil -} - -// Usage returns the usage message to be printed. -func (cmd *InspectCommand) Usage() string { - return strings.TrimSpace(` -usage: pilosactl inspect PATH - -Inspects a data file and provides stats. - -`) -} - -// Run executes the main program execution. +// Run executes the inspect command. func (cmd *InspectCommand) Run(ctx context.Context) error { // Open file handle. f, err := os.Open(cmd.Path) diff --git a/ctl/restore.go b/ctl/restore.go index b9a573704..a650df495 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -36,7 +36,7 @@ func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreComman } } -// Run executes the main program execution. +// Run executes the restore command. func (cmd *RestoreCommand) Run(ctx context.Context) error { // Validate arguments. if cmd.Path == "" { diff --git a/server/server.go b/server/server.go index b7b8ca2f9..845340485 100644 --- a/server/server.go +++ b/server/server.go @@ -103,7 +103,7 @@ func (m *Command) Close() error { return m.Server.Close() } -// ParseFlags parses command line flags from args. +// SetupConfig loads the config file if specified and sets state on the Command. func (m *Command) SetupConfig(args []string) error { // Load config, if specified. if m.ConfigPath != "" { From fa9586f64f4c122cf0e2fc503a8f7dc5440d610e Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 7 Mar 2017 13:18:51 -0600 Subject: [PATCH 60/61] remove useless print --- cmd/server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/server.go b/cmd/server.go index 64e45eed4..efff0ac7d 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -55,7 +55,6 @@ on the configured port.`, // Execute the program. if err := serve.Run(); err != nil { fmt.Fprintln(serve.Stderr, err) - fmt.Fprintln(serve.Stderr, "stopping profile") os.Exit(1) } From b6430bcc0917a92cbf74cbdae732b01b06859c74 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 7 Mar 2017 15:25:36 -0600 Subject: [PATCH 61/61] remove code that is no longer used: - `cacheoveride` - `Bitmap.SetCount()` --- bitmap.go | 6 ------ fragment.go | 2 -- 2 files changed, 8 deletions(-) diff --git a/bitmap.go b/bitmap.go index d30a9192a..2e27ded7a 100644 --- a/bitmap.go +++ b/bitmap.go @@ -16,8 +16,6 @@ type Bitmap struct { // Attributes associated with the bitmap. Attrs map[string]interface{} - - cacheoveride uint64 } // NewBitmap returns a new instance of Bitmap. @@ -191,10 +189,6 @@ func (b *Bitmap) DecrementCount(i uint64) { } } } -func (b *Bitmap) SetCount(i uint64, count uint64) { - seg := b.segment(i / SliceWidth) - seg.n = count -} // Count returns the number of set bits in the bitmap. func (b *Bitmap) Count() uint64 { diff --git a/fragment.go b/fragment.go index 18306bc5d..e70dab80c 100644 --- a/fragment.go +++ b/fragment.go @@ -337,10 +337,8 @@ func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCa slice: f.slice, writable: false, }}, - cacheoveride: 0, } bm.InvalidateCount() - bm.cacheoveride = bm.Count() if updateBitmapCache { f.bitmapCache.Add(bitmapID, bm)