From 2a44bdd25ccb52b8e63093da8c6492beb82194d0 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 6 Oct 2020 02:45:30 -0500 Subject: [PATCH 1/8] Collect size-on-disk usage data from all nodes --- api.go | 62 ++++++++++++++++++++++++++++++++++-------- client.go | 6 ++++ http/client.go | 31 +++++++++++++++++++++ http/handler.go | 29 +++++++------------- server/handler_test.go | 15 ++++++---- 5 files changed, 108 insertions(+), 35 deletions(-) diff --git a/api.go b/api.go index dfbd33e01..9d6916558 100644 --- a/api.go +++ b/api.go @@ -793,26 +793,43 @@ func (api *API) Node() *Node { return &node } -// Usage gets the disk usage per index -func (api *API) Usage() (map[string]int64, int64, error) { - indexSizes := make(map[string]int64) +// NodeUsage represents all usage measurements for one node. +type NodeUsage struct { + Disk DiskUsage `json:"bytesOnDisk"` +} + +// DiskUsage represents the storage space used on disk by one node. +type DiskUsage struct { + Total int64 `json:"total"` + Indexes map[string]int64 `json:"indexes"` +} + +// Usage gets the disk usage per index, in a map[nodeID]NodeUsage +func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.Usage") + defer span.Finish() + + nodeUsages := make(map[string]NodeUsage) var totalSize int64 + // Open storage directory. dirName, err := expandDirName(api.server.dataDir) if err != nil { - return indexSizes, totalSize, errors.Wrap(err, "expanding data directory") + return nodeUsages, errors.Wrap(err, "expanding data directory") } dir, err := os.Open(dirName) if err != nil { - return indexSizes, totalSize, errors.Wrap(err, "opening data directory") + return nodeUsages, errors.Wrap(err, "opening data directory") } defer dir.Close() files, err := dir.Readdir(-1) if err != nil { - return indexSizes, totalSize, errors.Wrap(err, "reading data directory") + return nodeUsages, errors.Wrap(err, "reading data directory") } + // Read size on disk for each index directory. + indexSizes := make(map[string]int64) for _, file := range files { if !file.IsDir() { continue @@ -821,17 +838,40 @@ func (api *API) Usage() (map[string]int64, int64, error) { continue } fullName := path.Join(dirName, file.Name()) - indexSizes[file.Name()], err = diskUsage(fullName) + indexSizes[file.Name()], err = directoryUsage(fullName) if err != nil { - break + return nodeUsages, errors.Wrap(err, "getting disk usage") } totalSize += indexSizes[file.Name()] } - return indexSizes, totalSize, nil + // Insert into result. + nodeUsage := NodeUsage{ + Disk: DiskUsage{ + Total: totalSize, + Indexes: indexSizes, + }, + } + nodeUsages[api.server.nodeID] = nodeUsage + + // Collect size on disk from remote nodes + if !remote { + nodes := api.cluster.Nodes() + for _, node := range nodes { + if node.ID == api.server.nodeID { + continue + } + nodeUsage, err := api.server.defaultClient.GetNodeUsage(ctx, &node.URI) + if err != nil { + return nil, errors.Wrapf(err, "collecting disk usage from %s", node.URI) + } + nodeUsages[node.ID] = nodeUsage[node.ID] + } + } + return nodeUsages, nil } -func diskUsage(fname string) (int64, error) { +func directoryUsage(fname string) (int64, error) { var size int64 dir, err := os.Open(fname) @@ -847,7 +887,7 @@ func diskUsage(fname string) (int64, error) { for _, file := range files { if file.IsDir() { - sz, err := diskUsage(path.Join(fname, file.Name())) + sz, err := directoryUsage(path.Join(fname, file.Name())) if err != nil { return 0, err } diff --git a/client.go b/client.go index 42ec51ba0..0cae10062 100644 --- a/client.go +++ b/client.go @@ -81,6 +81,8 @@ type InternalClient interface { FinishTransaction(ctx context.Context, id string) (*Transaction, error) Transactions(ctx context.Context) (map[string]*Transaction, error) GetTransaction(ctx context.Context, id string) (*Transaction, error) + + GetNodeUsage(ctx context.Context, uri *URI) (map[string]NodeUsage, error) } //=============== @@ -227,3 +229,7 @@ func (n nopInternalClient) Transactions(ctx context.Context) (map[string]*Transa func (n nopInternalClient) GetTransaction(ctx context.Context, id string) (*Transaction, error) { return nil, nil } + +func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *URI) (map[string]NodeUsage, error) { + return nil, nil +} diff --git a/http/client.go b/http/client.go index 5f4e25b8a..3e2df94b4 100644 --- a/http/client.go +++ b/http/client.go @@ -1247,6 +1247,37 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pilosa.URI, return tkresp.Keys, nil } +// GetNodeUsage retrieves the size-on-disk information for the specified node. +func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pilosa.URI) (map[string]pilosa.NodeUsage, error) { + u := uri.Path("/ui/usage?remote=true") + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.executeRequest(req.WithContext(ctx)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } + + nodeUsages := make(map[string]pilosa.NodeUsage) // map of size 1 + if err := json.Unmarshal(body, &nodeUsages); err != nil { + return nil, fmt.Errorf("unmarshal response: %s", err) + } + return nodeUsages, nil +} + func (c *InternalClient) Transactions(ctx context.Context) (map[string]*pilosa.Transaction, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Transactions") defer span.Finish() diff --git a/http/handler.go b/http/handler.go index 98634111d..bb8cdf9f6 100644 --- a/http/handler.go +++ b/http/handler.go @@ -665,34 +665,25 @@ func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return } - usageIndexes, usageTotal, err := h.api.Usage() + + q := r.URL.Query() + remoteStr := q.Get("remote") + var remote bool + if remoteStr == "true" { + remote = true + } + + nodeUsages, err := h.api.Usage(r.Context(), remote) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } - disk := diskUsage{ - Total: usageTotal, - Indexes: usageIndexes, - } - - usage := getUsageResponse{ - Disk: disk, - } - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(usage); err != nil { + if err := json.NewEncoder(w).Encode(nodeUsages); err != nil { h.logger.Printf("write status response error: %s", err) } } -type getUsageResponse struct { - Disk diskUsage `json:"bytesOnDisk"` -} -type diskUsage struct { - Total int64 `json:"total"` - Indexes map[string]int64 `json:"indexes"` -} - // handleGetStatus handles GET /status requests. func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { diff --git a/server/handler_test.go b/server/handler_test.go index 47e4698b0..4096dc970 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -385,13 +385,18 @@ func TestHandler_Endpoints(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/ui/usage", nil)) if w.Code != gohttp.StatusOK { + fmt.Printf("%+v\n", w.Body) t.Fatalf("unexpected status code: %d", w.Code) } - ret := mustJSONDecode(t, w.Body) - usage := ret["bytesOnDisk"].(map[string]interface{}) - indexes := usage["indexes"].(map[string]interface{}) - if len(indexes) != 2 { - t.Fatalf("wrong length index size list: %#v", indexes) + nodeUsages := make(map[string]pilosa.NodeUsage) + if err := json.Unmarshal(w.Body.Bytes(), &nodeUsages); err != nil { + t.Fatalf("unmarshal") + } + + for _, nodeUsage := range nodeUsages { + if len(nodeUsage.Disk.Indexes) != 2 { + t.Fatalf("wrong length index size list: %#v", nodeUsage.Disk.Indexes) + } } }) From ba4ca1a93ed2bd127381f583e649046cb97b8084 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 9 Oct 2020 02:33:32 -0500 Subject: [PATCH 2/8] Include disk capacity in usage response --- api.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/api.go b/api.go index 9d6916558..f0981bc2e 100644 --- a/api.go +++ b/api.go @@ -38,6 +38,7 @@ import ( "github.com/pilosa/pilosa/v2/stats" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" + "github.com/shirou/gopsutil/disk" "golang.org/x/sync/errgroup" ) @@ -800,8 +801,9 @@ type NodeUsage struct { // DiskUsage represents the storage space used on disk by one node. type DiskUsage struct { - Total int64 `json:"total"` - Indexes map[string]int64 `json:"indexes"` + Capacity uint64 `json:"capacity"` + TotalUse int64 `json:"totalInUse"` + Indexes map[string]int64 `json:"indexes"` } // Usage gets the disk usage per index, in a map[nodeID]NodeUsage @@ -845,11 +847,20 @@ func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, e totalSize += indexSizes[file.Name()] } + usageStats, err := disk.Usage("/") + capacity := usageStats.Total + + if err != nil { + capacity = uint64(0) + api.server.logger.Printf("failed to get disk capacity: %s", err) + } + // Insert into result. nodeUsage := NodeUsage{ Disk: DiskUsage{ - Total: totalSize, - Indexes: indexSizes, + Capacity: capacity, + TotalUse: totalSize, + Indexes: indexSizes, }, } nodeUsages[api.server.nodeID] = nodeUsage From 6a298590becd390022ba4f22e80f55dc3687e2c0 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 9 Oct 2020 09:27:48 -0500 Subject: [PATCH 3/8] Set omitempty for disk capacity json --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index f0981bc2e..180988a7e 100644 --- a/api.go +++ b/api.go @@ -801,7 +801,7 @@ type NodeUsage struct { // DiskUsage represents the storage space used on disk by one node. type DiskUsage struct { - Capacity uint64 `json:"capacity"` + Capacity uint64 `json:"capacity,omitempty"` TotalUse int64 `json:"totalInUse"` Indexes map[string]int64 `json:"indexes"` } From 7a32eadc6487aa4a4c95333a01c6d212d69dff00 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 9 Oct 2020 10:21:37 -0500 Subject: [PATCH 4/8] Move disk capacity lookup to gopsutil wrapper package --- api.go | 5 +---- diagnostics.go | 6 ++++++ gopsutil/systeminfo.go | 11 +++++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/api.go b/api.go index 180988a7e..4d52ab1d9 100644 --- a/api.go +++ b/api.go @@ -38,7 +38,6 @@ import ( "github.com/pilosa/pilosa/v2/stats" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" - "github.com/shirou/gopsutil/disk" "golang.org/x/sync/errgroup" ) @@ -847,11 +846,9 @@ func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, e totalSize += indexSizes[file.Name()] } - usageStats, err := disk.Usage("/") - capacity := usageStats.Total + capacity, err := api.server.systemInfo.DiskCapacity(api.server.dataDir) if err != nil { - capacity = uint64(0) api.server.logger.Printf("failed to get disk capacity: %s", err) } diff --git a/diagnostics.go b/diagnostics.go index 9c380c1b8..3899eb14d 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -275,6 +275,7 @@ type SystemInfo interface { CPUCores() (physical int, logical int, err error) CPUMHz() (int, error) CPUArch() string + DiskCapacity(string) (uint64, error) } // newNopSystemInfo creates a no-op implementation of SystemInfo. @@ -345,3 +346,8 @@ func (n *nopSystemInfo) CPUMHz() (int, error) { func (n *nopSystemInfo) CPUCores() (physical, logical int, err error) { return 0, 0, nil } + +// DiskCapacity returns the disk capacity +func (n *nopSystemInfo) DiskCapacity(path string) (uint64, error) { + return 0, nil +} diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go index b251c39a4..312dd3f36 100644 --- a/gopsutil/systeminfo.go +++ b/gopsutil/systeminfo.go @@ -21,6 +21,7 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/shirou/gopsutil/cpu" + "github.com/shirou/gopsutil/disk" "github.com/shirou/gopsutil/host" "github.com/shirou/gopsutil/mem" ) @@ -243,6 +244,16 @@ func (s *systemInfo) CPUCores() (physical, logical int, err error) { return s.cpuPhysicalCores, s.cpuLogicalCores, nil } +// DiskCapacity returns the disk capacity. +func (s *systemInfo) DiskCapacity(path string) (uint64, error) { + diskInfo, err := disk.Usage(path) + + if err != nil { + return 0, err + } + return diskInfo.Total, nil +} + // NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo. func NewSystemInfo() *systemInfo { return &systemInfo{} From e2cafd98efa2ca97c658ce1e021fed039acfc3b5 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 12 Oct 2020 16:41:55 -0500 Subject: [PATCH 5/8] Move index size calculation to TxFactory --- api.go | 75 +++++++-------------------------------------------- txfactory.go | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 66 deletions(-) diff --git a/api.go b/api.go index 4d52ab1d9..3005bafbe 100644 --- a/api.go +++ b/api.go @@ -25,8 +25,6 @@ import ( "io/ioutil" "math" "net/url" - "os" - "path" "sort" "strconv" "strings" @@ -811,45 +809,19 @@ func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, e defer span.Finish() nodeUsages := make(map[string]NodeUsage) + + indexSizes, err := api.holder.Txf().IndexSizes() + if err != nil { + return nil, errors.Wrap(err, "getting index usage") + } var totalSize int64 - - // Open storage directory. - dirName, err := expandDirName(api.server.dataDir) - if err != nil { - return nodeUsages, errors.Wrap(err, "expanding data directory") - } - dir, err := os.Open(dirName) - if err != nil { - return nodeUsages, errors.Wrap(err, "opening data directory") - } - defer dir.Close() - - files, err := dir.Readdir(-1) - if err != nil { - return nodeUsages, errors.Wrap(err, "reading data directory") + for _, s := range indexSizes { + totalSize += s } - // Read size on disk for each index directory. - indexSizes := make(map[string]int64) - for _, file := range files { - if !file.IsDir() { - continue - } - if api.holder.Txf().IsTxDatabasePath(file.Name()) { - continue - } - fullName := path.Join(dirName, file.Name()) - indexSizes[file.Name()], err = directoryUsage(fullName) - if err != nil { - return nodeUsages, errors.Wrap(err, "getting disk usage") - } - totalSize += indexSizes[file.Name()] - } - - capacity, err := api.server.systemInfo.DiskCapacity(api.server.dataDir) - + capacity, err := api.server.systemInfo.DiskCapacity(api.holder.path) if err != nil { - api.server.logger.Printf("failed to get disk capacity: %s", err) + api.server.logger.Printf("couldn't read disk capacity: %s", err) } // Insert into result. @@ -879,35 +851,6 @@ func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, e return nodeUsages, nil } -func directoryUsage(fname string) (int64, error) { - var size int64 - - dir, err := os.Open(fname) - if err != nil { - return 0, errors.Wrap(err, "opening data subdirectory") - } - defer dir.Close() - - files, err := dir.Readdir(-1) - if err != nil { - return 0, errors.Wrap(err, "reading data subdirectory") - } - - for _, file := range files { - if file.IsDir() { - sz, err := directoryUsage(path.Join(fname, file.Name())) - if err != nil { - return 0, err - } - size += sz - } else { - size += file.Size() - } - } - - return size, nil -} - // RecalculateCaches forces all TopN caches to be updated. // This is done internally within a TopN query, but a user may want to do it ahead of time? func (api *API) RecalculateCaches(ctx context.Context) error { diff --git a/txfactory.go b/txfactory.go index a72f451c3..521a9f0b3 100644 --- a/txfactory.go +++ b/txfactory.go @@ -18,6 +18,7 @@ import ( "fmt" "io" "os" + "path" "path/filepath" "strconv" "strings" @@ -564,6 +565,81 @@ func (f *TxFactory) DumpAll() { f.dbPerShard.DumpAll() } +func (f *TxFactory) IndexSizes() (map[string]int64, error) { + switch f.types[0] { + case roaringTxn: + return f.diskUsageFromFilesystem() + default: + return nil, errors.New("Not implemented") + } + +} + +func (f *TxFactory) diskUsageFromFilesystem() (map[string]int64, error) { + // Open storage directory. + indexSizes := make(map[string]int64) + dirName, err := expandDirName(f.holder.path) + if err != nil { + return indexSizes, errors.Wrap(err, "expanding data directory") + } + dir, err := os.Open(dirName) + if err != nil { + return indexSizes, errors.Wrap(err, "opening data directory") + } + defer dir.Close() + + files, err := dir.Readdir(-1) + if err != nil { + return indexSizes, errors.Wrap(err, "reading data directory") + } + + // Read size on disk for each index directory. + for _, file := range files { + if !file.IsDir() { + continue + } + if f.IsTxDatabasePath(file.Name()) { + continue + } + fullName := path.Join(dirName, file.Name()) + indexSizes[file.Name()], err = directoryUsage(fullName) + if err != nil { + return indexSizes, errors.Wrap(err, "getting disk usage") + } + } + + return indexSizes, nil +} + +func directoryUsage(fname string) (int64, error) { + var size int64 + + dir, err := os.Open(fname) + if err != nil { + return 0, errors.Wrap(err, "opening data subdirectory") + } + defer dir.Close() + + files, err := dir.Readdir(-1) + if err != nil { + return 0, errors.Wrap(err, "reading data subdirectory") + } + + for _, file := range files { + if file.IsDir() { + sz, err := directoryUsage(path.Join(fname, file.Name())) + if err != nil { + return 0, err + } + size += sz + } else { + size += file.Size() + } + } + + return size, nil +} + func (f *TxFactory) CloseIndex(idx *Index) error { // under roaring and all the new databases, this is a no-op. //idx.Dump("CloseIndex") From 3a40b586b3487e6506ce9f40fdb5e6c593264f80 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Wed, 14 Oct 2020 12:53:35 -0500 Subject: [PATCH 6/8] disk usage per index --- go.mod | 2 +- go.sum | 2 ++ txfactory.go | 32 ++++++++++++-------------------- 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 6f3b9fa55..7ffb4b20f 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 github.com/dustin/go-humanize v1.0.0 github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 - github.com/glycerine/lmdb-go v1.9.32 + github.com/glycerine/lmdb-go v1.9.34 github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.1 github.com/golang/protobuf v1.3.3 diff --git a/go.sum b/go.sum index 231a844ad..dd7dbd434 100644 --- a/go.sum +++ b/go.sum @@ -66,6 +66,8 @@ github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06A github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y= github.com/glycerine/lmdb-go v1.9.32 h1:thLnzCykFcmn2rACYnwpR4ovYauLNKaAuk+xj7YMbS0= github.com/glycerine/lmdb-go v1.9.32/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs= +github.com/glycerine/lmdb-go v1.9.34 h1:0lymJjpdelYnIMcNzsKROfIaApt99zhaHtjDJTHjGkE= +github.com/glycerine/lmdb-go v1.9.34/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= diff --git a/txfactory.go b/txfactory.go index 521a9f0b3..fe1769f43 100644 --- a/txfactory.go +++ b/txfactory.go @@ -582,30 +582,22 @@ func (f *TxFactory) diskUsageFromFilesystem() (map[string]int64, error) { if err != nil { return indexSizes, errors.Wrap(err, "expanding data directory") } - dir, err := os.Open(dirName) - if err != nil { - return indexSizes, errors.Wrap(err, "opening data directory") - } - defer dir.Close() - files, err := dir.Readdir(-1) - if err != nil { - return indexSizes, errors.Wrap(err, "reading data directory") - } + idxs := f.holder.Indexes() - // Read size on disk for each index directory. - for _, file := range files { - if !file.IsDir() { - continue - } - if f.IsTxDatabasePath(file.Name()) { - continue - } - fullName := path.Join(dirName, file.Name()) - indexSizes[file.Name()], err = directoryUsage(fullName) + for _, idx := range idxs { + index := idx.name + fullName := path.Join(dirName, index) + roaringAndMeta, err := directoryUsage(fullName) if err != nil { - return indexSizes, errors.Wrap(err, "getting disk usage") + return indexSizes, errors.Wrap(err, "getting disk usage for roaring and meta") } + fullName = index + ".index.txstores@@@" + rbfOrLmdb, err := directoryUsage(fullName) + if err != nil { + return indexSizes, errors.Wrap(err, "getting disk usage for backend") + } + indexSizes[index] = roaringAndMeta + rbfOrLmdb } return indexSizes, nil From 244ba21da7b27dc825dfd88976230f1ff7828084 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Wed, 14 Oct 2020 13:19:14 -0500 Subject: [PATCH 7/8] better names --- txfactory.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/txfactory.go b/txfactory.go index fe1769f43..4589fa10f 100644 --- a/txfactory.go +++ b/txfactory.go @@ -575,12 +575,13 @@ func (f *TxFactory) IndexSizes() (map[string]int64, error) { } -func (f *TxFactory) diskUsageFromFilesystem() (map[string]int64, error) { +// +func (f *TxFactory) diskUsageFromFilesystem() (index2bytes map[string]int64, err error) { // Open storage directory. - indexSizes := make(map[string]int64) + index2bytes = make(map[string]int64) dirName, err := expandDirName(f.holder.path) if err != nil { - return indexSizes, errors.Wrap(err, "expanding data directory") + return index2bytes, errors.Wrap(err, "expanding data directory") } idxs := f.holder.Indexes() @@ -590,17 +591,17 @@ func (f *TxFactory) diskUsageFromFilesystem() (map[string]int64, error) { fullName := path.Join(dirName, index) roaringAndMeta, err := directoryUsage(fullName) if err != nil { - return indexSizes, errors.Wrap(err, "getting disk usage for roaring and meta") + return index2bytes, errors.Wrap(err, "getting disk usage for roaring and meta") } fullName = index + ".index.txstores@@@" rbfOrLmdb, err := directoryUsage(fullName) if err != nil { - return indexSizes, errors.Wrap(err, "getting disk usage for backend") + return index2bytes, errors.Wrap(err, "getting disk usage for backend") } - indexSizes[index] = roaringAndMeta + rbfOrLmdb + index2bytes[index] = roaringAndMeta + rbfOrLmdb } - return indexSizes, nil + return index2bytes, nil } func directoryUsage(fname string) (int64, error) { From aff5c0fc5723fcc83425989faa62bbfb420b4c1f Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 14 Oct 2020 15:46:19 -0500 Subject: [PATCH 8/8] Skip missing directories --- go.sum | 2 -- txfactory.go | 17 +++++------------ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/go.sum b/go.sum index dd7dbd434..2f6d5c213 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,6 @@ github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E= github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y= -github.com/glycerine/lmdb-go v1.9.32 h1:thLnzCykFcmn2rACYnwpR4ovYauLNKaAuk+xj7YMbS0= -github.com/glycerine/lmdb-go v1.9.32/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs= github.com/glycerine/lmdb-go v1.9.34 h1:0lymJjpdelYnIMcNzsKROfIaApt99zhaHtjDJTHjGkE= github.com/glycerine/lmdb-go v1.9.34/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= diff --git a/txfactory.go b/txfactory.go index 4589fa10f..9f01e845a 100644 --- a/txfactory.go +++ b/txfactory.go @@ -565,18 +565,7 @@ func (f *TxFactory) DumpAll() { f.dbPerShard.DumpAll() } -func (f *TxFactory) IndexSizes() (map[string]int64, error) { - switch f.types[0] { - case roaringTxn: - return f.diskUsageFromFilesystem() - default: - return nil, errors.New("Not implemented") - } - -} - -// -func (f *TxFactory) diskUsageFromFilesystem() (index2bytes map[string]int64, err error) { +func (f *TxFactory) IndexSizes() (index2bytes map[string]int64, err error) { // Open storage directory. index2bytes = make(map[string]int64) dirName, err := expandDirName(f.holder.path) @@ -605,6 +594,10 @@ func (f *TxFactory) diskUsageFromFilesystem() (index2bytes map[string]int64, err } func directoryUsage(fname string) (int64, error) { + if !DirExists(fname) { + return 0, nil + } + var size int64 dir, err := os.Open(fname)