From 85fad859e2245dfdb770241ac97859e2bfd7f6e6 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 15 Jan 2021 03:45:10 -0600 Subject: [PATCH] Correct some disk usage computations --- api.go | 10 ++-- server/handler_test.go | 10 +++- server/server.go | 2 +- txfactory.go | 110 +++++++++++++++++++++++++++-------------- 4 files changed, 86 insertions(+), 46 deletions(-) diff --git a/api.go b/api.go index 62b445484..913433807 100644 --- a/api.go +++ b/api.go @@ -816,7 +816,7 @@ func (api *API) Node() *Node { // NodeUsage represents all usage measurements for one node. type NodeUsage struct { - Disk DiskUsage `json:"bytesOnDisk"` + Disk DiskUsage `json:"diskUsage"` } // DiskUsage represents the storage space used on disk by one node. @@ -849,11 +849,11 @@ func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, e nodeUsages := make(map[string]NodeUsage) - indexDetails, err := api.holder.Txf().IndexUsageDetails() + indexDetails, nodeMetadataBytes, err := api.holder.Txf().IndexUsageDetails() if err != nil { - return nil, errors.Wrap(err, "getting index usage") + return nil, errors.Wrap(err, "getting node usage") } - var totalSize uint64 + totalSize := nodeMetadataBytes for _, s := range indexDetails { totalSize += s.Total } @@ -873,7 +873,7 @@ func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, e } nodeUsages[api.server.nodeID] = nodeUsage - // Collect size on disk from remote nodes + // Collect diskUsage from remote nodes if !remote { nodes := api.cluster.Nodes() for _, node := range nodes { diff --git a/server/handler_test.go b/server/handler_test.go index 705cfdb5c..5154349e7 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -396,10 +396,16 @@ func TestHandler_Endpoints(t *testing.T) { } for _, nodeUsage := range nodeUsages { - if len(nodeUsage.Disk.Indexes) != 2 { - t.Fatalf("wrong length index size list: %#v", nodeUsage.Disk.Indexes) + numIndexes := len(nodeUsage.Disk.IndexUsage) + if numIndexes != 2 { + t.Fatalf("wrong length index usage list: expected %d, got %d", 2, numIndexes) + } + numFields := len(nodeUsage.Disk.IndexUsage["i1"].Fields) + if numFields != len(i1.Fields()) { + t.Fatalf("wrong length field usage list: expected %d, got %d", len(i1.Fields()), numFields) } } + }) t.Run("UI/shard-distribution", func(t *testing.T) { diff --git a/server/server.go b/server/server.go index 51729a8b1..530e4102a 100644 --- a/server/server.go +++ b/server/server.go @@ -185,7 +185,7 @@ func (m *Command) Start() (err error) { return errors.Wrap(err, "opening server") } - m.logger.Printf("listening! as %s\n", m.listenURI) + m.logger.Printf("listening as %s\n", m.listenURI) go func() { if err := m.grpcServer.Serve(); err != nil { m.logger.Printf("grpc server error: %v", err) diff --git a/txfactory.go b/txfactory.go index a7984e518..a93fa9c3d 100644 --- a/txfactory.go +++ b/txfactory.go @@ -593,11 +593,13 @@ func (f *TxFactory) DumpAll() { f.dbPerShard.DumpAll() } -func (f *TxFactory) IndexUsageDetails() (map[string]IndexUsage, error) { +// IndexUsageDetails computes the sum of filesizes used by the node, broken down +// by index, field, fragments and keys. +func (f *TxFactory) IndexUsageDetails() (map[string]IndexUsage, uint64, error) { indexUsage := make(map[string]IndexUsage) - dirName, err := expandDirName(f.holder.path) + holderPath, err := expandDirName(f.holder.path) if err != nil { - return indexUsage, errors.Wrap(err, "expanding data directory") + return indexUsage, 0, errors.Wrap(err, "expanding data directory") } idxs := f.holder.Indexes() @@ -605,8 +607,7 @@ func (f *TxFactory) IndexUsageDetails() (map[string]IndexUsage, error) { qcx := f.NewQcx() for _, idx := range idxs { index := idx.name - println(" i:" + index) - indexPath := path.Join(dirName, index) + indexPath := path.Join(holderPath, index) // field usage fieldUsages := make(map[string]FieldUsage) @@ -615,16 +616,16 @@ func (f *TxFactory) IndexUsageDetails() (map[string]IndexUsage, error) { flds := idx.Fields() for _, fld := range flds { field := fld.Name() - fUsage, err := f.FieldUsage(indexPath, fld) - if err != nil { - return indexUsage, errors.Wrapf(err, "getting disk usage for index (%s)", index) + if field == "_keys" { + continue + } + fUsage, err := f.fieldUsage(indexPath, fld) + if err != nil { + return indexUsage, 0, errors.Wrapf(err, "getting disk usage for index (%s)", index) } - fieldUsages[field] = fUsage - keysBytes := fieldUsages[field].Keys - fieldKeysTotal += keysBytes - fragmentsTotal += fieldUsages[field].Fragments - fieldUsage := FieldUsage{Keys: keysBytes} + // non-roaring field usage + fragmentUsage := uint64(0) for _, shard := range fld.AvailableShards(true).Slice() { if err := func() error { @@ -634,74 +635,107 @@ func (f *TxFactory) IndexUsageDetails() (map[string]IndexUsage, error) { } defer finisher(nil) - // non-roaring field usage fieldBytes, err := tx.GetFieldSizeBytes(index, field) if err != nil { return errors.Wrapf(err, "getting disk usage for non-roaring fragments (%s)", field) } - fieldUsage.Total += fieldBytes + fragmentUsage += fieldBytes return nil }(); err != nil { - return indexUsage, err + return indexUsage, 0, err } } - fieldUsage.Fragments = fieldUsage.Total - keysBytes - fieldUsages[field] = fieldUsage + // add non-roaring to roaring + fUsage.Fragments += fragmentUsage + fUsage.Total += fragmentUsage + + // add to running total + fieldKeysTotal += fUsage.Keys + fragmentsTotal += fUsage.Fragments + + fieldUsages[field] = fUsage + } + + // index metadata, e.g. columnAttrs + indexMetaBytes, err := directoryUsage(indexPath, false) + if err != nil { + return indexUsage, 0, errors.Wrapf(err, "getting disk usage for index metadata (%s)", index) } // index keys usage - keysBytes := uint64(0) + indexKeysBytes := uint64(0) if idx.keys { keysPath := path.Join(indexPath, translateStoreDir) - keysBytes, err = directoryUsage(keysPath) + indexKeysBytes, err = directoryUsage(keysPath, true) if err != nil { - return indexUsage, errors.Wrapf(err, "getting disk usage for index keys (%s)", index) + return indexUsage, 0, errors.Wrapf(err, "getting disk usage for index keys (%s)", index) } } indexUsage[index] = IndexUsage{ - Total: keysBytes + fieldKeysTotal + fragmentsTotal, - IndexKeys: keysBytes, + Total: indexMetaBytes + indexKeysBytes + fieldKeysTotal + fragmentsTotal, + IndexKeys: indexKeysBytes, FieldKeysTotal: fieldKeysTotal, Fragments: fragmentsTotal, Fields: fieldUsages, } } - return indexUsage, nil + // node metadata, e.g. id allocator + nodeMetaBytes, err := directoryUsage(holderPath, false) + if err != nil { + return indexUsage, 0, errors.Wrapf(err, "getting disk usage for node metadata") + } + + return indexUsage, nodeMetaBytes, nil } -func (f *TxFactory) FieldUsage(indexPath string, fld *Field) (FieldUsage, error) { +// fieldUsage computes the sum of filesizes used by a field in +// the filesystem tree (roaring storage), broken down by keys and fragments. +func (f *TxFactory) fieldUsage(indexPath string, fld *Field) (FieldUsage, error) { fieldUsage := FieldUsage{} field := fld.name - println(" f:" + field) - // roaring field usage - fieldPath := path.Join(indexPath, field) - fieldBytes, err := directoryUsage(fieldPath) - if err != nil { - return fieldUsage, errors.Wrapf(err, "getting disk usage for field (%s)", field) - } + // row keys keysBytes := int64(0) + var err error if fld.usesKeys { keysBytes, err = fileSize(fld.TranslateStorePath()) if err != nil { return fieldUsage, errors.Wrapf(err, "getting disk usage for field keys (%s)", field) } } + + // field metadata, e.g. rowAttrs + fieldPath := path.Join(indexPath, field) + metaBytes, err := directoryUsage(fieldPath, false) // this includes keys + if err != nil { + return fieldUsage, errors.Wrapf(err, "getting disk usage for field meta (%s)", field) + } + + // fragment data + viewsPath := path.Join(fieldPath, "views") + fragmentBytes := uint64(0) + if dirExists(viewsPath) { + fragmentBytes, err = directoryUsage(viewsPath, true) + if err != nil { + return fieldUsage, errors.Wrapf(err, "getting disk usage for field fragments (%s)", field) + } + } + fieldUsage = FieldUsage{ - Total: fieldBytes, - Fragments: fieldBytes - uint64(keysBytes), + Total: metaBytes + fragmentBytes, + Fragments: fragmentBytes, Keys: uint64(keysBytes), } return fieldUsage, nil } -func directoryUsage(fname string) (uint64, error) { - if !DirExists(fname) { +func directoryUsage(fname string, recursive bool) (uint64, error) { + if !dirExists(fname) { return 0, errors.Errorf("directory does not exist (%s)", fname) } @@ -719,8 +753,8 @@ func directoryUsage(fname string) (uint64, error) { } for _, file := range files { - if file.IsDir() { - sz, err := directoryUsage(path.Join(fname, file.Name())) + if recursive && file.IsDir() { + sz, err := directoryUsage(path.Join(fname, file.Name()), true) if err != nil { return 0, err }