diff --git a/api.go b/api.go index 2bd8eb2e4..62b445484 100644 --- a/api.go +++ b/api.go @@ -821,25 +821,41 @@ type NodeUsage struct { // DiskUsage represents the storage space used on disk by one node. type DiskUsage struct { - Capacity uint64 `json:"capacity,omitempty"` - TotalUse int64 `json:"totalInUse"` - Indexes map[string]int64 `json:"indexes"` + Capacity uint64 `json:"capacity,omitempty"` + TotalUse uint64 `json:"totalInUse"` + IndexUsage map[string]IndexUsage `json:"indexes"` } -// Usage gets the disk usage per index, in a map[nodeID]NodeUsage +// IndexUsage represents the storage space used on disk by one index, on one node. +type IndexUsage struct { + Total uint64 `json:"total"` + IndexKeys uint64 `json:"indexKeys"` + FieldKeysTotal uint64 `json:"fieldKeysTotal"` + Fragments uint64 `json:"fragments"` + Fields map[string]FieldUsage `json:"fields"` +} + +// FieldUsage represents the storage space used on disk by one field, on one node +type FieldUsage struct { + Total uint64 `json:"total"` + Fragments uint64 `json:"fragments"` + Keys uint64 `json:"keys"` +} + +// Usage gets the disk usage, in a map[nodeID]NodeUsage. func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.Usage") defer span.Finish() nodeUsages := make(map[string]NodeUsage) - indexSizes, err := api.holder.Txf().IndexSizes() + indexDetails, err := api.holder.Txf().IndexUsageDetails() if err != nil { return nil, errors.Wrap(err, "getting index usage") } - var totalSize int64 - for _, s := range indexSizes { - totalSize += s + var totalSize uint64 + for _, s := range indexDetails { + totalSize += s.Total } capacity, err := api.server.systemInfo.DiskCapacity(api.holder.path) @@ -850,9 +866,9 @@ func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, e // Insert into result. nodeUsage := NodeUsage{ Disk: DiskUsage{ - Capacity: capacity, - TotalUse: totalSize, - Indexes: indexSizes, + Capacity: capacity, + TotalUse: totalSize, + IndexUsage: indexDetails, }, } nodeUsages[api.server.nodeID] = nodeUsage diff --git a/bluegreentx.go b/bluegreentx.go index 62f1c0bd8..e0d9ddee9 100644 --- a/bluegreentx.go +++ b/bluegreentx.go @@ -663,6 +663,10 @@ func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64, return bgi, bfound, errB } +func (tx *blueGreenTx) GetFieldSizeBytes(index, field string) (uint64, error) { + return 0, nil +} + func NewBlueGreenIterator(tx *blueGreenTx, ait, bit roaring.ContainerIterator) *blueGreenIterator { return &blueGreenIterator{ tx: tx, diff --git a/bolt.go b/bolt.go index b8a3f25be..3b37a2bec 100644 --- a/bolt.go +++ b/bolt.go @@ -759,6 +759,10 @@ func (tx *BoltTx) ContainerIterator(index, field, view string, shard uint64, fir return bi, bytes.Equal(bi.lastKey, needle), nil } +func (tx *BoltTx) GetFieldSizeBytes(index, field string) (uint64, error) { + return 0, nil +} + // BoltIterator is the iterator returned from a BoltTx.ContainerIterator() call. // It implements the roaring.ContainerIterator interface. type BoltIterator struct { diff --git a/catcher.go b/catcher.go index 671ee7662..09f24ce93 100644 --- a/catcher.go +++ b/catcher.go @@ -317,3 +317,7 @@ func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey ui func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) { return c.b.GetSortedFieldViewList(idx, shard) } + +func (tx *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) { + return 0, nil +} diff --git a/rbf.go b/rbf.go index dff5a1123..21c1a321a 100644 --- a/rbf.go +++ b/rbf.go @@ -435,6 +435,10 @@ func (tx *RBFTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.F return tx.tx.GetSortedFieldViewList() } +func (tx *RBFTx) GetFieldSizeBytes(index, field string) (uint64, error) { + return 0, nil +} + // rbfName returns a NULL-separated key used for identifying bitmap maps in RBF. func rbfName(index, field, view string, shard uint64) string { return string(txkey.Prefix(index, field, view, shard)) diff --git a/rbf/tx.go b/rbf/tx.go index a952104ec..f89b38048 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -839,6 +839,25 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) { return m, nil } +func (tx *Tx) GetFieldSizeBytes(index, field string) (uint64, error) { + + fmt.Printf("getting RBF field size %s/%s\n", index, field) + + var pgno uint32 + var parent uint32 + + var pageCount uint64 + + if err := tx.walkTree(pgno, parent, func(pgno, parent, typ uint32) error { + pageCount++ + return nil + }); err != nil { + return 0, err + } + + return uint64(pageCount * PageSize), nil +} + // walkTree recursively iterates over a page and all its children. func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) error) error { // Read page and iterate over children. @@ -964,6 +983,7 @@ func (tx *Tx) deallocateTree(pgno uint32) error { func (tx *Tx) readPage(pgno uint32) (_ []byte, isHeap bool, err error) { // Meta page is always cached on the transaction. + //fmt.Printf("readPage %d\n", pgno) if pgno == 0 { return tx.meta[:], false, nil } @@ -971,7 +991,7 @@ func (tx *Tx) readPage(pgno uint32) (_ []byte, isHeap bool, err error) { // Verify page number requested is within current size of database. pageN := readMetaPageN(tx.meta[:]) if pgno > pageN { - return nil, false, fmt.Errorf("rbf: page read out of bounds: pgno=%d max=%d", pgno, pageN) + return nil, false, fmt.Errorf("rbf: page read out of bounds: pgno=%d max=%d", pgno, pageN-1) } // Check if page has been updated in this tx. diff --git a/rrtx.go b/rrtx.go index 8c75a2924..ca89b6d3d 100644 --- a/rrtx.go +++ b/rrtx.go @@ -589,6 +589,10 @@ func (tx *RoaringTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txk return } +func (tx *RoaringTx) GetFieldSizeBytes(index, field string) (uint64, error) { + return 0, nil +} + //////// registrar and wrapper machinery // roaringRegistrar mirrors the machinery expected diff --git a/stattx.go b/stattx.go index 02b60d6e1..7009de9ac 100644 --- a/stattx.go +++ b/stattx.go @@ -681,3 +681,7 @@ func (c *statTx) Sn() int64 { func (c *statTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) { return c.b.GetSortedFieldViewList(idx, shard) } + +func (tx *statTx) GetFieldSizeBytes(index, field string) (uint64, error) { + return 0, nil +} diff --git a/tx.go b/tx.go index 771e94ff8..f33648cb0 100644 --- a/tx.go +++ b/tx.go @@ -219,6 +219,8 @@ type Tx interface { // GetSortedFieldViewList gets the set of FieldView(s) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) + + GetFieldSizeBytes(index, field string) (uint64, error) } // Closer is used by Finders diff --git a/txfactory.go b/txfactory.go index d8aaf3aee..5e2ad647b 100644 --- a/txfactory.go +++ b/txfactory.go @@ -593,40 +593,115 @@ func (f *TxFactory) DumpAll() { f.dbPerShard.DumpAll() } -func (f *TxFactory) IndexSizes() (index2bytes map[string]int64, err error) { - // Open storage directory. - index2bytes = make(map[string]int64) +func (f *TxFactory) IndexUsageDetails() (map[string]IndexUsage, error) { + indexUsage := make(map[string]IndexUsage) dirName, err := expandDirName(f.holder.path) if err != nil { - return index2bytes, errors.Wrap(err, "expanding data directory") + return indexUsage, errors.Wrap(err, "expanding data directory") } idxs := f.holder.Indexes() + /* + qcx := f.NewQcx() + tx, finisher, err := qcx.GetTx(Txo{Write: !writable}) + if err != nil { + return indexUsage, errors.Wrap(err, "qcx.GetTx") + } + defer finisher(nil) + */ + for _, idx := range idxs { index := idx.name - fullName := path.Join(dirName, index) - roaringAndMeta, err := directoryUsage(fullName) - if err != nil { - return index2bytes, errors.Wrap(err, "getting disk usage for roaring and meta") + println(" i:" + index) + indexPath := path.Join(dirName, index) + + // field usage + fieldUsages := make(map[string]FieldUsage) + fragmentsTotal := uint64(0) + fieldKeysTotal := uint64(0) + flds := idx.Fields() + for _, fld := range flds { + field := fld.Name() + fUsage, err := f.FieldUsage(indexPath, fld) + if err != nil { + return indexUsage, errors.Wrapf(err, "getting disk usage for index (%s)", index) + } + fieldUsages[field] = fUsage + keysBytes := fieldUsages[field].Keys + fieldKeysTotal += keysBytes + fragmentsTotal += fieldUsages[field].Fragments + + // non-roaring field usage + /* + fieldBytes, err := tx.GetFieldSizeBytes(index, field) + if err != nil { + return indexUsage, errors.Wrapf(err, "getting disk usage for non-roaring fragments (%s)", field) + } + fieldUsages[field] = FieldUsage{ + Total: fieldBytes, + Fragments: fieldBytes - keysBytes, + Keys: keysBytes, + } + */ } - fullName += ".index.txstores@@@" - rbfOrLmdb, err := directoryUsage(fullName) - if err != nil { - return index2bytes, errors.Wrap(err, "getting disk usage for backend") + + // index keys usage + keysBytes := uint64(0) + if idx.keys { + keysPath := path.Join(indexPath, translateStoreDir) + keysBytes, err = directoryUsage(keysPath) + if err != nil { + return indexUsage, errors.Wrapf(err, "getting disk usage for index keys (%s)", index) + } + } + + indexUsage[index] = IndexUsage{ + Total: keysBytes + fieldKeysTotal + fragmentsTotal, + IndexKeys: keysBytes, + FieldKeysTotal: fieldKeysTotal, + Fragments: fragmentsTotal, + Fields: fieldUsages, } - index2bytes[index] = roaringAndMeta + rbfOrLmdb } - return index2bytes, nil + return indexUsage, nil } -func directoryUsage(fname string) (int64, error) { - if !DirExists(fname) { - return 0, nil +func (f *TxFactory) FieldUsage(indexPath string, fld *Field) (FieldUsage, error) { + fieldUsage := FieldUsage{} + + field := fld.name + println(" f:" + field) + + // roaring field usage + fieldPath := path.Join(indexPath, field) + fieldBytes, err := directoryUsage(fieldPath) + if err != nil { + return fieldUsage, errors.Wrapf(err, "getting disk usage for field (%s)", field) + } + keysBytes := int64(0) + if fld.usesKeys { + keysBytes, err = fileSize(fld.TranslateStorePath()) + if err != nil { + return fieldUsage, errors.Wrapf(err, "getting disk usage for field keys (%s)", field) + } + } + fieldUsage = FieldUsage{ + Total: fieldBytes, + Fragments: fieldBytes - uint64(keysBytes), + Keys: uint64(keysBytes), } - var size int64 + return fieldUsage, nil +} + +func directoryUsage(fname string) (uint64, error) { + if !DirExists(fname) { + return 0, errors.Errorf("directory does not exist (%s)", fname) + } + + var size uint64 dir, err := os.Open(fname) if err != nil { @@ -647,7 +722,7 @@ func directoryUsage(fname string) (int64, error) { } size += sz } else { - size += file.Size() + size += uint64(file.Size()) // NOTE this cast is safe for regular files, not necessarily others } }