Include roaring field and key details in usage endpoint

api.go

@@ -821,25 +821,41 @@ type NodeUsage struct {
 
 // DiskUsage represents the storage space used on disk by one node.
 type DiskUsage struct {
-	Capacity uint64           `json:"capacity,omitempty"`
-	TotalUse int64            `json:"totalInUse"`
-	Indexes  map[string]int64 `json:"indexes"`
+	Capacity   uint64                `json:"capacity,omitempty"`
+	TotalUse   uint64                `json:"totalInUse"`
+	IndexUsage map[string]IndexUsage `json:"indexes"`
 }
 
-// Usage gets the disk usage per index, in a map[nodeID]NodeUsage
+// IndexUsage represents the storage space used on disk by one index, on one node.
+type IndexUsage struct {
+	Total          uint64                `json:"total"`
+	IndexKeys      uint64                `json:"indexKeys"`
+	FieldKeysTotal uint64                `json:"fieldKeysTotal"`
+	Fragments      uint64                `json:"fragments"`
+	Fields         map[string]FieldUsage `json:"fields"`
+}
+
+// FieldUsage represents the storage space used on disk by one field, on one node
+type FieldUsage struct {
+	Total     uint64 `json:"total"`
+	Fragments uint64 `json:"fragments"`
+	Keys      uint64 `json:"keys"`
+}
+
+// Usage gets the disk usage, in a map[nodeID]NodeUsage.
 func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, error) {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.Usage")
 	defer span.Finish()
 
 	nodeUsages := make(map[string]NodeUsage)
 
-	indexSizes, err := api.holder.Txf().IndexSizes()
+	indexDetails, err := api.holder.Txf().IndexUsageDetails()
 	if err != nil {
 		return nil, errors.Wrap(err, "getting index usage")
 	}
-	var totalSize int64
-	for _, s := range indexSizes {
-		totalSize += s
+	var totalSize uint64
+	for _, s := range indexDetails {
+		totalSize += s.Total
 	}
 
 	capacity, err := api.server.systemInfo.DiskCapacity(api.holder.path)
@@ -850,9 +866,9 @@ func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, e
 	// Insert into result.
 	nodeUsage := NodeUsage{
 		Disk: DiskUsage{
-			Capacity: capacity,
-			TotalUse: totalSize,
-			Indexes:  indexSizes,
+			Capacity:   capacity,
+			TotalUse:   totalSize,
+			IndexUsage: indexDetails,
 		},
 	}
 	nodeUsages[api.server.nodeID] = nodeUsage

bluegreentx.go

@@ -663,6 +663,10 @@ func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64,
 	return bgi, bfound, errB
 }
 
+func (tx *blueGreenTx) GetFieldSizeBytes(index, field string) (uint64, error) {
+	return 0, nil
+}
+
 func NewBlueGreenIterator(tx *blueGreenTx, ait, bit roaring.ContainerIterator) *blueGreenIterator {
 	return &blueGreenIterator{
 		tx:  tx,

bolt.go

@@ -759,6 +759,10 @@ func (tx *BoltTx) ContainerIterator(index, field, view string, shard uint64, fir
 	return bi, bytes.Equal(bi.lastKey, needle), nil
 }
 
+func (tx *BoltTx) GetFieldSizeBytes(index, field string) (uint64, error) {
+	return 0, nil
+}
+
 // BoltIterator is the iterator returned from a BoltTx.ContainerIterator() call.
 // It implements the roaring.ContainerIterator interface.
 type BoltIterator struct {

catcher.go

@@ -317,3 +317,7 @@ func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey ui
 func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
 	return c.b.GetSortedFieldViewList(idx, shard)
 }
+
+func (tx *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) {
+	return 0, nil
+}

rbf.go

@@ -435,6 +435,10 @@ func (tx *RBFTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.F
 	return tx.tx.GetSortedFieldViewList()
 }
 
+func (tx *RBFTx) GetFieldSizeBytes(index, field string) (uint64, error) {
+	return 0, nil
+}
+
 // rbfName returns a NULL-separated key used for identifying bitmap maps in RBF.
 func rbfName(index, field, view string, shard uint64) string {
 	return string(txkey.Prefix(index, field, view, shard))

rbf/tx.go

@@ -839,6 +839,25 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) {
 	return m, nil
 }
 
+func (tx *Tx) GetFieldSizeBytes(index, field string) (uint64, error) {
+
+	fmt.Printf("getting RBF field size %s/%s\n", index, field)
+
+	var pgno uint32
+	var parent uint32
+
+	var pageCount uint64
+
+	if err := tx.walkTree(pgno, parent, func(pgno, parent, typ uint32) error {
+		pageCount++
+		return nil
+	}); err != nil {
+		return 0, err
+	}
+
+	return uint64(pageCount * PageSize), nil
+}
+
 // walkTree recursively iterates over a page and all its children.
 func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) error) error {
 	// Read page and iterate over children.
@@ -964,6 +983,7 @@ func (tx *Tx) deallocateTree(pgno uint32) error {
 
 func (tx *Tx) readPage(pgno uint32) (_ []byte, isHeap bool, err error) {
 	// Meta page is always cached on the transaction.
+	//fmt.Printf("readPage %d\n", pgno)
 	if pgno == 0 {
 		return tx.meta[:], false, nil
 	}
@@ -971,7 +991,7 @@ func (tx *Tx) readPage(pgno uint32) (_ []byte, isHeap bool, err error) {
 	// Verify page number requested is within current size of database.
 	pageN := readMetaPageN(tx.meta[:])
 	if pgno > pageN {
-		return nil, false, fmt.Errorf("rbf: page read out of bounds: pgno=%d max=%d", pgno, pageN)
+		return nil, false, fmt.Errorf("rbf: page read out of bounds: pgno=%d max=%d", pgno, pageN-1)
 	}
 
 	// Check if page has been updated in this tx.

rrtx.go

@@ -589,6 +589,10 @@ func (tx *RoaringTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txk
 	return
 }
 
+func (tx *RoaringTx) GetFieldSizeBytes(index, field string) (uint64, error) {
+	return 0, nil
+}
+
 //////// registrar and wrapper machinery
 
 // roaringRegistrar mirrors the machinery expected

stattx.go

@@ -681,3 +681,7 @@ func (c *statTx) Sn() int64 {
 func (c *statTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
 	return c.b.GetSortedFieldViewList(idx, shard)
 }
+
+func (tx *statTx) GetFieldSizeBytes(index, field string) (uint64, error) {
+	return 0, nil
+}

tx.go

@@ -219,6 +219,8 @@ type Tx interface {
 
 	// GetSortedFieldViewList gets the set of FieldView(s)
 	GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error)
+
+	GetFieldSizeBytes(index, field string) (uint64, error)
 }
 
 // Closer is used by Finders

txfactory.go

@@ -593,40 +593,115 @@ func (f *TxFactory) DumpAll() {
 	f.dbPerShard.DumpAll()
 }
 
-func (f *TxFactory) IndexSizes() (index2bytes map[string]int64, err error) {
-	// Open storage directory.
-	index2bytes = make(map[string]int64)
+func (f *TxFactory) IndexUsageDetails() (map[string]IndexUsage, error) {
+	indexUsage := make(map[string]IndexUsage)
 	dirName, err := expandDirName(f.holder.path)
 	if err != nil {
-		return index2bytes, errors.Wrap(err, "expanding data directory")
+		return indexUsage, errors.Wrap(err, "expanding data directory")
 	}
 
 	idxs := f.holder.Indexes()
 
+	/*
+		qcx := f.NewQcx()
+		tx, finisher, err := qcx.GetTx(Txo{Write: !writable})
+		if err != nil {
+			return indexUsage, errors.Wrap(err, "qcx.GetTx")
+		}
+		defer finisher(nil)
+	*/
+
 	for _, idx := range idxs {
 		index := idx.name
-		fullName := path.Join(dirName, index)
-		roaringAndMeta, err := directoryUsage(fullName)
-		if err != nil {
-			return index2bytes, errors.Wrap(err, "getting disk usage for roaring and meta")
+		println(" i:" + index)
+		indexPath := path.Join(dirName, index)
+
+		// field usage
+		fieldUsages := make(map[string]FieldUsage)
+		fragmentsTotal := uint64(0)
+		fieldKeysTotal := uint64(0)
+		flds := idx.Fields()
+		for _, fld := range flds {
+			field := fld.Name()
+			fUsage, err := f.FieldUsage(indexPath, fld)
+			if err != nil {
+				return indexUsage, errors.Wrapf(err, "getting disk usage for index (%s)", index)
+			}
+			fieldUsages[field] = fUsage
+			keysBytes := fieldUsages[field].Keys
+			fieldKeysTotal += keysBytes
+			fragmentsTotal += fieldUsages[field].Fragments
+
+			// non-roaring field usage
+			/*
+				fieldBytes, err := tx.GetFieldSizeBytes(index, field)
+				if err != nil {
+					return indexUsage, errors.Wrapf(err, "getting disk usage for non-roaring fragments (%s)", field)
+				}
+				fieldUsages[field] = FieldUsage{
+					Total:     fieldBytes,
+					Fragments: fieldBytes - keysBytes,
+					Keys:      keysBytes,
+				}
+			*/
 		}
-		fullName += ".index.txstores@@@"
-		rbfOrLmdb, err := directoryUsage(fullName)
-		if err != nil {
-			return index2bytes, errors.Wrap(err, "getting disk usage for backend")
+
+		// index keys usage
+		keysBytes := uint64(0)
+		if idx.keys {
+			keysPath := path.Join(indexPath, translateStoreDir)
+			keysBytes, err = directoryUsage(keysPath)
+			if err != nil {
+				return indexUsage, errors.Wrapf(err, "getting disk usage for index keys (%s)", index)
+			}
+		}
+
+		indexUsage[index] = IndexUsage{
+			Total:          keysBytes + fieldKeysTotal + fragmentsTotal,
+			IndexKeys:      keysBytes,
+			FieldKeysTotal: fieldKeysTotal,
+			Fragments:      fragmentsTotal,
+			Fields:         fieldUsages,
 		}
-		index2bytes[index] = roaringAndMeta + rbfOrLmdb
 	}
 
-	return index2bytes, nil
+	return indexUsage, nil
 }
 
-func directoryUsage(fname string) (int64, error) {
-	if !DirExists(fname) {
-		return 0, nil
+func (f *TxFactory) FieldUsage(indexPath string, fld *Field) (FieldUsage, error) {
+	fieldUsage := FieldUsage{}
+
+	field := fld.name
+	println("  f:" + field)
+
+	// roaring field usage
+	fieldPath := path.Join(indexPath, field)
+	fieldBytes, err := directoryUsage(fieldPath)
+	if err != nil {
+		return fieldUsage, errors.Wrapf(err, "getting disk usage for field (%s)", field)
+	}
+	keysBytes := int64(0)
+	if fld.usesKeys {
+		keysBytes, err = fileSize(fld.TranslateStorePath())
+		if err != nil {
+			return fieldUsage, errors.Wrapf(err, "getting disk usage for field keys (%s)", field)
+		}
+	}
+	fieldUsage = FieldUsage{
+		Total:     fieldBytes,
+		Fragments: fieldBytes - uint64(keysBytes),
+		Keys:      uint64(keysBytes),
 	}
 
-	var size int64
+	return fieldUsage, nil
+}
+
+func directoryUsage(fname string) (uint64, error) {
+	if !DirExists(fname) {
+		return 0, errors.Errorf("directory does not exist (%s)", fname)
+	}
+
+	var size uint64
 
 	dir, err := os.Open(fname)
 	if err != nil {
@@ -647,7 +722,7 @@ func directoryUsage(fname string) (int64, error) {
 			}
 			size += sz
 		} else {
-			size += file.Size()
+			size += uint64(file.Size()) // NOTE this cast is safe for regular files, not necessarily others
 		}
 	}