mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 17:15:56 +00:00
reesporte
GitHub
commit
da3ce449ba
19 changed files with 33 additions and 1281 deletions
291
api.go
291
api.go
|
|
@ -50,7 +50,6 @@ type API struct {
|
|||
importWorkerPoolSize int
|
||||
importWork chan importJob
|
||||
|
||||
usageCache *usageCache
|
||||
schemaDetailsOn bool
|
||||
|
||||
Serializer Serializer
|
||||
|
|
@ -938,260 +937,6 @@ func (api *API) PrimaryNode() *topology.Node {
|
|||
return snap.PrimaryFieldTranslationNode()
|
||||
}
|
||||
|
||||
// Cache of disk usage statistics
|
||||
type usageCache struct {
|
||||
data map[string]NodeUsage
|
||||
refreshInterval time.Duration
|
||||
lastUpdated time.Time
|
||||
resetTrigger chan bool
|
||||
lastCalcDuration time.Duration
|
||||
waitMultiplier float64
|
||||
disable bool
|
||||
|
||||
muCalculate sync.Mutex
|
||||
muAssign sync.Mutex
|
||||
}
|
||||
|
||||
var usageCacheMinDuration = 5 * time.Second // If usage takes less than this duration to calculate, don't use the cache.
|
||||
var usageCacheMinInterval = time.Hour // Refresh interval is forced to be >= this duration.
|
||||
var usageCacheInitialInterval = time.Hour // Refresh interval starts with this duration.
|
||||
|
||||
// NodeUsage represents all usage measurements for one node.
|
||||
type NodeUsage struct {
|
||||
Disk DiskUsage `json:"diskUsage"`
|
||||
Memory MemoryUsage `json:"memoryUsage"`
|
||||
LastUpdated time.Time `json:"lastUpdated"`
|
||||
}
|
||||
|
||||
// DiskUsage represents the storage space used on disk by one node.
|
||||
type DiskUsage struct {
|
||||
Capacity uint64 `json:"capacity,omitempty"`
|
||||
TotalUse uint64 `json:"totalInUse"`
|
||||
IndexUsage map[string]IndexUsage `json:"indexes"`
|
||||
}
|
||||
|
||||
// IndexUsage represents the storage space used on disk by one index, on one node.
|
||||
type IndexUsage struct {
|
||||
Total uint64 `json:"total"`
|
||||
IndexKeys uint64 `json:"indexKeys"`
|
||||
FieldKeysTotal uint64 `json:"fieldKeysTotal"`
|
||||
Fragments uint64 `json:"fragments"`
|
||||
Metadata uint64 `json:"metadata"`
|
||||
Fields map[string]FieldUsage `json:"fields"`
|
||||
}
|
||||
|
||||
// FieldUsage represents the storage space used on disk by one field, on one node
|
||||
type FieldUsage struct {
|
||||
Total uint64 `json:"total"`
|
||||
Fragments uint64 `json:"fragments"`
|
||||
Keys uint64 `json:"keys"`
|
||||
Metadata uint64 `json:"metadata"`
|
||||
}
|
||||
|
||||
// MemoryUsage represents the memory used by one node.
|
||||
type MemoryUsage struct {
|
||||
Capacity uint64 `json:"capacity"`
|
||||
TotalUse uint64 `json:"totalInUse"`
|
||||
}
|
||||
|
||||
// Returns disk usage from cache if cache is large. It will recalculate on the spot if the last cacluation was under 5 seconds.
|
||||
func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, error) {
|
||||
span, _ := tracing.StartSpanFromContext(ctx, "API.Usage")
|
||||
defer span.Finish()
|
||||
|
||||
if api.usageCache.disable {
|
||||
resp := make(map[string]NodeUsage)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
api.usageCache.muAssign.Lock()
|
||||
lastCalc := api.usageCache.lastCalcDuration
|
||||
api.usageCache.muAssign.Unlock()
|
||||
if lastCalc < usageCacheMinDuration {
|
||||
err := api.ResetUsageCache()
|
||||
if err != nil {
|
||||
api.server.logger.Infof("could not reset usageCache: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
api.usageCache.muAssign.Lock()
|
||||
lastUpdated := api.usageCache.lastUpdated
|
||||
api.usageCache.muAssign.Unlock()
|
||||
if lastUpdated == (time.Time{}) {
|
||||
api.calculateUsage()
|
||||
}
|
||||
|
||||
if !remote {
|
||||
api.requestUsageOfNodes()
|
||||
}
|
||||
|
||||
return api.usageCache.data, nil
|
||||
}
|
||||
|
||||
// Makes a ui/usage request for each node in cluster to calculates its usage and adds it to the cache
|
||||
func (api *API) requestUsageOfNodes() {
|
||||
nodes := api.cluster.Nodes()
|
||||
for _, node := range nodes {
|
||||
if node.ID == api.server.nodeID {
|
||||
continue
|
||||
}
|
||||
|
||||
nodeUsage, err := api.server.defaultClient.GetNodeUsage(context.Background(), &node.URI)
|
||||
if err != nil {
|
||||
api.server.logger.Infof("couldn't collect disk usage from %s: %s", node.URI, err)
|
||||
}
|
||||
|
||||
api.usageCache.muAssign.Lock()
|
||||
api.usageCache.data[node.ID] = nodeUsage[node.ID]
|
||||
api.usageCache.muAssign.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// Calculates disk usage from scratch if cache has expired for each index and stores the results in the usage cache
|
||||
func (api *API) calculateUsage() {
|
||||
// don't need to calculateUsage if we're about to close!
|
||||
if api.isClosing() {
|
||||
return
|
||||
}
|
||||
|
||||
api.usageCache.muCalculate.Lock()
|
||||
defer api.usageCache.muCalculate.Unlock()
|
||||
if ok := api.server.addToWaitGroup(1); !ok {
|
||||
// the server is closing, so just stop!
|
||||
return
|
||||
}
|
||||
defer api.server.wg.Done()
|
||||
|
||||
api.usageCache.muAssign.Lock()
|
||||
lastUpdated := api.usageCache.lastUpdated
|
||||
api.usageCache.muAssign.Unlock()
|
||||
|
||||
if time.Since(lastUpdated) <= api.usageCache.refreshInterval {
|
||||
return
|
||||
}
|
||||
indexDetails, nodeMetadataBytes, err := api.holder.Txf().IndexUsageDetails(api.isClosing)
|
||||
if err != nil {
|
||||
api.server.logger.Infof("couldn't get index usage details: %s", err)
|
||||
}
|
||||
totalSize := nodeMetadataBytes
|
||||
for _, s := range indexDetails {
|
||||
totalSize += s.Total
|
||||
}
|
||||
|
||||
// NOTE: these errors are ignored in api.Info(), but checked here
|
||||
si := api.server.systemInfo
|
||||
diskCapacity, err := si.DiskCapacity(api.holder.path)
|
||||
if err != nil {
|
||||
api.server.logger.Infof("couldn't read disk capacity: %s", err)
|
||||
}
|
||||
|
||||
memoryCapacity, err := si.MemTotal()
|
||||
if err != nil {
|
||||
api.server.logger.Infof("couldn't read memory capacity: %s", err)
|
||||
}
|
||||
memoryUse, err := si.MemUsed()
|
||||
if err != nil {
|
||||
api.server.logger.Infof("couldn't read memory usage: %s", err)
|
||||
}
|
||||
|
||||
lastUpdated = time.Now()
|
||||
// Insert into result.
|
||||
nodeUsage := NodeUsage{
|
||||
Disk: DiskUsage{
|
||||
Capacity: diskCapacity,
|
||||
TotalUse: totalSize,
|
||||
IndexUsage: indexDetails,
|
||||
},
|
||||
Memory: MemoryUsage{
|
||||
Capacity: memoryCapacity,
|
||||
TotalUse: memoryUse,
|
||||
},
|
||||
LastUpdated: lastUpdated,
|
||||
}
|
||||
api.usageCache.muAssign.Lock()
|
||||
api.usageCache.data = make(map[string]NodeUsage)
|
||||
api.usageCache.data[api.server.nodeID] = nodeUsage
|
||||
api.usageCache.lastUpdated = lastUpdated
|
||||
api.usageCache.muAssign.Unlock()
|
||||
}
|
||||
|
||||
// Periodically calculates disk/memory usage in terms of the duty cycle. The duty cycle represents the percentage of
|
||||
// time that is spent recalculating this cache. It is specified relatively, rather than by a set interval, because
|
||||
// scans can take an unpredictably long time.
|
||||
func (api *API) RefreshUsageCache(dutyCycle float64) {
|
||||
|
||||
if dutyCycle == 0 {
|
||||
api.server.logger.Warnf("usage-duty-cycle set to 0, usage cache and /ui/usage endpoint are disabled")
|
||||
api.usageCache = &usageCache{
|
||||
disable: true,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
trigger := make(chan bool)
|
||||
defer close(trigger)
|
||||
|
||||
multiplier := 100/dutyCycle - 1
|
||||
|
||||
api.usageCache = &usageCache{
|
||||
data: make(map[string]NodeUsage),
|
||||
refreshInterval: usageCacheInitialInterval,
|
||||
resetTrigger: trigger,
|
||||
lastCalcDuration: 0,
|
||||
waitMultiplier: multiplier,
|
||||
}
|
||||
api.server.logger.Infof("monitoring resource usage with duty cycle %v%%\n", dutyCycle)
|
||||
for {
|
||||
start := time.Now()
|
||||
api.calculateUsage()
|
||||
api.setRefreshInterval(time.Since(start))
|
||||
api.server.logger.Infof("updated resource usage cache at %v, took %v, next update in %v\n", api.usageCache.lastUpdated.Format(time.RFC3339), api.usageCache.lastCalcDuration.Truncate(time.Millisecond), api.usageCache.refreshInterval.Truncate(100*time.Millisecond))
|
||||
select {
|
||||
case <-trigger:
|
||||
continue
|
||||
case <-api.server.closing:
|
||||
return
|
||||
case <-time.After(api.usageCache.refreshInterval):
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh interval set in relation to how long the last calculation took.
|
||||
func (api *API) setRefreshInterval(dur time.Duration) {
|
||||
refresh := time.Duration(float64(dur) * api.usageCache.waitMultiplier)
|
||||
if refresh < usageCacheMinInterval {
|
||||
refresh = usageCacheMinInterval
|
||||
}
|
||||
api.usageCache.muAssign.Lock()
|
||||
api.usageCache.refreshInterval = refresh
|
||||
api.usageCache.lastCalcDuration = dur
|
||||
api.usageCache.muAssign.Unlock()
|
||||
}
|
||||
|
||||
// Resets the lastUpdated time and awakens RefreshUsageCache()
|
||||
func (api *API) ResetUsageCache() error {
|
||||
if api.usageCache != nil {
|
||||
api.usageCache.muAssign.Lock()
|
||||
api.usageCache.lastUpdated = time.Time{}
|
||||
api.usageCache.muAssign.Unlock()
|
||||
} else {
|
||||
return errors.New("invalidating cache: cache not initialized")
|
||||
}
|
||||
api.usageCache.resetTrigger <- true
|
||||
return nil
|
||||
}
|
||||
|
||||
// isClosing returns true if the server is shutting down.
|
||||
func (api *API) isClosing() bool {
|
||||
select {
|
||||
case <-api.server.closing:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// RecalculateCaches forces all TopN caches to be updated.
|
||||
// This is done internally within a TopN query, but a user may want to do it ahead of time?
|
||||
func (api *API) RecalculateCaches(ctx context.Context) error {
|
||||
|
|
@ -3256,24 +3001,24 @@ var methodsResizing = map[apiMethod]struct{}{
|
|||
apiSchema: {},
|
||||
}
|
||||
|
||||
var methodsDegraded = map[apiMethod]struct{}{
|
||||
apiExportCSV: {},
|
||||
apiFragmentBlockData: {},
|
||||
apiFragmentBlocks: {},
|
||||
apiField: {},
|
||||
apiIndex: {},
|
||||
apiQuery: {},
|
||||
apiRecalculateCaches: {},
|
||||
apiRemoveNode: {},
|
||||
apiShardNodes: {},
|
||||
apiSchema: {},
|
||||
apiViews: {},
|
||||
apiStartTransaction: {},
|
||||
apiFinishTransaction: {},
|
||||
apiTransactions: {},
|
||||
apiGetTransaction: {},
|
||||
apiActiveQueries: {},
|
||||
}
|
||||
// var methodsDegraded = map[apiMethod]struct{}{
|
||||
// apiExportCSV: {},
|
||||
// apiFragmentBlockData: {},
|
||||
// apiFragmentBlocks: {},
|
||||
// apiField: {},
|
||||
// apiIndex: {},
|
||||
// apiQuery: {},
|
||||
// apiRecalculateCaches: {},
|
||||
// apiRemoveNode: {},
|
||||
// apiShardNodes: {},
|
||||
// apiSchema: {},
|
||||
// apiViews: {},
|
||||
// apiStartTransaction: {},
|
||||
// apiFinishTransaction: {},
|
||||
// apiTransactions: {},
|
||||
// apiGetTransaction: {},
|
||||
// apiActiveQueries: {},
|
||||
// }
|
||||
|
||||
var methodsNormal = map[apiMethod]struct{}{
|
||||
apiCreateField: {},
|
||||
|
|
|
|||
|
|
@ -90,9 +90,6 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
|
|||
flags.Uint16Var(&srv.Config.Postgres.ConnectionLimit, "postgres.connection-limit", srv.Config.Postgres.ConnectionLimit, "Maximum number of simultaneous postgres connections to allow. (set 0 to disable)")
|
||||
flags.Uint16Var(&srv.Config.Postgres.SqlVersion, "postgres.sql-version", srv.Config.Postgres.SqlVersion, "Molecula Sql Handling Version (default 1)")
|
||||
|
||||
// Disk and Memory usage cache for ui/usage endpoint
|
||||
flags.Float64Var(&srv.Config.UsageDutyCycle, "usage-duty-cycle", srv.Config.UsageDutyCycle, "Sets the percentage of time that is spent recalculating the disk and memory usage cache. 100.0 for always-running, 0 disables the cache and the /ui/usage endpoint.")
|
||||
|
||||
// Future flags.
|
||||
flags.BoolVar(&srv.Config.Future.Rename, "future.rename", false, "Present application name as FeatureBase. Defaults to false, will default to true in an upcoming release.")
|
||||
|
||||
|
|
|
|||
|
|
@ -452,7 +452,6 @@ func newRouter(handler *Handler) http.Handler {
|
|||
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion")
|
||||
|
||||
// /ui endpoints are for UI use; they may change at any time.
|
||||
router.HandleFunc("/ui/usage", handler.chkAuthZ(handler.handleGetUsage, authz.Read)).Methods("GET").Name("GetUsage")
|
||||
router.HandleFunc("/ui/transaction", handler.chkAuthZ(handler.handleGetTransactionList, authz.Read)).Methods("GET").Name("GetTransactionList")
|
||||
router.HandleFunc("/ui/transaction/", handler.chkAuthZ(handler.handleGetTransactionList, authz.Read)).Methods("GET").Name("GetTransactionList")
|
||||
router.HandleFunc("/ui/shard-distribution", handler.chkAuthZ(handler.handleGetShardDistribution, authz.Admin)).Methods("GET").Name("GetShardDistribution")
|
||||
|
|
@ -987,63 +986,6 @@ func (h *Handler) handlePostSchema(w http.ResponseWriter, r *http.Request) {
|
|||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handleGetUsage handles GET /ui/usage requests.
|
||||
func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) {
|
||||
if !validHeaderAcceptJSON(r.Header) {
|
||||
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
|
||||
return
|
||||
}
|
||||
|
||||
q := r.URL.Query()
|
||||
remoteStr := q.Get("remote")
|
||||
var remote bool
|
||||
if remoteStr == "true" {
|
||||
remote = true
|
||||
}
|
||||
|
||||
nodeUsages, err := h.api.Usage(r.Context(), remote)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
// if auth is turned on, filter results
|
||||
if h.auth != nil {
|
||||
g := r.Context().Value(contextKeyGroupMembership)
|
||||
if g == nil {
|
||||
http.Error(w, "Forbidden", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if !h.permissions.IsAdmin(g.([]authn.Group)) {
|
||||
allowed := h.permissions.GetAuthorizedIndexList(g.([]authn.Group), authz.Read)
|
||||
filteredNodeUsages := map[string]NodeUsage{}
|
||||
|
||||
for nodeId, nodeUsage := range nodeUsages {
|
||||
filteredIndexUsage := NodeUsage{
|
||||
Disk: DiskUsage{
|
||||
IndexUsage: map[string]IndexUsage{},
|
||||
},
|
||||
}
|
||||
for index, idxUsage := range nodeUsage.Disk.IndexUsage {
|
||||
// is it in auth list
|
||||
for _, authd := range allowed {
|
||||
if index == authd {
|
||||
filteredIndexUsage.Disk.IndexUsage[index] = idxUsage
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
filteredNodeUsages[nodeId] = filteredIndexUsage
|
||||
}
|
||||
nodeUsages = filteredNodeUsages
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(nodeUsages); err != nil {
|
||||
h.logger.Errorf("write status response error: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// handleGetShardDistribution handles GET /ui/shard-distribution requests.
|
||||
func (h *Handler) handleGetShardDistribution(w http.ResponseWriter, r *http.Request) {
|
||||
dist := h.api.ShardDistribution(r.Context())
|
||||
|
|
|
|||
|
|
@ -244,28 +244,6 @@ log-path = "/var/log/molecula/featurebase.log"
|
|||
# enable-client-verification = true
|
||||
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Usage Duty Cycle - Featurebase maintains a disk/memory usage cache that is
|
||||
# calculated periodically in the background and accessed by the UI/usage
|
||||
# endpoint. Since this disk scan can take a long and unpredictable amount of
|
||||
# time, its timing behavior is specified in a relative, rather than absolute
|
||||
# sense. That is, the duty cycle sets the percentage of time that is spent
|
||||
# recalculating this cache. This setting affects the results received from
|
||||
# the "/ui/usage" http endpoint, as well as all data file and memory usage
|
||||
# values and graphs on the webui "tables" page
|
||||
|
||||
# Special considerations:
|
||||
# * If disk usage can be calculated quickly (less than 5 seconds), fresh
|
||||
# results will be calculated when accessed
|
||||
# * When disk usage takes longer to calculate, there is a minimum of one
|
||||
# hour wait between cache recalculations
|
||||
# Setting this value to 0 will completely disable the calculation of disk usage
|
||||
#
|
||||
# usage-duty-cycle = 20
|
||||
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Use [metric] stanza to define attributes for monitoring.
|
||||
# [metric]
|
||||
|
|
|
|||
22
internal/clustertests/testdata/featurebase.conf
vendored
22
internal/clustertests/testdata/featurebase.conf
vendored
|
|
@ -244,28 +244,6 @@
|
|||
# enable-client-verification = true
|
||||
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Usage Duty Cycle - Featurebase maintains a disk/memory usage cache that is
|
||||
# calculated periodically in the background and accessed by the UI/usage
|
||||
# endpoint. Since this disk scan can take a long and unpredictable amount of
|
||||
# time, its timing behavior is specified in a relative, rather than absolute
|
||||
# sense. That is, the duty cycle sets the percentage of time that is spent
|
||||
# recalculating this cache. This setting affects the results received from
|
||||
# the "/ui/usage" http endpoint, as well as all data file and memory usage
|
||||
# values and graphs on the webui "tables" page
|
||||
|
||||
# Special considerations:
|
||||
# * If disk usage can be calculated quickly (less than 5 seconds), fresh
|
||||
# results will be calculated when accessed
|
||||
# * When disk usage takes longer to calculate, there is a minimum of one
|
||||
# hour wait between cache recalculations
|
||||
# Setting this value to 0 will completely disable the calculation of disk usage
|
||||
#
|
||||
# usage-duty-cycle = 20
|
||||
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Use [metric] stanza to define attributes for monitoring.
|
||||
# [metric]
|
||||
|
|
|
|||
|
|
@ -1383,38 +1383,6 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in
|
|||
return tkresp.Keys, nil
|
||||
}
|
||||
|
||||
// GetNodeUsage retrieves the size-on-disk information for the specified node.
|
||||
func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) {
|
||||
u := uri.Path("/ui/usage?remote=true")
|
||||
req, err := http.NewRequest("GET", u, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "creating request")
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("User-Agent", "pilosa/"+Version)
|
||||
req = AddAuthToken(ctx, req)
|
||||
|
||||
// Execute request against the host.
|
||||
resp, err := c.executeRequest(req.WithContext(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read body and unmarshal response.
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "reading")
|
||||
}
|
||||
|
||||
nodeUsages := make(map[string]NodeUsage) // map of size 1
|
||||
if err := json.Unmarshal(body, &nodeUsages); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal response: %s", err)
|
||||
}
|
||||
return nodeUsages, nil
|
||||
}
|
||||
|
||||
// GetPastQueries retrieves the query history log for the specified node.
|
||||
func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) {
|
||||
u := uri.Path("/query-history?remote=true")
|
||||
|
|
|
|||
|
|
@ -19,14 +19,12 @@ export const ClusterHealth: FC = () => {
|
|||
const [cluster, setCluster] = useState<any>();
|
||||
const [metrics, setMetrics] = useState<any>();
|
||||
const [info, setInfo] = useState<any>();
|
||||
const [clusterData, setClusterData] = useState<any>();
|
||||
const [expanded, setExpanded] = useState<string[]>([]);
|
||||
const [showMetrics, setShowMetrics] = useState<any>();
|
||||
const allExpanded = cluster && expanded.length === cluster.nodes.length;
|
||||
|
||||
useEffectOnce(() => {
|
||||
getClusterHealth();
|
||||
getClusterData();
|
||||
});
|
||||
|
||||
const refreshMetrics = useCallback(() => {
|
||||
|
|
@ -38,15 +36,11 @@ export const ClusterHealth: FC = () => {
|
|||
|
||||
useEffect(() => {
|
||||
const interval = setInterval(() => {
|
||||
if (!clusterData) {
|
||||
getClusterData();
|
||||
}
|
||||
|
||||
getClusterHealth();
|
||||
refreshMetrics();
|
||||
}, 15000);
|
||||
return () => clearInterval(interval);
|
||||
}, [refreshMetrics, cluster, clusterData]);
|
||||
}, [refreshMetrics, cluster]);
|
||||
|
||||
const getClusterHealth = () => {
|
||||
pilosa.get
|
||||
|
|
@ -76,13 +70,6 @@ export const ClusterHealth: FC = () => {
|
|||
.catch(() => setMetrics(undefined));
|
||||
};
|
||||
|
||||
const getClusterData = () => {
|
||||
pilosa.get
|
||||
.usage()
|
||||
.then((res) => setClusterData(res.data))
|
||||
.catch(() => setClusterData(undefined));
|
||||
};
|
||||
|
||||
const toggleAccordion = (nodeId: string) => {
|
||||
const isExpanded = expanded.includes(nodeId);
|
||||
if (isExpanded) {
|
||||
|
|
@ -140,7 +127,6 @@ export const ClusterHealth: FC = () => {
|
|||
key={node.id}
|
||||
node={node}
|
||||
info={info}
|
||||
usage={clusterData ? clusterData[node.id] : undefined}
|
||||
expanded={expanded.includes(node.id)}
|
||||
onToggle={() => toggleAccordion(node.id)}
|
||||
onMetricClick={() => setShowMetrics(node)}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import React, { FC, Fragment, useState } from 'react';
|
||||
import React, { FC, useState } from 'react';
|
||||
import Button from '@material-ui/core/Button';
|
||||
import copy from 'copy-to-clipboard';
|
||||
import EqualizerIcon from '@material-ui/icons/EqualizerSharp';
|
||||
|
|
@ -11,7 +11,6 @@ import Find from 'lodash/find';
|
|||
import IconButton from '@material-ui/core/IconButton';
|
||||
import InfoIcon from '@material-ui/icons/Info';
|
||||
import Tooltip from '@material-ui/core/Tooltip';
|
||||
import Typography from '@material-ui/core/Typography';
|
||||
import { formatBytes } from 'shared/utils/formatBytes';
|
||||
import { nodeInfo } from './nodeInfo';
|
||||
import { NODE_STATE } from './nodeStatus';
|
||||
|
|
@ -21,7 +20,6 @@ import css from './Node.module.scss';
|
|||
type NodeType = {
|
||||
node: any;
|
||||
info: any;
|
||||
usage: any;
|
||||
expanded: boolean;
|
||||
onToggle: () => void;
|
||||
onMetricClick: () => void;
|
||||
|
|
@ -30,24 +28,13 @@ type NodeType = {
|
|||
export const Node: FC<NodeType> = ({
|
||||
node,
|
||||
info,
|
||||
usage,
|
||||
expanded,
|
||||
onToggle,
|
||||
onMetricClick
|
||||
onMetricClick,
|
||||
}) => {
|
||||
const [copyHost, setCopyHost] = useState<string>('Copy Host');
|
||||
const [copyID, setCopyID] = useState<string>('Click to Copy');
|
||||
const { id, isPrimary, state } = node;
|
||||
const diskTotalInUse = usage?.diskUsage?.totalInUse;
|
||||
const diskCapacity = usage?.diskUsage?.capacity;
|
||||
const diskUsagePercentage = diskCapacity
|
||||
? (diskTotalInUse / diskCapacity) * 100
|
||||
: undefined;
|
||||
const memoryTotalInUse = usage?.memoryUsage?.totalInUse;
|
||||
const memoryCapacity = usage?.memoryUsage?.capacity;
|
||||
const memoryUsagePercentage = memoryCapacity
|
||||
? (memoryTotalInUse / memoryCapacity) * 100
|
||||
: undefined;
|
||||
const keys = Object.keys(info);
|
||||
|
||||
const onCopyHostClick = () => {
|
||||
|
|
@ -103,154 +90,6 @@ export const Node: FC<NodeType> = ({
|
|||
</span>
|
||||
</Tooltip>
|
||||
</div>
|
||||
<div className={css.nodeUsage}>
|
||||
<div>
|
||||
<div className={css.label}>Disk Usage:</div>
|
||||
<div>
|
||||
{usage ? (
|
||||
<Fragment>
|
||||
<Typography variant="caption">
|
||||
{formatBytes(diskTotalInUse)}
|
||||
{diskCapacity
|
||||
? ` used out of ${formatBytes(diskCapacity)}`
|
||||
: null}
|
||||
</Typography>
|
||||
<div className={css.totalCapacity}>
|
||||
{diskUsagePercentage ? (
|
||||
<Tooltip
|
||||
title={
|
||||
<Typography variant="caption">
|
||||
{diskUsagePercentage < 1
|
||||
? '< 1'
|
||||
: diskUsagePercentage.toLocaleString(
|
||||
undefined,
|
||||
{ maximumFractionDigits: 1 }
|
||||
)}
|
||||
% used
|
||||
</Typography>
|
||||
}
|
||||
placement="top"
|
||||
arrow
|
||||
>
|
||||
<div
|
||||
className={css.totalInUse}
|
||||
style={{
|
||||
width: `${
|
||||
diskUsagePercentage < 1
|
||||
? 1
|
||||
: diskUsagePercentage
|
||||
}%`
|
||||
}}
|
||||
/>
|
||||
</Tooltip>
|
||||
) : (
|
||||
<Fragment>
|
||||
<Tooltip
|
||||
title={
|
||||
<Typography variant="caption">
|
||||
{formatBytes(diskTotalInUse)} used
|
||||
</Typography>
|
||||
}
|
||||
placement="top"
|
||||
arrow
|
||||
>
|
||||
<div
|
||||
className={css.totalInUse}
|
||||
style={{ width: '2%' }}
|
||||
/>
|
||||
</Tooltip>
|
||||
<Typography
|
||||
className={css.unknownCapacity}
|
||||
variant="caption"
|
||||
color="textSecondary"
|
||||
>
|
||||
Node disk capacity unknown
|
||||
</Typography>
|
||||
</Fragment>
|
||||
)}
|
||||
</div>
|
||||
</Fragment>
|
||||
) : (
|
||||
<Typography variant="caption" paragraph>
|
||||
Calculating...
|
||||
</Typography>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className={css.label}>Memory Usage:</div>
|
||||
<div>
|
||||
{usage ? (
|
||||
<Fragment>
|
||||
<Typography variant="caption">
|
||||
{formatBytes(memoryTotalInUse)}
|
||||
{memoryCapacity
|
||||
? ` used out of ${formatBytes(memoryCapacity)}`
|
||||
: null}
|
||||
</Typography>
|
||||
<div className={css.totalCapacity}>
|
||||
{memoryUsagePercentage ? (
|
||||
<Tooltip
|
||||
title={
|
||||
<Typography variant="caption">
|
||||
{memoryUsagePercentage < 1
|
||||
? '< 1'
|
||||
: memoryUsagePercentage.toLocaleString(
|
||||
undefined,
|
||||
{ maximumFractionDigits: 1 }
|
||||
)}
|
||||
% used
|
||||
</Typography>
|
||||
}
|
||||
placement="top"
|
||||
arrow
|
||||
>
|
||||
<div
|
||||
className={css.totalInUse}
|
||||
style={{
|
||||
width: `${
|
||||
memoryUsagePercentage < 1
|
||||
? 1
|
||||
: memoryUsagePercentage
|
||||
}%`
|
||||
}}
|
||||
/>
|
||||
</Tooltip>
|
||||
) : (
|
||||
<Fragment>
|
||||
<Tooltip
|
||||
title={
|
||||
<Typography variant="caption">
|
||||
{formatBytes(memoryTotalInUse)} used
|
||||
</Typography>
|
||||
}
|
||||
placement="top"
|
||||
arrow
|
||||
>
|
||||
<div
|
||||
className={css.totalInUse}
|
||||
style={{ width: '2%' }}
|
||||
/>
|
||||
</Tooltip>
|
||||
<Typography
|
||||
className={css.unknownCapacity}
|
||||
variant="caption"
|
||||
color="textSecondary"
|
||||
>
|
||||
Node memory capacity unknown
|
||||
</Typography>
|
||||
</Fragment>
|
||||
)}
|
||||
</div>
|
||||
</Fragment>
|
||||
) : (
|
||||
<Typography variant="caption" paragraph>
|
||||
Calculating...
|
||||
</Typography>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className={css.nodeSettings}>
|
||||
{keys.map((key) => {
|
||||
const showNode = Find(nodeInfo, (node) => node.name === key);
|
||||
|
|
|
|||
|
|
@ -4,75 +4,38 @@ import Breadcrumbs from '@material-ui/core/Breadcrumbs';
|
|||
import classNames from 'classnames';
|
||||
import Fuse from 'fuse.js';
|
||||
import Highlighter from 'react-highlight-words';
|
||||
import isEmpty from 'lodash/isEmpty';
|
||||
import Link from '@material-ui/core/Link';
|
||||
import map from 'lodash/map';
|
||||
import moment from 'moment';
|
||||
import OrderBy from 'lodash/orderBy';
|
||||
import Reduce from 'lodash/reduce';
|
||||
import Table from '@material-ui/core/Table';
|
||||
import TableBody from '@material-ui/core/TableBody';
|
||||
import TableCell from '@material-ui/core/TableCell';
|
||||
import TableHead from '@material-ui/core/TableHead';
|
||||
import TableRow from '@material-ui/core/TableRow';
|
||||
import TextField from '@material-ui/core/TextField';
|
||||
import Tooltip from '@material-ui/core/Tooltip';
|
||||
import Typography from '@material-ui/core/Typography';
|
||||
import { Block } from 'shared/Block';
|
||||
import { Pager } from 'shared/Pager';
|
||||
import { UsageBreakdown } from '../UsageBreakdown';
|
||||
import css from './MoleculaTable.module.scss';
|
||||
|
||||
type MoleculaTableProps = {
|
||||
table: any;
|
||||
dataDistribution: any;
|
||||
lastUpdated: string;
|
||||
};
|
||||
|
||||
export const MoleculaTable: FC<MoleculaTableProps> = ({
|
||||
table,
|
||||
dataDistribution,
|
||||
lastUpdated
|
||||
lastUpdated,
|
||||
}) => {
|
||||
const [page, setPage] = useState<number>(1);
|
||||
const [resultsPerPage, setResultsPerPage] = useState<number>(10);
|
||||
const sliceStart = (page - 1) * resultsPerPage;
|
||||
const [searchText, setSearchText] = useState<string>('');
|
||||
const [filteredFields, setFiltereedFields] = useState(table.fields);
|
||||
const [fieldsData, setFieldsData] = useState<{}>({});
|
||||
const [maxFieldSize, setMaxFieldSize] = useState<number>(0);
|
||||
const [fieldsData] = useState<{}>({});
|
||||
const [sort, setSort] = useState<string>('total');
|
||||
const [sortDir, setSortDir] = useState<'asc' | 'desc'>('desc');
|
||||
const lastUpdatedMoment = lastUpdated ? moment(lastUpdated).utc() : undefined;
|
||||
|
||||
useEffect(() => {
|
||||
if (dataDistribution && !dataDistribution.uncached) {
|
||||
const aggregatedFieldsData = Reduce(
|
||||
dataDistribution.fields,
|
||||
(result, value) => {
|
||||
let newResult = {};
|
||||
const keys = Object.keys(value);
|
||||
keys.forEach(
|
||||
(key) =>
|
||||
(newResult[key] = {
|
||||
total: result[key].total + value[key].total,
|
||||
fragments: result[key].fragments + value[key].fragments,
|
||||
keys: result[key].keys + value[key].keys,
|
||||
metadata: result[key].metadata + value[key].metadata
|
||||
})
|
||||
);
|
||||
return newResult;
|
||||
}
|
||||
);
|
||||
|
||||
const sorted = OrderBy(aggregatedFieldsData, ['total'], ['desc']);
|
||||
if (sorted.length > 0) {
|
||||
setMaxFieldSize(sorted[0].total);
|
||||
}
|
||||
|
||||
setFieldsData(aggregatedFieldsData);
|
||||
}
|
||||
}, [dataDistribution]);
|
||||
|
||||
useEffect(() => {
|
||||
if (searchText.length > 1) {
|
||||
|
|
@ -80,7 +43,7 @@ export const MoleculaTable: FC<MoleculaTableProps> = ({
|
|||
keys: ['name'],
|
||||
minMatchCharLength: 2,
|
||||
ignoreLocation: true,
|
||||
threshold: 0
|
||||
threshold: 0,
|
||||
});
|
||||
const result = fuse.search(searchText);
|
||||
|
||||
|
|
@ -131,46 +94,6 @@ export const MoleculaTable: FC<MoleculaTableProps> = ({
|
|||
<Typography variant="h5" color="textSecondary">
|
||||
{table.name}
|
||||
</Typography>
|
||||
{lastUpdatedMoment ? (
|
||||
<div className={css.infoMessage}>
|
||||
{dataDistribution && dataDistribution.uncached ? (
|
||||
<Fragment>
|
||||
Disk usage will be calculated at the next{` `}
|
||||
<Tooltip
|
||||
title={
|
||||
<Fragment>
|
||||
Disk and memory information shown here are read from a
|
||||
cache, the behavior of which can be controlled with the{` `}
|
||||
<code style={{ whiteSpace: 'nowrap' }}>
|
||||
--usage-duty-cycle
|
||||
</code>{' '}
|
||||
command line flag.
|
||||
</Fragment>
|
||||
}
|
||||
placement="top"
|
||||
arrow
|
||||
>
|
||||
<span className={css.infoTooltip}>cache refresh</span>
|
||||
</Tooltip>
|
||||
.
|
||||
</Fragment>
|
||||
) : (
|
||||
<Fragment>
|
||||
Disk usage last updated{' '}
|
||||
<Tooltip
|
||||
title={`${lastUpdatedMoment.format('M/D/YYYY hh:mm a')} UTC`}
|
||||
placement="top"
|
||||
arrow
|
||||
>
|
||||
<span className={css.infoTooltip}>
|
||||
{lastUpdatedMoment.fromNow()}
|
||||
</span>
|
||||
</Tooltip>
|
||||
.
|
||||
</Fragment>
|
||||
)}
|
||||
</div>
|
||||
) : null}
|
||||
<div className={css.layout}>
|
||||
<div>
|
||||
<label className={css.label}>keys</label>
|
||||
|
|
@ -180,9 +103,6 @@ export const MoleculaTable: FC<MoleculaTableProps> = ({
|
|||
</code>
|
||||
</div>
|
||||
</div>
|
||||
<div className={css.breakdown}>
|
||||
<UsageBreakdown data={dataDistribution} />
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div>
|
||||
|
|
@ -211,14 +131,14 @@ export const MoleculaTable: FC<MoleculaTableProps> = ({
|
|||
<TableCell className={css.tableHeader}>
|
||||
<span
|
||||
className={classNames(css.sortable, {
|
||||
[css.currentSort]: sort === 'name'
|
||||
[css.currentSort]: sort === 'name',
|
||||
})}
|
||||
onClick={() => onSortClick('name')}
|
||||
>
|
||||
Name{' '}
|
||||
<ArrowDropDownIcon
|
||||
className={classNames(css.sortArrow, {
|
||||
[css.asc]: sortDir === 'asc'
|
||||
[css.asc]: sortDir === 'asc',
|
||||
})}
|
||||
/>
|
||||
</span>
|
||||
|
|
@ -226,21 +146,6 @@ export const MoleculaTable: FC<MoleculaTableProps> = ({
|
|||
<TableCell className={css.tableHeader}>Type</TableCell>
|
||||
<TableCell className={css.tableHeader}>Cardinality</TableCell>
|
||||
<TableCell className={css.tableHeader}>Options</TableCell>
|
||||
<TableCell className={css.tableHeader}>
|
||||
<span
|
||||
className={classNames(css.sortable, {
|
||||
[css.currentSort]: sort === 'total'
|
||||
})}
|
||||
onClick={() => onSortClick('total')}
|
||||
>
|
||||
Disk Usage{' '}
|
||||
<ArrowDropDownIcon
|
||||
className={classNames(css.sortArrow, {
|
||||
[css.asc]: sortDir === 'asc'
|
||||
})}
|
||||
/>
|
||||
</span>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
<TableBody>
|
||||
|
|
@ -295,22 +200,6 @@ export const MoleculaTable: FC<MoleculaTableProps> = ({
|
|||
})}
|
||||
</div>
|
||||
</TableCell>
|
||||
<TableCell className={css.tableCell}>
|
||||
<UsageBreakdown
|
||||
data={
|
||||
isEmpty(field)
|
||||
? field
|
||||
: dataDistribution
|
||||
? dataDistribution.uncached
|
||||
? dataDistribution
|
||||
: field
|
||||
: field
|
||||
}
|
||||
width={`${(field.total / maxFieldSize) * 150}px`}
|
||||
showLabel={false}
|
||||
usageValueSize="small"
|
||||
/>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
);
|
||||
})}
|
||||
|
|
|
|||
|
|
@ -8,42 +8,29 @@ import Tooltip from '@material-ui/core/Tooltip';
|
|||
import Typography from '@material-ui/core/Typography';
|
||||
import { Block } from 'shared/Block';
|
||||
import { SortBy } from 'shared/SortBy';
|
||||
import { UsageBreakdown } from './UsageBreakdown';
|
||||
import { useHistory } from 'react-router-dom';
|
||||
import css from './MoleculaTables.module.scss';
|
||||
|
||||
type MoleculaTablesProps = {
|
||||
tables: any;
|
||||
dataDistribution: any;
|
||||
lastUpdated: string;
|
||||
maxSize: number;
|
||||
};
|
||||
|
||||
export const MoleculaTables: FC<MoleculaTablesProps> = ({
|
||||
tables,
|
||||
dataDistribution,
|
||||
lastUpdated,
|
||||
maxSize
|
||||
maxSize,
|
||||
}) => {
|
||||
const history = useHistory();
|
||||
const [sortedTables, setSortedTables] = useState<any>([]);
|
||||
const lastUpdatedMoment = lastUpdated ? moment(lastUpdated).utc() : undefined;
|
||||
|
||||
useEffect(() => {
|
||||
if (tables && dataDistribution) {
|
||||
let aggregatedData: any[] = [];
|
||||
tables.forEach((i) =>
|
||||
aggregatedData.push({
|
||||
...dataDistribution[i.name],
|
||||
...i
|
||||
})
|
||||
);
|
||||
|
||||
setSortedTables(aggregatedData);
|
||||
} else if (tables) {
|
||||
if (tables) {
|
||||
setSortedTables(tables);
|
||||
}
|
||||
}, [tables, dataDistribution]);
|
||||
}, [tables]);
|
||||
|
||||
const handleSortChange = (value: any) => {
|
||||
const sortDirection = value === 'name' ? 'asc' : 'desc';
|
||||
|
|
@ -96,7 +83,7 @@ export const MoleculaTables: FC<MoleculaTablesProps> = ({
|
|||
{ label: 'Index Keys Size', value: 'indexKeys' },
|
||||
{ label: 'Fragment Size', value: 'fragments' },
|
||||
{ label: 'Field Keys Size', value: 'fieldKeysTotal' },
|
||||
{ label: 'Metadata Size', value: 'metadata' }
|
||||
{ label: 'Metadata Size', value: 'metadata' },
|
||||
]}
|
||||
defaultValue="name"
|
||||
onChange={handleSortChange}
|
||||
|
|
@ -111,22 +98,6 @@ export const MoleculaTables: FC<MoleculaTablesProps> = ({
|
|||
<Card key={name} className={css.tableTile}>
|
||||
<CardContent>
|
||||
<div className={css.header}>{name}</div>
|
||||
<div className={css.section}>
|
||||
<UsageBreakdown
|
||||
data={
|
||||
dataDistribution
|
||||
? dataDistribution[name]
|
||||
? dataDistribution[name]
|
||||
: { uncached: true }
|
||||
: undefined
|
||||
}
|
||||
width={
|
||||
dataDistribution && dataDistribution[name]
|
||||
? `${(dataDistribution[name].total / maxSize) * 100}%`
|
||||
: '0px'
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
<label className={css.label}>Options</label>
|
||||
<div className={css.cell}>
|
||||
<span className={css.label}>keys</span>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import React, { useEffect, useState } from 'react';
|
||||
import OrderBy from 'lodash/orderBy';
|
||||
import { MoleculaTable } from './MoleculaTable';
|
||||
import { MoleculaTables } from './MoleculaTables';
|
||||
import { pilosa } from 'services/eventServices';
|
||||
|
|
@ -12,9 +11,8 @@ export const MoleculaTablesContainer = () => {
|
|||
const history = useHistory();
|
||||
const [tables, setTables] = useState<any>();
|
||||
const [selectedTable, setSelectedTable] = useState<any>();
|
||||
const [dataDistribution, setDataDistribution] = useState<any>();
|
||||
const [maxSize, setMaxSize] = useState<number>(0);
|
||||
const [lastUpdated, setLastUpdated] = useState<string>('');
|
||||
const [maxSize] = useState<number>(0);
|
||||
const [lastUpdated] = useState<string>('');
|
||||
|
||||
useEffectOnce(() => {
|
||||
pilosa.get
|
||||
|
|
@ -26,48 +24,6 @@ export const MoleculaTablesContainer = () => {
|
|||
.then((res) => setTables(res.data.indexes))
|
||||
.catch((err) => console.log(err))
|
||||
);
|
||||
|
||||
pilosa.get.usage().then((res) => {
|
||||
const nodes = Object.keys(res.data);
|
||||
let data = {};
|
||||
nodes.forEach((node) => {
|
||||
const nodeIndexes = res.data[node].diskUsage.indexes;
|
||||
const indexList = Object.keys(nodeIndexes);
|
||||
indexList.forEach((i) => {
|
||||
const nodeData = nodeIndexes[i];
|
||||
if (data[i]) {
|
||||
data[i] = {
|
||||
total: data[i].total + nodeData.total,
|
||||
fieldKeysTotal: data[i].fieldKeysTotal + nodeData.fieldKeysTotal,
|
||||
indexKeys: data[i].indexKeys + nodeData.indexKeys,
|
||||
fragments: data[i].fragments + nodeData.fragments,
|
||||
metadata: data[i].metadata + nodeData.metadata,
|
||||
fields: [...data[i].fields, nodeData.fields]
|
||||
};
|
||||
} else {
|
||||
data[i] = {
|
||||
total: nodeData.total,
|
||||
fieldKeysTotal: nodeData.fieldKeysTotal,
|
||||
indexKeys: nodeData.indexKeys,
|
||||
fragments: nodeData.fragments,
|
||||
metadata: nodeData.metadata,
|
||||
fields: [nodeData.fields]
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
if(!lastUpdated) {
|
||||
setLastUpdated(res.data[node].lastUpdated);
|
||||
}
|
||||
});
|
||||
|
||||
const sorted = OrderBy(data, ['total'], ['desc']);
|
||||
if (sorted.length > 0) {
|
||||
setMaxSize(sorted[0].total);
|
||||
}
|
||||
|
||||
setDataDistribution(data);
|
||||
});
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -85,21 +41,10 @@ export const MoleculaTablesContainer = () => {
|
|||
}, [match, tables, history]);
|
||||
|
||||
return selectedTable ? (
|
||||
<MoleculaTable
|
||||
table={selectedTable}
|
||||
dataDistribution={
|
||||
dataDistribution
|
||||
? dataDistribution[selectedTable.name]
|
||||
? dataDistribution[selectedTable.name]
|
||||
: { uncached: true }
|
||||
: undefined
|
||||
}
|
||||
lastUpdated={lastUpdated}
|
||||
/>
|
||||
<MoleculaTable table={selectedTable} lastUpdated={lastUpdated} />
|
||||
) : (
|
||||
<MoleculaTables
|
||||
tables={tables}
|
||||
dataDistribution={dataDistribution}
|
||||
lastUpdated={lastUpdated}
|
||||
maxSize={maxSize}
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -1,62 +0,0 @@
|
|||
.label {
|
||||
font-size: 0.75rem;
|
||||
color: var(--text-secondary);
|
||||
margin-bottom: 4px;
|
||||
font-weight: 400;
|
||||
}
|
||||
|
||||
.usageBreakdown {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
||||
.usageBreakdownLabel {
|
||||
white-space: nowrap;
|
||||
margin-right: 8px;
|
||||
|
||||
&.smallLabel {
|
||||
font-size: 12px;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.breakdown {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
height: 13px;
|
||||
border-radius: 4px;
|
||||
background: rgba(var(--contrast-rgb), 0.1);
|
||||
|
||||
.fieldKeysTotal {
|
||||
height: 13px;
|
||||
background: rgba(88, 80, 141, 0.7);
|
||||
}
|
||||
|
||||
.indexKeys {
|
||||
height: 13px;
|
||||
background: rgba(255, 99, 97, 0.7);
|
||||
}
|
||||
|
||||
.keys {
|
||||
height: 13px;
|
||||
background: rgba(88, 80, 141, 0.7);
|
||||
}
|
||||
|
||||
.fragments {
|
||||
height: 13px;
|
||||
background: rgba(255, 166, 0, 0.7);
|
||||
}
|
||||
|
||||
.metadata {
|
||||
height: 13px;
|
||||
background: rgba(188, 80, 144, 0.7);
|
||||
}
|
||||
|
||||
.bar:first-child {
|
||||
border-top-left-radius: 4px;
|
||||
border-bottom-left-radius: 4px;
|
||||
}
|
||||
.bar:last-child {
|
||||
border-top-right-radius: 4px;
|
||||
border-bottom-right-radius: 4px;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,183 +0,0 @@
|
|||
import React, { FC, Fragment } from 'react';
|
||||
import classNames from 'classnames';
|
||||
import Tooltip from '@material-ui/core/Tooltip';
|
||||
import Typography from '@material-ui/core/Typography';
|
||||
import { formatBytes } from 'shared/utils/formatBytes';
|
||||
import css from './UsageBreakdown.module.scss';
|
||||
|
||||
type UsageBreakdownProps = {
|
||||
data: any;
|
||||
width?: string;
|
||||
showLabel?: boolean;
|
||||
usageValueSize?: 'small' | 'medium';
|
||||
};
|
||||
|
||||
export const UsageBreakdown: FC<UsageBreakdownProps> = ({
|
||||
data = {},
|
||||
width,
|
||||
showLabel = true,
|
||||
usageValueSize = 'medium'
|
||||
}) => {
|
||||
const {
|
||||
total,
|
||||
fieldKeysTotal,
|
||||
indexKeys,
|
||||
fragments,
|
||||
metadata,
|
||||
keys,
|
||||
uncached
|
||||
} = data;
|
||||
const fieldKeysPercentage =
|
||||
fieldKeysTotal && total ? (fieldKeysTotal / total) * 100 : 0;
|
||||
const indexKeysPercentage = indexKeys ? (indexKeys / total) * 100 : 0;
|
||||
const fragmentsPercentage = fragments ? (fragments / total) * 100 : 0;
|
||||
const metadataPercentage = metadata ? (metadata / total) * 100 : 0;
|
||||
const keysPercentage = keys && total ? (keys / total) * 100 : 0;
|
||||
|
||||
return (
|
||||
<Fragment>
|
||||
{showLabel ? <label className={css.label}>Data</label> : null}
|
||||
<div className={css.usageBreakdown}>
|
||||
{total ? (
|
||||
<Fragment>
|
||||
<span
|
||||
className={classNames(css.usageBreakdownLabel, {
|
||||
[css.smallLabel]: usageValueSize === 'small'
|
||||
})}
|
||||
>
|
||||
{formatBytes(total)}
|
||||
</span>
|
||||
<div
|
||||
className={css.breakdown}
|
||||
style={{ width: width ? width : '100%' }}
|
||||
>
|
||||
{fieldKeysTotal ? (
|
||||
<Tooltip
|
||||
title={
|
||||
<Fragment>
|
||||
<label>Field Keys:</label>
|
||||
<Typography variant="caption" component="div">
|
||||
{formatBytes(fieldKeysTotal)} (
|
||||
{fieldKeysPercentage.toLocaleString(undefined, {
|
||||
maximumFractionDigits: 1
|
||||
})}
|
||||
%)
|
||||
</Typography>
|
||||
</Fragment>
|
||||
}
|
||||
placement="top"
|
||||
arrow
|
||||
>
|
||||
<div
|
||||
className={classNames(css.bar, css.fieldKeysTotal)}
|
||||
style={{ width: `${fieldKeysPercentage}%` }}
|
||||
/>
|
||||
</Tooltip>
|
||||
) : null}
|
||||
{indexKeys ? (
|
||||
<Tooltip
|
||||
title={
|
||||
<Fragment>
|
||||
<label>Index Keys:</label>
|
||||
<Typography variant="caption" component="div">
|
||||
{formatBytes(indexKeys)} (
|
||||
{indexKeysPercentage.toLocaleString(undefined, {
|
||||
maximumFractionDigits: 1
|
||||
})}
|
||||
%)
|
||||
</Typography>
|
||||
</Fragment>
|
||||
}
|
||||
placement="top"
|
||||
arrow
|
||||
>
|
||||
<div
|
||||
className={classNames(css.bar, css.indexKeys)}
|
||||
style={{ width: `${indexKeysPercentage}%` }}
|
||||
/>
|
||||
</Tooltip>
|
||||
) : null}
|
||||
{keys ? (
|
||||
<Tooltip
|
||||
title={
|
||||
<Fragment>
|
||||
<label>Keys:</label>
|
||||
<Typography variant="caption" component="div">
|
||||
{formatBytes(keys)} (
|
||||
{keysPercentage.toLocaleString(undefined, {
|
||||
maximumFractionDigits: 1
|
||||
})}
|
||||
%)
|
||||
</Typography>
|
||||
</Fragment>
|
||||
}
|
||||
placement="top"
|
||||
arrow
|
||||
>
|
||||
<div
|
||||
className={classNames(css.bar, css.keys)}
|
||||
style={{ width: `${keysPercentage}%` }}
|
||||
/>
|
||||
</Tooltip>
|
||||
) : null}
|
||||
{fragments ? (
|
||||
<Tooltip
|
||||
title={
|
||||
<Fragment>
|
||||
<label>Fragments:</label>
|
||||
<Typography variant="caption" component="div">
|
||||
{formatBytes(fragments)} (
|
||||
{fragmentsPercentage.toLocaleString(undefined, {
|
||||
maximumFractionDigits: 1
|
||||
})}
|
||||
%)
|
||||
</Typography>
|
||||
</Fragment>
|
||||
}
|
||||
placement="top"
|
||||
arrow
|
||||
>
|
||||
<div
|
||||
className={classNames(css.bar, css.fragments)}
|
||||
style={{ width: `${fragmentsPercentage}%` }}
|
||||
/>
|
||||
</Tooltip>
|
||||
) : null}
|
||||
{metadata ? (
|
||||
<Tooltip
|
||||
title={
|
||||
<Fragment>
|
||||
<label>Metadata:</label>
|
||||
<Typography variant="caption" component="div">
|
||||
{formatBytes(metadata)} (
|
||||
{metadataPercentage.toLocaleString(undefined, {
|
||||
maximumFractionDigits: 1
|
||||
})}
|
||||
%)
|
||||
</Typography>
|
||||
</Fragment>
|
||||
}
|
||||
placement="top"
|
||||
arrow
|
||||
>
|
||||
<div
|
||||
className={classNames(css.bar, css.metadata)}
|
||||
style={{ width: `${metadataPercentage}%` }}
|
||||
/>
|
||||
</Tooltip>
|
||||
) : null}
|
||||
</div>
|
||||
</Fragment>
|
||||
) : uncached ? (
|
||||
<Typography variant="caption" component="div">
|
||||
Waiting...
|
||||
</Typography>
|
||||
) : (
|
||||
<Typography variant="caption" component="div">
|
||||
Calculating...
|
||||
</Typography>
|
||||
)}
|
||||
</div>
|
||||
</Fragment>
|
||||
);
|
||||
};
|
||||
|
|
@ -1 +0,0 @@
|
|||
export * from './UsageBreakdown';
|
||||
|
|
@ -42,9 +42,6 @@ export const pilosa = {
|
|||
metrics() {
|
||||
return api.get('/metrics.json');
|
||||
},
|
||||
usage() {
|
||||
return api.get('/ui/usage');
|
||||
},
|
||||
queryHistory() {
|
||||
return api.get('/query-history');
|
||||
},
|
||||
|
|
|
|||
|
|
@ -214,9 +214,6 @@ type Config struct {
|
|||
// LookupDBDSN is an external database to connect to for `ExternalLookup` queries.
|
||||
LookupDBDSN string `toml:"lookup-db-dsn"`
|
||||
|
||||
// The percentage of time spent recalculating the disk and memory usage cache.
|
||||
UsageDutyCycle float64 `toml:"usage-duty-cycle"`
|
||||
|
||||
// Future flags are used to represent features or functionality which is not
|
||||
// yet the default behavior, but will be in a future release.
|
||||
Future struct {
|
||||
|
|
@ -390,9 +387,6 @@ func NewConfig() *Config {
|
|||
c.Etcd.PeerCertFile = ""
|
||||
c.Etcd.PeerKeyFile = ""
|
||||
|
||||
// Disk and Memory Usage
|
||||
c.UsageDutyCycle = 20.0
|
||||
|
||||
// Future flags.
|
||||
c.Future.Rename = false
|
||||
|
||||
|
|
|
|||
|
|
@ -517,48 +517,6 @@ func TestHandler_Endpoints(t *testing.T) {
|
|||
}
|
||||
})
|
||||
|
||||
// UI/usage returns disk and memory usage from a precalculated cache.
|
||||
// Since the cache calculates the cache on server startup, and tests create indexes thereafter
|
||||
// the cache initially has 0 indexes when the test suite is ran. Therefore, this test first
|
||||
// resets the cache.
|
||||
t.Run("UI/usage", func(t *testing.T) {
|
||||
if cmd.API.ResetUsageCache() != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/ui/usage", nil))
|
||||
if w.Code != gohttp.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
}
|
||||
nodeUsages := make(map[string]pilosa.NodeUsage)
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &nodeUsages); err != nil {
|
||||
t.Fatalf("unmarshal")
|
||||
}
|
||||
|
||||
for _, nodeUsage := range nodeUsages {
|
||||
if nodeUsage.Disk.TotalUse < 1 {
|
||||
t.Fatalf("expected some disk use, got %d", nodeUsage.Disk.TotalUse)
|
||||
}
|
||||
if nodeUsage.Disk.Capacity < 1 {
|
||||
t.Fatalf("expected some disk capacity, got %d", nodeUsage.Disk.Capacity)
|
||||
}
|
||||
if nodeUsage.Memory.TotalUse < 1 {
|
||||
t.Fatalf("expected some memory use, got %d", nodeUsage.Memory.TotalUse)
|
||||
}
|
||||
if nodeUsage.Memory.Capacity < 1 {
|
||||
t.Fatalf("expected some memory capacity, got %d", nodeUsage.Memory.Capacity)
|
||||
}
|
||||
numIndexes := len(nodeUsage.Disk.IndexUsage)
|
||||
if numIndexes != 3 {
|
||||
t.Fatalf("wrong length index usage list: expected %d, got %d", 3, numIndexes)
|
||||
}
|
||||
numFields := len(nodeUsage.Disk.IndexUsage["i1"].Fields)
|
||||
if numFields != len(i1.Fields()) {
|
||||
t.Fatalf("wrong length field usage list: expected %d, got %d", len(i1.Fields()), numFields)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("UI/shard-distribution", func(t *testing.T) {
|
||||
// This tests the response structure, not the cluster behavior.
|
||||
w := httptest.NewRecorder()
|
||||
|
|
|
|||
|
|
@ -271,8 +271,6 @@ func (m *Command) Start() (err error) {
|
|||
}
|
||||
}
|
||||
|
||||
go m.API.RefreshUsageCache(m.Config.UsageDutyCycle)
|
||||
|
||||
_ = testhook.Opened(pilosa.NewAuditor(), m, nil)
|
||||
close(m.Started)
|
||||
return nil
|
||||
|
|
|
|||
187
txfactory.go
187
txfactory.go
|
|
@ -4,7 +4,6 @@ package pilosa
|
|||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
|
|
@ -471,192 +470,6 @@ func (f *TxFactory) DeleteFragmentFromStore(
|
|||
return f.dbPerShard.DeleteFragment(index, field, view, shard, frag)
|
||||
}
|
||||
|
||||
// IndexUsageDetails computes the sum of filesizes used by the node, broken down
|
||||
// by index, field, fragments and keys.
|
||||
func (f *TxFactory) IndexUsageDetails(isClosing func() bool) (map[string]IndexUsage, uint64, error) {
|
||||
indexUsage := make(map[string]IndexUsage)
|
||||
holderPath, err := expandDirName(f.holder.path)
|
||||
if err != nil {
|
||||
return indexUsage, 0, errors.Wrap(err, "expanding data directory")
|
||||
}
|
||||
indexesPath, err := expandDirName(f.holder.IndexesPath())
|
||||
if err != nil {
|
||||
return indexUsage, 0, errors.Wrap(err, "expanding indexes directory")
|
||||
}
|
||||
|
||||
idxs := f.holder.Indexes()
|
||||
|
||||
qcx := f.NewQcx()
|
||||
defer qcx.Abort()
|
||||
for _, idx := range idxs {
|
||||
index := idx.name
|
||||
indexPath := path.Join(indexesPath, index)
|
||||
|
||||
// field usage
|
||||
fieldUsages := make(map[string]FieldUsage)
|
||||
fragmentsTotal := uint64(0)
|
||||
fieldKeysTotal := uint64(0)
|
||||
fieldMetaBytesTotal := uint64(0)
|
||||
fieldsTotal := uint64(0)
|
||||
flds := idx.Fields()
|
||||
for _, fld := range flds {
|
||||
field := fld.Name()
|
||||
if field == "_keys" {
|
||||
continue
|
||||
}
|
||||
fUsage, err := f.fieldUsage(indexPath, fld)
|
||||
if err != nil {
|
||||
return indexUsage, 0, errors.Wrapf(err, "getting disk usage for index (%s)", index)
|
||||
}
|
||||
|
||||
// non-roaring field usage
|
||||
fragmentUsage := uint64(0)
|
||||
|
||||
for _, shard := range fld.AvailableShards(true).Slice() {
|
||||
if isClosing() {
|
||||
return nil, 0, nil
|
||||
}
|
||||
if err := func() error {
|
||||
tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "qcx.GetTx")
|
||||
}
|
||||
defer finisher(nil)
|
||||
|
||||
fieldBytes, err := tx.GetFieldSizeBytes(index, field)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "getting disk usage for non-roaring fragments (%s)", field)
|
||||
}
|
||||
fragmentUsage += fieldBytes
|
||||
return nil
|
||||
}(); err != nil {
|
||||
return indexUsage, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
// add non-roaring to roaring
|
||||
fUsage.Fragments += fragmentUsage
|
||||
fUsage.Total += fragmentUsage
|
||||
|
||||
// add to running total
|
||||
fieldMetaBytesTotal += fUsage.Metadata
|
||||
fieldKeysTotal += fUsage.Keys
|
||||
fragmentsTotal += fUsage.Fragments
|
||||
fieldsTotal += fUsage.Total
|
||||
|
||||
fieldUsages[field] = fUsage
|
||||
}
|
||||
|
||||
// index metadata
|
||||
indexMetaBytes, err := directoryUsage(indexPath, false)
|
||||
if err != nil {
|
||||
return indexUsage, 0, errors.Wrapf(err, "getting disk usage for index metadata (%s)", index)
|
||||
}
|
||||
|
||||
// index keys usage
|
||||
indexKeysBytes := uint64(0)
|
||||
if idx.keys {
|
||||
keysPath := path.Join(indexPath, translateStoreDir)
|
||||
indexKeysBytes, _ = directoryUsage(keysPath, true) // if directory doesn't exist, size = 0
|
||||
}
|
||||
|
||||
indexUsage[index] = IndexUsage{
|
||||
Total: indexMetaBytes + indexKeysBytes + fieldsTotal,
|
||||
Metadata: indexMetaBytes + fieldMetaBytesTotal,
|
||||
IndexKeys: indexKeysBytes,
|
||||
FieldKeysTotal: fieldKeysTotal,
|
||||
Fragments: fragmentsTotal,
|
||||
Fields: fieldUsages,
|
||||
}
|
||||
}
|
||||
|
||||
// node metadata, e.g. id allocator
|
||||
nodeMetaBytes, err := directoryUsage(holderPath, false)
|
||||
if err != nil {
|
||||
return indexUsage, 0, errors.Wrapf(err, "getting disk usage for node metadata")
|
||||
}
|
||||
|
||||
return indexUsage, nodeMetaBytes, nil
|
||||
}
|
||||
|
||||
// fieldUsage computes the sum of filesizes used by a field in
|
||||
// the filesystem tree (roaring storage), broken down by keys and fragments.
|
||||
func (f *TxFactory) fieldUsage(indexPath string, fld *Field) (FieldUsage, error) {
|
||||
fieldUsage := FieldUsage{}
|
||||
|
||||
field := fld.name
|
||||
|
||||
// row keys
|
||||
keysBytes := int64(0)
|
||||
var err error
|
||||
keysBytes, err = fileSize(fld.TranslateStorePath())
|
||||
if err != nil {
|
||||
// if file doesn't exist, size = 0
|
||||
keysBytes = 0
|
||||
}
|
||||
|
||||
// field metadata
|
||||
fieldPath := path.Join(indexPath, FieldsDir, field)
|
||||
metaBytes, err := directoryUsage(fieldPath, false) // this includes keys
|
||||
if err != nil {
|
||||
return fieldUsage, errors.Wrapf(err, "getting disk usage for field meta (%s)", field)
|
||||
}
|
||||
|
||||
// fragment data
|
||||
viewsPath := path.Join(fieldPath, "views")
|
||||
fragmentBytes := uint64(0)
|
||||
if dirExists(viewsPath) {
|
||||
fragmentBytes, err = directoryUsage(viewsPath, true)
|
||||
if err != nil {
|
||||
return fieldUsage, errors.Wrapf(err, "getting disk usage for field fragments (%s)", field)
|
||||
}
|
||||
}
|
||||
|
||||
fieldUsage = FieldUsage{
|
||||
Total: metaBytes + fragmentBytes, // metaBytes includes keys
|
||||
Metadata: metaBytes - uint64(keysBytes),
|
||||
Fragments: fragmentBytes,
|
||||
Keys: uint64(keysBytes),
|
||||
}
|
||||
|
||||
return fieldUsage, nil
|
||||
}
|
||||
|
||||
// NOTE: Go 1.16 introduced a new Readdir() method that is supposed to be more performant.
|
||||
// Not yet upgraded b/c new method is not compatible with older versions of Go.
|
||||
func directoryUsage(fname string, recursive bool) (uint64, error) {
|
||||
if !dirExists(fname) {
|
||||
return 0, errors.Errorf("directory does not exist (%s)", fname)
|
||||
}
|
||||
|
||||
var size uint64
|
||||
|
||||
dir, err := os.Open(fname)
|
||||
if err != nil {
|
||||
return 0, errors.Wrap(err, "opening data subdirectory")
|
||||
}
|
||||
defer dir.Close()
|
||||
|
||||
files, err := dir.Readdir(-1)
|
||||
if err != nil {
|
||||
return 0, errors.Wrap(err, "reading data subdirectory")
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
if recursive && file.IsDir() {
|
||||
sz, err := directoryUsage(path.Join(fname, file.Name()), true)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
size += sz
|
||||
} else {
|
||||
size += uint64(file.Size()) // NOTE this cast is safe for regular files, not necessarily others
|
||||
}
|
||||
}
|
||||
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// CloseIndex is a no-op. This seems to be in place for debugging purposes.
|
||||
func (f *TxFactory) CloseIndex(idx *Index) error {
|
||||
return nil
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue