mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 17:15:56 +00:00
play with timing
This commit is contained in:
Samir Patel
Samir Patel
parent
dbdd3c4998
commit
a0ba9327f7
4 changed files with 77 additions and 52 deletions
29
api.go
29
api.go
|
|
@ -944,19 +944,24 @@ type MemoryUsage struct {
|
|||
|
||||
// Returns disk usage from cache. Waits for calculation if cache is empty.
|
||||
func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, error) {
|
||||
// time.Sleep(time.Second * 2)
|
||||
fmt.Println("Usage: 1")
|
||||
span, _ := tracing.StartSpanFromContext(ctx, "API.Usage")
|
||||
defer span.Finish()
|
||||
api.usageCache.muRead.Lock()
|
||||
defer api.usageCache.muRead.Unlock()
|
||||
fmt.Println("Usage: 2")
|
||||
|
||||
var t time.Time
|
||||
if api.usageCache.lastUpdated == t {
|
||||
api.calculateUsage()
|
||||
}
|
||||
fmt.Printf("Usage: 3, %+v\n", api.usageCache.data)
|
||||
|
||||
if !remote {
|
||||
api.requestUsageOfNodes()
|
||||
}
|
||||
fmt.Println("Usage: 4")
|
||||
|
||||
api.server.logger.Infof("disk usage results last updated: %v", api.usageCache.lastUpdated.Format(time.RFC1123))
|
||||
return api.usageCache.data, nil
|
||||
|
|
@ -964,6 +969,13 @@ func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, e
|
|||
|
||||
// Makes a ui/usage request for each node in cluster to calculates its usage and adds it to the cache
|
||||
func (api *API) requestUsageOfNodes() {
|
||||
// err := api.server.cluster.no
|
||||
// if err != nil {
|
||||
// t.Fatalf("starting cluster: %v", err)
|
||||
// }
|
||||
|
||||
// time.Sleep(time.Second * 10)
|
||||
|
||||
nodes := api.cluster.Nodes()
|
||||
for _, node := range nodes {
|
||||
if node.ID == api.server.nodeID {
|
||||
|
|
@ -978,7 +990,9 @@ func (api *API) requestUsageOfNodes() {
|
|||
if err != nil {
|
||||
api.server.logger.Infof("couldn't collect disk usage from %s: %s", node.URI, err)
|
||||
}
|
||||
|
||||
fmt.Println("NU: 2")
|
||||
fmt.Printf("NU: %+v\n", nodeUsage)
|
||||
// api.usageCache.muRead.Lock()
|
||||
fmt.Println("NU: 3")
|
||||
api.usageCache.data[node.ID] = nodeUsage[node.ID]
|
||||
|
|
@ -995,6 +1009,7 @@ func (api *API) calculateUsage() {
|
|||
|
||||
// api.usageCache.muRead.Lock()
|
||||
lastUpdated := api.usageCache.lastUpdated
|
||||
// lastUpdated := api.usageCache.data[api.server.nodeID].LastUpdated
|
||||
// api.usageCache.muRead.Unlock()
|
||||
|
||||
if time.Since(lastUpdated) > api.usageCache.refreshInterval {
|
||||
|
|
@ -1004,6 +1019,7 @@ func (api *API) calculateUsage() {
|
|||
if err != nil {
|
||||
api.server.logger.Infof("couldn't get index usage details: %s", err)
|
||||
}
|
||||
fmt.Printf("Calculate - Index Details: %+v\n", indexDetails)
|
||||
totalSize := nodeMetadataBytes
|
||||
for _, s := range indexDetails {
|
||||
totalSize += s.Total
|
||||
|
|
@ -1041,23 +1057,28 @@ func (api *API) calculateUsage() {
|
|||
// api.usageCache.muRead.Lock()
|
||||
// defer api.usageCache.muRead.Unlock()
|
||||
api.usageCache.data = make(map[string]NodeUsage)
|
||||
fmt.Printf("Calculate - Node Usage: %+v\n", nodeUsage)
|
||||
api.usageCache.data[api.server.nodeID] = nodeUsage
|
||||
fmt.Printf("Calculate - NU from Map: %+v\n", api.usageCache.data[api.server.nodeID])
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// Periodically calculates disk usage
|
||||
func (api *API) RefreshUsageCache(refresh time.Duration) {
|
||||
fmt.Println("STARTING REFRESH")
|
||||
api.usageCache = &usageCache{
|
||||
data: make(map[string]NodeUsage),
|
||||
refreshInterval: refresh,
|
||||
}
|
||||
// time.Sleep(time.Second * 5)
|
||||
fmt.Println("Loop: 0")
|
||||
time.Sleep(time.Second * 2)
|
||||
for {
|
||||
fmt.Println(1)
|
||||
fmt.Println("Loop: 1")
|
||||
api.calculateUsage()
|
||||
fmt.Println(2)
|
||||
fmt.Printf("Loop: 2, %+v\n", api.usageCache.data)
|
||||
api.usageCache.lastUpdated = time.Now()
|
||||
fmt.Println(3)
|
||||
fmt.Println("Loop: 3")
|
||||
time.Sleep(api.usageCache.refreshInterval)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -502,56 +502,60 @@ func TestHandler_Endpoints(t *testing.T) {
|
|||
}
|
||||
})
|
||||
|
||||
// t.Run("UI/usage", func(t *testing.T) {
|
||||
// // time.Sleep(time.Second * 10)
|
||||
// w := httptest.NewRecorder()
|
||||
// h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/ui/usage", nil))
|
||||
// if w.Code != gohttp.StatusOK {
|
||||
// fmt.Printf("%+v\n", w.Body)
|
||||
// t.Fatalf("unexpected status code: %d", w.Code)
|
||||
// }
|
||||
// t.Logf("Usage w body string: %+v\n", w.Body.String())
|
||||
// nodeUsages := make(map[string]pilosa.NodeUsage)
|
||||
// if err := json.Unmarshal(w.Body.Bytes(), &nodeUsages); err != nil {
|
||||
// t.Fatalf("unmarshal")
|
||||
// }
|
||||
t.Run("UI/usage", func(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/ui/usage", nil))
|
||||
if w.Code != gohttp.StatusOK {
|
||||
fmt.Printf("%+v\n", w.Body)
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
}
|
||||
fmt.Printf("Cluster Size: %v\n", cluster.Len())
|
||||
t.Logf("Usage w body string: %+v\n", w.Body.String())
|
||||
nodeUsages := make(map[string]pilosa.NodeUsage)
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &nodeUsages); err != nil {
|
||||
t.Fatalf("unmarshal")
|
||||
}
|
||||
|
||||
// // w2 := httptest.NewRecorder()
|
||||
// // h.ServeHTTP(w2, test.MustNewHTTPRequest("GET", "/schema", nil))
|
||||
// // if w2.Code != gohttp.StatusOK {
|
||||
// // t.Fatalf("unexpected status code: %d", w2.Code)
|
||||
// // }
|
||||
// // schemaBody := w2.Body.String()
|
||||
// // t.Fatalf("Schema: %+v\n", schemaBody)
|
||||
// // if schemaBody != "{\"indexes\":[]}\n" {
|
||||
// // t.Fatalf("unexpected empty schema: '%v'", schemaBody)
|
||||
// // }
|
||||
// w2 := httptest.NewRecorder()
|
||||
// h.ServeHTTP(w2, test.MustNewHTTPRequest("GET", "/schema", nil))
|
||||
// if w2.Code != gohttp.StatusOK {
|
||||
// t.Fatalf("unexpected status code: %d", w2.Code)
|
||||
// }
|
||||
// schemaBody := w2.Body.String()
|
||||
// t.Fatalf("Schema: %+v\n", schemaBody)
|
||||
// if schemaBody != "{\"indexes\":[]}\n" {
|
||||
// t.Fatalf("unexpected empty schema: '%v'", schemaBody)
|
||||
// }
|
||||
|
||||
// for _, nodeUsage := range nodeUsages {
|
||||
// t.Logf("Len of Indexes: %+v\n", len(nodeUsage.Disk.IndexUsage))
|
||||
// numIndexes := len(nodeUsage.Disk.IndexUsage)
|
||||
// fmt.Printf("Node Usage: %+v\n", nodeUsage)
|
||||
// fmt.Printf("Disk Usage: %+v\n", nodeUsage.Disk)
|
||||
// for k, v := range nodeUsage.Disk.IndexUsage {
|
||||
// fmt.Printf("Index Usage: K: %+v V: %+v \n", k, v)
|
||||
// }
|
||||
// // if nodeUsage.Disk.TotalUse < 75000 || nodeUsage.Disk.TotalUse > 700000 {
|
||||
// if nodeUsage.Disk.TotalUse < 1 {
|
||||
// // Usage measurements are not consistent between machines, or
|
||||
// // over time, as features and implementations change, so checking
|
||||
// // for a range of sizes may be most useful way to test the details of this.
|
||||
// t.Fatalf("expected 75k < total < 500k, got %d", nodeUsage.Disk.TotalUse)
|
||||
// }
|
||||
// if numIndexes != 3 {
|
||||
// t.Fatalf("wrong length index usage list: expected %d, got %d", 2, numIndexes)
|
||||
// }
|
||||
// numFields := len(nodeUsage.Disk.IndexUsage["i1"].Fields)
|
||||
// if numFields != len(i1.Fields()) {
|
||||
// t.Fatalf("wrong length field usage list: expected %d, got %d", len(i1.Fields()), numFields)
|
||||
// }
|
||||
// }
|
||||
numNodes := len(nodeUsages)
|
||||
fmt.Printf("num nodes: %+v\n", numNodes)
|
||||
|
||||
// })
|
||||
for _, nodeUsage := range nodeUsages {
|
||||
t.Logf("Len of Indexes: %+v\n", len(nodeUsage.Disk.IndexUsage))
|
||||
numIndexes := len(nodeUsage.Disk.IndexUsage)
|
||||
fmt.Printf("Node Usage: %+v\n", nodeUsage)
|
||||
fmt.Printf("Disk Usage: %+v\n", nodeUsage.Disk)
|
||||
for k, v := range nodeUsage.Disk.IndexUsage {
|
||||
fmt.Printf("Index Usage: K: %+v V: %+v \n", k, v)
|
||||
}
|
||||
// if nodeUsage.Disk.TotalUse < 75000 || nodeUsage.Disk.TotalUse > 700000 {
|
||||
if nodeUsage.Disk.TotalUse < 1 {
|
||||
// Usage measurements are not consistent between machines, or
|
||||
// over time, as features and implementations change, so checking
|
||||
// for a range of sizes may be most useful way to test the details of this.
|
||||
t.Fatalf("expected 75k < total < 500k, got %d", nodeUsage.Disk.TotalUse)
|
||||
}
|
||||
if numIndexes != 3 {
|
||||
t.Fatalf("wrong length index usage list: expected %d, got %d", 2, numIndexes)
|
||||
}
|
||||
numFields := len(nodeUsage.Disk.IndexUsage["i1"].Fields)
|
||||
if numFields != len(i1.Fields()) {
|
||||
t.Fatalf("wrong length field usage list: expected %d, got %d", len(i1.Fields()), numFields)
|
||||
}
|
||||
}
|
||||
t.Fatalf("end")
|
||||
|
||||
})
|
||||
|
||||
t.Run("UI/shard-distribution", func(t *testing.T) {
|
||||
// This tests the response structure, not the cluster behavior.
|
||||
|
|
|
|||
|
|
@ -277,7 +277,6 @@ func (m *Command) Start() (err error) {
|
|||
}
|
||||
}
|
||||
|
||||
fmt.Println("STARTING REFRESH")
|
||||
go m.API.RefreshUsageCache(time.Duration(m.Config.Usage.Interval))
|
||||
|
||||
_ = testhook.Opened(pilosa.NewAuditor(), m, nil)
|
||||
|
|
|
|||
|
|
@ -584,6 +584,7 @@ func (f *TxFactory) IndexUsageDetails() (map[string]IndexUsage, uint64, error) {
|
|||
}
|
||||
|
||||
idxs := f.holder.Indexes()
|
||||
fmt.Printf("IndexUsageDetails: %+v\n", idxs)
|
||||
|
||||
qcx := f.NewQcx()
|
||||
defer qcx.Abort()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue