diff --git a/api.go b/api.go index 4803d5f3a..6c3bd37a0 100644 --- a/api.go +++ b/api.go @@ -944,19 +944,24 @@ type MemoryUsage struct { // Returns disk usage from cache. Waits for calculation if cache is empty. func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, error) { + // time.Sleep(time.Second * 2) + fmt.Println("Usage: 1") span, _ := tracing.StartSpanFromContext(ctx, "API.Usage") defer span.Finish() api.usageCache.muRead.Lock() defer api.usageCache.muRead.Unlock() + fmt.Println("Usage: 2") var t time.Time if api.usageCache.lastUpdated == t { api.calculateUsage() } + fmt.Printf("Usage: 3, %+v\n", api.usageCache.data) if !remote { api.requestUsageOfNodes() } + fmt.Println("Usage: 4") api.server.logger.Infof("disk usage results last updated: %v", api.usageCache.lastUpdated.Format(time.RFC1123)) return api.usageCache.data, nil @@ -964,6 +969,13 @@ func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, e // Makes a ui/usage request for each node in cluster to calculates its usage and adds it to the cache func (api *API) requestUsageOfNodes() { + // err := api.server.cluster.no + // if err != nil { + // t.Fatalf("starting cluster: %v", err) + // } + + // time.Sleep(time.Second * 10) + nodes := api.cluster.Nodes() for _, node := range nodes { if node.ID == api.server.nodeID { @@ -978,7 +990,9 @@ func (api *API) requestUsageOfNodes() { if err != nil { api.server.logger.Infof("couldn't collect disk usage from %s: %s", node.URI, err) } + fmt.Println("NU: 2") + fmt.Printf("NU: %+v\n", nodeUsage) // api.usageCache.muRead.Lock() fmt.Println("NU: 3") api.usageCache.data[node.ID] = nodeUsage[node.ID] @@ -995,6 +1009,7 @@ func (api *API) calculateUsage() { // api.usageCache.muRead.Lock() lastUpdated := api.usageCache.lastUpdated + // lastUpdated := api.usageCache.data[api.server.nodeID].LastUpdated // api.usageCache.muRead.Unlock() if time.Since(lastUpdated) > api.usageCache.refreshInterval { @@ -1004,6 +1019,7 @@ func (api *API) calculateUsage() { if err != nil { api.server.logger.Infof("couldn't get index usage details: %s", err) } + fmt.Printf("Calculate - Index Details: %+v\n", indexDetails) totalSize := nodeMetadataBytes for _, s := range indexDetails { totalSize += s.Total @@ -1041,23 +1057,28 @@ func (api *API) calculateUsage() { // api.usageCache.muRead.Lock() // defer api.usageCache.muRead.Unlock() api.usageCache.data = make(map[string]NodeUsage) + fmt.Printf("Calculate - Node Usage: %+v\n", nodeUsage) api.usageCache.data[api.server.nodeID] = nodeUsage + fmt.Printf("Calculate - NU from Map: %+v\n", api.usageCache.data[api.server.nodeID]) + } } // Periodically calculates disk usage func (api *API) RefreshUsageCache(refresh time.Duration) { + fmt.Println("STARTING REFRESH") api.usageCache = &usageCache{ data: make(map[string]NodeUsage), refreshInterval: refresh, } - // time.Sleep(time.Second * 5) + fmt.Println("Loop: 0") + time.Sleep(time.Second * 2) for { - fmt.Println(1) + fmt.Println("Loop: 1") api.calculateUsage() - fmt.Println(2) + fmt.Printf("Loop: 2, %+v\n", api.usageCache.data) api.usageCache.lastUpdated = time.Now() - fmt.Println(3) + fmt.Println("Loop: 3") time.Sleep(api.usageCache.refreshInterval) } } diff --git a/server/handler_test.go b/server/handler_test.go index 4de6a126d..4ed976f5e 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -502,56 +502,60 @@ func TestHandler_Endpoints(t *testing.T) { } }) - // t.Run("UI/usage", func(t *testing.T) { - // // time.Sleep(time.Second * 10) - // w := httptest.NewRecorder() - // h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/ui/usage", nil)) - // if w.Code != gohttp.StatusOK { - // fmt.Printf("%+v\n", w.Body) - // t.Fatalf("unexpected status code: %d", w.Code) - // } - // t.Logf("Usage w body string: %+v\n", w.Body.String()) - // nodeUsages := make(map[string]pilosa.NodeUsage) - // if err := json.Unmarshal(w.Body.Bytes(), &nodeUsages); err != nil { - // t.Fatalf("unmarshal") - // } + t.Run("UI/usage", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/ui/usage", nil)) + if w.Code != gohttp.StatusOK { + fmt.Printf("%+v\n", w.Body) + t.Fatalf("unexpected status code: %d", w.Code) + } + fmt.Printf("Cluster Size: %v\n", cluster.Len()) + t.Logf("Usage w body string: %+v\n", w.Body.String()) + nodeUsages := make(map[string]pilosa.NodeUsage) + if err := json.Unmarshal(w.Body.Bytes(), &nodeUsages); err != nil { + t.Fatalf("unmarshal") + } - // // w2 := httptest.NewRecorder() - // // h.ServeHTTP(w2, test.MustNewHTTPRequest("GET", "/schema", nil)) - // // if w2.Code != gohttp.StatusOK { - // // t.Fatalf("unexpected status code: %d", w2.Code) - // // } - // // schemaBody := w2.Body.String() - // // t.Fatalf("Schema: %+v\n", schemaBody) - // // if schemaBody != "{\"indexes\":[]}\n" { - // // t.Fatalf("unexpected empty schema: '%v'", schemaBody) - // // } + // w2 := httptest.NewRecorder() + // h.ServeHTTP(w2, test.MustNewHTTPRequest("GET", "/schema", nil)) + // if w2.Code != gohttp.StatusOK { + // t.Fatalf("unexpected status code: %d", w2.Code) + // } + // schemaBody := w2.Body.String() + // t.Fatalf("Schema: %+v\n", schemaBody) + // if schemaBody != "{\"indexes\":[]}\n" { + // t.Fatalf("unexpected empty schema: '%v'", schemaBody) + // } - // for _, nodeUsage := range nodeUsages { - // t.Logf("Len of Indexes: %+v\n", len(nodeUsage.Disk.IndexUsage)) - // numIndexes := len(nodeUsage.Disk.IndexUsage) - // fmt.Printf("Node Usage: %+v\n", nodeUsage) - // fmt.Printf("Disk Usage: %+v\n", nodeUsage.Disk) - // for k, v := range nodeUsage.Disk.IndexUsage { - // fmt.Printf("Index Usage: K: %+v V: %+v \n", k, v) - // } - // // if nodeUsage.Disk.TotalUse < 75000 || nodeUsage.Disk.TotalUse > 700000 { - // if nodeUsage.Disk.TotalUse < 1 { - // // Usage measurements are not consistent between machines, or - // // over time, as features and implementations change, so checking - // // for a range of sizes may be most useful way to test the details of this. - // t.Fatalf("expected 75k < total < 500k, got %d", nodeUsage.Disk.TotalUse) - // } - // if numIndexes != 3 { - // t.Fatalf("wrong length index usage list: expected %d, got %d", 2, numIndexes) - // } - // numFields := len(nodeUsage.Disk.IndexUsage["i1"].Fields) - // if numFields != len(i1.Fields()) { - // t.Fatalf("wrong length field usage list: expected %d, got %d", len(i1.Fields()), numFields) - // } - // } + numNodes := len(nodeUsages) + fmt.Printf("num nodes: %+v\n", numNodes) - // }) + for _, nodeUsage := range nodeUsages { + t.Logf("Len of Indexes: %+v\n", len(nodeUsage.Disk.IndexUsage)) + numIndexes := len(nodeUsage.Disk.IndexUsage) + fmt.Printf("Node Usage: %+v\n", nodeUsage) + fmt.Printf("Disk Usage: %+v\n", nodeUsage.Disk) + for k, v := range nodeUsage.Disk.IndexUsage { + fmt.Printf("Index Usage: K: %+v V: %+v \n", k, v) + } + // if nodeUsage.Disk.TotalUse < 75000 || nodeUsage.Disk.TotalUse > 700000 { + if nodeUsage.Disk.TotalUse < 1 { + // Usage measurements are not consistent between machines, or + // over time, as features and implementations change, so checking + // for a range of sizes may be most useful way to test the details of this. + t.Fatalf("expected 75k < total < 500k, got %d", nodeUsage.Disk.TotalUse) + } + if numIndexes != 3 { + t.Fatalf("wrong length index usage list: expected %d, got %d", 2, numIndexes) + } + numFields := len(nodeUsage.Disk.IndexUsage["i1"].Fields) + if numFields != len(i1.Fields()) { + t.Fatalf("wrong length field usage list: expected %d, got %d", len(i1.Fields()), numFields) + } + } + t.Fatalf("end") + + }) t.Run("UI/shard-distribution", func(t *testing.T) { // This tests the response structure, not the cluster behavior. diff --git a/server/server.go b/server/server.go index 286644c18..a8d5ecfc4 100644 --- a/server/server.go +++ b/server/server.go @@ -277,7 +277,6 @@ func (m *Command) Start() (err error) { } } - fmt.Println("STARTING REFRESH") go m.API.RefreshUsageCache(time.Duration(m.Config.Usage.Interval)) _ = testhook.Opened(pilosa.NewAuditor(), m, nil) diff --git a/txfactory.go b/txfactory.go index 01b000b87..1a85f05c7 100644 --- a/txfactory.go +++ b/txfactory.go @@ -584,6 +584,7 @@ func (f *TxFactory) IndexUsageDetails() (map[string]IndexUsage, uint64, error) { } idxs := f.holder.Indexes() + fmt.Printf("IndexUsageDetails: %+v\n", idxs) qcx := f.NewQcx() defer qcx.Abort()