mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 17:15:56 +00:00
Use simpler test
This commit is contained in:
Alan Bernstein
parent
c81ea88847
commit
5beb2664b0
1 changed files with 6 additions and 136 deletions
|
|
@ -25,8 +25,6 @@ import (
|
|||
"math"
|
||||
gohttp "net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
|
|
@ -37,7 +35,6 @@ import (
|
|||
"github.com/pilosa/pilosa/v2"
|
||||
"github.com/pilosa/pilosa/v2/boltdb"
|
||||
"github.com/pilosa/pilosa/v2/encoding/proto"
|
||||
"github.com/pilosa/pilosa/v2/gopsutil"
|
||||
"github.com/pilosa/pilosa/v2/http"
|
||||
"github.com/pilosa/pilosa/v2/pql"
|
||||
pb "github.com/pilosa/pilosa/v2/proto"
|
||||
|
|
@ -400,6 +397,12 @@ func TestHandler_Endpoints(t *testing.T) {
|
|||
|
||||
for _, nodeUsage := range nodeUsages {
|
||||
numIndexes := len(nodeUsage.Disk.IndexUsage)
|
||||
if nodeUsage.Disk.TotalUse < 75000 || nodeUsage.Disk.TotalUse > 300000 {
|
||||
// Usage measurements are not consistent between machines, or
|
||||
// over time, as features and implementations change, so checking
|
||||
// for a range of sizes may be most useful way to test the details of this.
|
||||
t.Fatalf("expected 75k < total < 300k, got %d", nodeUsage.Disk.TotalUse)
|
||||
}
|
||||
if numIndexes != 2 {
|
||||
t.Fatalf("wrong length index usage list: expected %d, got %d", 2, numIndexes)
|
||||
}
|
||||
|
|
@ -1574,139 +1577,6 @@ func TestQueryHistory(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func Test_DiskUsage_Roaring(t *testing.T) {
|
||||
|
||||
// roaring-usage-1: keys, existence
|
||||
// roaring-usage-1/f1: set, no keys
|
||||
// roaring-usage-1/f2: set, keys
|
||||
// roaring-usage-2: no keys, no existence
|
||||
// roaring-usage-2/g1: no keys, no existence
|
||||
// roaring-usage-3: keys, no existence, no fields
|
||||
schemaString := `{"indexes": [{"fields": [{"options": {"keys": false,"cacheSize": 50000,"cacheType": "ranked","type": "set"},"name": "f1"},{"options": {"keys": true,"cacheSize": 50000,"cacheType": "ranked","type": "set"},"name": "f2"}],"options": {"trackExistence": true,"keys": true},"name": "roaring-usage-1"},{"fields": [{"options": {"keys": true,"cacheSize": 50000,"cacheType": "ranked","type": "set"},"name": "g1"}],"options": {"trackExistence": false,"keys": false},"name": "roaring-usage-2"},{"fields": [],"options": {"trackExistence": false,"keys": true},"name": "roaring-usage-3"}]}
|
||||
`
|
||||
|
||||
txsrc := []string{"roaring", "rbf"}
|
||||
|
||||
exp0f := []string{`{"Test_DiskUsage_Roaring__0":{"diskUsage":{"capacity":%d,"totalInUse":16562,"indexes":{}}},"Test_DiskUsage_Roaring__1":{"diskUsage":{"capacity":%[1]d,"totalInUse":16562,"indexes":{}}},"Test_DiskUsage_Roaring__2":{"diskUsage":{"capacity":%[1]d,"totalInUse":16562,"indexes":{}}}}
|
||||
`, `{"Test_DiskUsage_Roaring__0":{"diskUsage":{"capacity":%d,"totalInUse":16562,"indexes":{}}},"Test_DiskUsage_Roaring__1":{"diskUsage":{"capacity":%[1]d,"totalInUse":16562,"indexes":{}}},"Test_DiskUsage_Roaring__2":{"diskUsage":{"capacity":%[1]d,"totalInUse":16562,"indexes":{}}}}
|
||||
`}
|
||||
|
||||
exp1f := []string{`{"Test_DiskUsage_Roaring__0":{"diskUsage":{"capacity":%[1]d,"totalInUse":115896,"indexes":{"roaring-usage-1":{"total":33156,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{"_exists":{"total":32791,"fragments":0,"keys":0},"f1":{"total":32791,"fragments":0,"keys":0},"f2":{"total":32793,"fragments":0,"keys":0}}},"roaring-usage-2":{"total":32896,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{"g1":{"total":32793,"fragments":0,"keys":0}}},"roaring-usage-3":{"total":32770,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{}}}}},"Test_DiskUsage_Roaring__1":{"diskUsage":{"capacity":%[1]d,"totalInUse":115896,"indexes":{"roaring-usage-1":{"total":33156,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{"_exists":{"total":32791,"fragments":0,"keys":0},"f1":{"total":32791,"fragments":0,"keys":0},"f2":{"total":32793,"fragments":0,"keys":0}}},"roaring-usage-2":{"total":32896,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{"g1":{"total":32793,"fragments":0,"keys":0}}},"roaring-usage-3":{"total":32770,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{}}}}},"Test_DiskUsage_Roaring__2":{"diskUsage":{"capacity":%[1]d,"totalInUse":115896,"indexes":{"roaring-usage-1":{"total":33156,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{"_exists":{"total":32791,"fragments":0,"keys":0},"f1":{"total":32791,"fragments":0,"keys":0},"f2":{"total":32793,"fragments":0,"keys":0}}},"roaring-usage-2":{"total":32896,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{"g1":{"total":32793,"fragments":0,"keys":0}}},"roaring-usage-3":{"total":32770,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{}}}}}}
|
||||
`, `{"Test_DiskUsage_Roaring__0":{"diskUsage":{"capacity":%[1]d,"totalInUse":115896,"indexes":{"roaring-usage-1":{"total":33156,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{"_exists":{"total":32791,"fragments":0,"keys":0},"f1":{"total":32791,"fragments":0,"keys":0},"f2":{"total":32793,"fragments":0,"keys":0}}},"roaring-usage-2":{"total":32896,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{"g1":{"total":32793,"fragments":0,"keys":0}}},"roaring-usage-3":{"total":32770,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{}}}}},"Test_DiskUsage_Roaring__1":{"diskUsage":{"capacity":%[1]d,"totalInUse":115896,"indexes":{"roaring-usage-1":{"total":33156,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{"_exists":{"total":32791,"fragments":0,"keys":0},"f1":{"total":32791,"fragments":0,"keys":0},"f2":{"total":32793,"fragments":0,"keys":0}}},"roaring-usage-2":{"total":32896,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{"g1":{"total":32793,"fragments":0,"keys":0}}},"roaring-usage-3":{"total":32770,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{}}}}},"Test_DiskUsage_Roaring__2":{"diskUsage":{"capacity":%[1]d,"totalInUse":115896,"indexes":{"roaring-usage-1":{"total":33156,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{"_exists":{"total":32791,"fragments":0,"keys":0},"f1":{"total":32791,"fragments":0,"keys":0},"f2":{"total":32793,"fragments":0,"keys":0}}},"roaring-usage-2":{"total":32896,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{"g1":{"total":32793,"fragments":0,"keys":0}}},"roaring-usage-3":{"total":32770,"indexKeys":0,"fieldKeysTotal":0,"fragments":0,"fields":{}}}}}}
|
||||
`}
|
||||
|
||||
for n := 0; n < 2; n++ {
|
||||
cluster := test.MustRunCluster(t, 3, []server.CommandOption{test.OptTxSrc(txsrc[n])})
|
||||
defer cluster.Close()
|
||||
cmd := cluster.GetNode(0)
|
||||
h := cmd.Handler.(*http.Handler).Handler
|
||||
holder := cmd.Server.Holder()
|
||||
|
||||
sysInfo := gopsutil.NewSystemInfo()
|
||||
capacity, err := sysInfo.DiskCapacity(holder.Path())
|
||||
if err != nil {
|
||||
t.Fatalf("unable to check disk capacity: %s", err)
|
||||
}
|
||||
|
||||
// check usage for empty cluster
|
||||
dumpFile(holder.Path() + "/.id")
|
||||
dumpFile(holder.Path() + "/.startup.log")
|
||||
dumpFile(holder.Path() + "/.topology")
|
||||
dumpFile(holder.Path() + "/idalloc.db")
|
||||
dumpDir(holder.Path())
|
||||
exp0 := fmt.Sprintf(exp0f[n], capacity)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/ui/usage", nil))
|
||||
if w.Code != gohttp.StatusOK {
|
||||
fmt.Printf("%+v\n", w.Body)
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
}
|
||||
body := w.Body.String()
|
||||
if body != exp0 {
|
||||
t.Fatalf("unexpected body:\n %s\nexpected:\n %s", body, exp0)
|
||||
}
|
||||
|
||||
// create schema
|
||||
w = httptest.NewRecorder()
|
||||
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(schemaString)))
|
||||
|
||||
if w.Code != gohttp.StatusNoContent {
|
||||
bod, err := ioutil.ReadAll(w.Result().Body)
|
||||
if err != nil {
|
||||
t.Errorf("reading body: %v", err)
|
||||
}
|
||||
t.Fatalf("unexpected code: %v, bod: %s", w.Code, bod)
|
||||
}
|
||||
|
||||
idx, err := cmd.API.Index(context.Background(), "roaring-usage-1")
|
||||
if err != nil {
|
||||
t.Fatalf("getting index: %v", err)
|
||||
}
|
||||
if idx.Name() != "roaring-usage-1" {
|
||||
t.Fatalf("index did not get set, got %v", idx.Name())
|
||||
}
|
||||
|
||||
// set some bits
|
||||
test.MustNewHTTPRequest("POST", "/index/roaring-usage-1/query", strings.NewReader(`Set("col1", f2=10)`))
|
||||
test.MustNewHTTPRequest("POST", "/index/roaring-usage-1/query", strings.NewReader(`Set("col1", f2="row1")`))
|
||||
test.MustNewHTTPRequest("POST", "/index/roaring-usage-2/query", strings.NewReader(`Set(21, g1=42)`))
|
||||
// check usage with data
|
||||
exp1 := fmt.Sprintf(exp1f[n], capacity)
|
||||
|
||||
dumpDir(holder.Path())
|
||||
|
||||
w = httptest.NewRecorder()
|
||||
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/ui/usage", nil))
|
||||
if w.Code != gohttp.StatusOK {
|
||||
fmt.Printf("%+v\n", w.Body)
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
}
|
||||
body = w.Body.String()
|
||||
if body != exp1 {
|
||||
t.Fatalf("unexpected body:\n %s\nexpected:\n %s", body, exp1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func dumpFile(pth string) {
|
||||
file, err := os.Open(pth)
|
||||
defer file.Close()
|
||||
msg := pth
|
||||
if err != nil {
|
||||
fmt.Printf("<error: %s>\n", err)
|
||||
return
|
||||
}
|
||||
b, err := ioutil.ReadAll(file)
|
||||
if err != nil {
|
||||
fmt.Printf("<error: %s>\n", err)
|
||||
}
|
||||
msg += fmt.Sprintf(" (%d bytes):", len(b))
|
||||
if len(b) <= 1000 {
|
||||
msg += fmt.Sprintf(string(b))
|
||||
}
|
||||
fmt.Printf(msg)
|
||||
fmt.Printf("\n")
|
||||
}
|
||||
|
||||
func dumpDir(pth string) {
|
||||
var files []string
|
||||
|
||||
err := filepath.Walk(pth, func(path string, info os.FileInfo, err error) error {
|
||||
files = append(files, path)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for _, pth2 := range files {
|
||||
file, _ := os.Open(pth2)
|
||||
b, _ := ioutil.ReadAll(file)
|
||||
fmt.Printf("%10d %s\n", len(b), pth2)
|
||||
file.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) {
|
||||
dec := json.NewDecoder(r)
|
||||
err := dec.Decode(&ret)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue