scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-09 20:37:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1131 from seebs/randomerQuery

Browse source 
Improve random-query
This commit is contained in:
seebs 2020-11-23 11:58:53 -06:00 • committed by GitHub
commit 55d0c12703
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 213 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

239
cmd/random-query/main.go
View file

@ -18,13 +18,16 @@ import (
"context"
"flag"
"fmt"
"math"
"math/rand"
nethttp "net/http"
"os"
"strconv"
"strings"
"time"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/http"
)
@ -36,6 +39,12 @@ type RandomQueryConfig struct {
TreeDepth int // -d
QueryCount int // -n
Verbose bool // -v
VeryVerbose bool // -V
TimeFromArg string // --time.from
TimeToArg string // --time.to
TimeFrom time.Time // parsed time
TimeTo time.Time // parsed time
TimeRange int64 // hours between parsed times
IndexMap map[string]*Features
@ -76,12 +85,19 @@ func wrapApiToInternalClient(api *pilosa.API) *wrapper {
return &wrapper{api: api}
}
// These times are copied from the "kitchen sink" data generator to serve as defaults.
var defaultEndTime = time.Date(2020, time.May, 4, 12, 2, 28, 0, time.UTC)
var defaultStartTime = defaultEndTime.Add(-5 * 365 * 24 * time.Hour)
// call DefineFlags before myflags.Parse()
func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) {
fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.")
fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.")
fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.")
fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated")
fs.BoolVar(&cfg.VeryVerbose, "V", false, "show query results")
fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)")
fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)")
}
// call c.ValidateConfig() after myflags.Parse()
@ -92,6 +108,20 @@ func (c *RandomQueryConfig) ValidateConfig() error {
if c.QueryCount < 0 {
return fmt.Errorf("-n count must be 0 or greater; saw %v", c.QueryCount)
}
var err error
c.TimeFrom, err = time.Parse(time.RFC3339, c.TimeFromArg)
if err != nil {
return fmt.Errorf("-time.from value couldn't be parsed: %w", err)
}
c.TimeTo, err = time.Parse(time.RFC3339, c.TimeToArg)
if err != nil {
return fmt.Errorf("-time.to value couldn't be parsed: %w", err)
}
c.TimeRange = int64(c.TimeTo.Sub(c.TimeFrom).Hours())
if c.TimeRange < 1 {
return fmt.Errorf("time.to (%s) should be at least one hour after time.from (%s)",
c.TimeToArg, c.TimeFromArg)
}
return nil
}
@ -133,6 +163,17 @@ func (cfg *RandomQueryConfig) Run() (err error) {
loops := 0
t0 := time.Now()
report := func() {
dur := time.Since(t0)
if dur > 0 {
qps := 1e9 * float64(totalQ) / float64(dur)
AlwaysPrintf("totalQueries run: %v elapsed: %v qps: %0.02f", totalQ, dur, qps)
} else {
AlwaysPrintf("totalQueries run: %v elapsed: %v qps: N/A", totalQ, dur)
}
}
defer report()
NewSetup:
err = cfg.Setup(cli)
if err != nil {
@ -162,15 +203,11 @@ NewSetup:
goto NewSetup
}
}
if totalQ%100 == 0 {
dur := time.Since(t0)
if dur > 0 {
qps := 1e9 * float64(totalQ) / float64(dur)
AlwaysPrintf("totalQueries run: %v elapsed: %v qps: %0.02f", totalQ, dur, qps)
}
if totalQ > 0 && totalQ%100 == 0 {
report()
}
index := indexes[rand.Intn(len(indexes))]
index := indexes[cfg.Rnd.Intn(len(indexes))]
pql, err := cfg.GenQuery(index)
panicOn(err)
@ -185,11 +222,12 @@ NewSetup:
AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql)
return err
}
if cfg.Verbose {
if cfg.VeryVerbose {
fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0])
}
totalQ++
loops++
}
return nil
@ -197,6 +235,21 @@ NewSetup:
type Features struct {
Slc []IndexFieldRow
Ranges []IndexFieldRange
Distinctables []IndexFieldRange
SlcWeight int
RangeWeight int
}
// Pick either a feature entry or a random query on a range, weighted
// by number of features and approximate weight of ranges
func (f *Features) RandomQuery(cfg *RandomQueryConfig) *Tree {
r := cfg.Rnd.Intn(f.SlcWeight + f.RangeWeight)
if r < f.SlcWeight {
return f.Slc[r].Query(cfg)
}
r = cfg.Rnd.Intn(len(f.Ranges))
return f.Ranges[r].Query(cfg)
}
func NewRandomQueryConfig() *RandomQueryConfig {
@ -211,28 +264,99 @@ type IndexFieldRow struct {
RowID uint64
RowKey string
IsRowKey bool
HasTime bool
IsInt bool
}
func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree {
fromTo := ""
// 5% of queries on a time field will use the standard view
// anyway.
if fea.HasTime && cfg.Rnd.Int63n(20) != 0 {
startHours := (cfg.Rnd.Int63n(cfg.TimeRange - 1))
endHours := cfg.Rnd.Int63n(cfg.TimeRange - startHours) + 1 + startHours
startTime := cfg.TimeFrom.Add(time.Duration(startHours) * time.Hour)
endTime := cfg.TimeFrom.Add(time.Duration(endHours) * time.Hour)
fromTo = fmt.Sprintf(", from=%s, to=%s",
startTime.Format(pilosaTimeFmt),
endTime.Format(pilosaTimeFmt))
}
if fea.IsRowKey {
return &Tree{S: fmt.Sprintf("Row(%v='%v'%s)", fea.Field, fea.RowKey, fromTo)}
}
return &Tree{S: fmt.Sprintf("Row(%v=%v%s)", fea.Field, fea.RowID, fromTo)}
}
type IndexFieldRange struct {
Index string
Field string
Min, Max, Scale int64
ScaleDiv float64
Range uint64
}
// We want to pick one of (1) a single-operation filter, (2) a
// between-filter of some kind.
// So, that's one of <=, >=, ==, !=, >, <, or
// one of [<, <], [<, <=], [<=, <=], [<=, <].
var binaryOps = []string{
"<=", ">=", "==", "!=", "<", ">",
}
func (i *IndexFieldRange) Query(cfg *RandomQueryConfig) *Tree {
r := cfg.Rnd.Int63n(10)
// this is unevenly weighted, but there's no Uint64N, and
// Int63n can't represent the whole range.
v1 := cfg.Rnd.Uint64() % i.Range
v2 := cfg.Rnd.Uint64() % i.Range
if v1 > v2 {
v1, v2 = v2, v1
}
v1 = v1 + uint64(i.Min)
v2 = v2 + uint64(i.Min)
var v1s, v2s string
if i.Scale != 0 {
v1s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v1)) / i.ScaleDiv)
v2s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v2)) / i.ScaleDiv)
} else {
v1s = strconv.FormatInt(int64(v1), 10)
v2s = strconv.FormatInt(int64(v2), 10)
}
if r < 4 {
lte := "<="
op1 := lte[:1+(r&1)]
op2 := lte[:1+((r>>1)&1)]
return &Tree{S: fmt.Sprintf("Row(%s %s %s %s %s)",
v1s, op1, i.Field, op2, v2s)}
} else {
if cfg.Rnd.Int63n(2) == 1 {
v1s = v2s
}
return &Tree{S: fmt.Sprintf("Row(%s %s %s)", i.Field, binaryOps[r - 4], v1s)}
}
}
// Run a RandomQuery takes a list of RowIDFeatures and ColumnKeyObjects
// and spits back a PQL query
//
func (cfg *RandomQueryConfig) Setup(api API) (err error) {
ctx := context.Background()
cfg.Info, err = api.Schema(ctx)
if err != nil {
return err
}
foundIntField := false
for i, ii := range cfg.Info {
_ = i
for k, fld := range ii.Fields {
_ = k
if fld.Options.Type == "set" {
switch fld.Options.Type {
case "set", "mutex", "time":
pql := fmt.Sprintf("Rows(%v)", fld.Name)
res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql})
panicOn(err)
if cfg.Verbose {
if cfg.VeryVerbose {
fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0])
}
// if the option is set to use RowKeys, then must get the Keys instead of the Rows from the RowIdentifiers.
@ -243,31 +367,87 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) {
switch x := res.Results[0].(type) {
case *pilosa.RowIdentifiers:
// internalClient gets this
cfg.AddResponse(ii.Name, fld.Name, x)
cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time")
case pilosa.RowIdentifiers:
// test gets this
cfg.AddResponse(ii.Name, fld.Name, &x)
cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time")
}
case "int":
foundIntField = true
fallthrough // I bet you thought you'd never see this used
case "decimal":
// we'll ignore row keys and just use value ranges
cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal")
default:
AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type)
}
}
}
cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"}
if foundIntField {
cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct")
}
seed := int64(42)
cfg.Rnd = rand.New(rand.NewSource(seed))
return nil
}
func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers) {
func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool) {
for _, rowID := range x.Rows {
cfg.AddFeature(index, field, rowID, "", false)
cfg.AddFeature(index, field, rowID, "", false, hasTime)
}
for _, rowKey := range x.Keys {
cfg.AddFeature(index, field, 0, rowKey, true)
cfg.AddFeature(index, field, 0, rowKey, true, hasTime)
}
}
const maxEffectiveRange = 1000000
func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Decimal, scale int64, decimal bool) {
f, ok := cfg.IndexMap[index]
if !ok {
f = &Features{}
cfg.IndexMap[index] = f
}
if min.Scale != scale || max.Scale != scale {
panic(fmt.Sprintf("scale error; min scale %d, max scale %d, field scale %d, assumed they'd be equal",
min.Scale, max.Scale, scale))
}
effectiveRange := uint64(max.Value) - uint64(min.Value) + 1
// if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which
// wraps to 0. Anything closer together will be fine. We accept the loss
// of accuracy in the range from not representing quite the full value
// in that edge case.
if effectiveRange == 0 {
effectiveRange--
}
// we assume that the Value of the field is already scaled, I guess?
newRange := IndexFieldRange{
Index: index,
Field: field,
Min: min.Value,
Max: max.Value,
Scale: scale,
ScaleDiv: math.Pow(10, float64(scale)),
Range: effectiveRange,
}
f.Ranges = append(f.Ranges, newRange)
if !decimal {
f.Distinctables = append(f.Distinctables, newRange)
}
// We want to add more values for larger int fields, but the
// default KitchenSink field has a range of 1<<64 which would make
// it completely dominate weights, so...
if effectiveRange > maxEffectiveRange {
effectiveRange = maxEffectiveRange
}
f.RangeWeight += int(effectiveRange)
}
func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) {
tree := cfg.GenTree(index, cfg.TreeDepth)
@ -284,8 +464,8 @@ func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) {
type Tree struct {
Chd []*Tree
S string
Args []string // Extra args to pass after children, such as a field for Distinct.
}
func (tr *Tree) StringIndent(ind int) (s string) {
@ -315,16 +495,11 @@ func (tr *Tree) StringIndent(ind int) (s string) {
return
}
const pilosaTimeFmt = "2006-01-02T15:04"
func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) {
features := cfg.IndexMap[index]
if depth == 0 {
slc := cfg.IndexMap[index].Slc
//vv("depth is 0, slc = '%#v'", slc)
r := cfg.Rnd.Intn(len(slc))
fea := slc[r]
if fea.IsRowKey {
return &Tree{S: fmt.Sprintf("Row(%v='%v')", fea.Field, fea.RowKey)}
}
return &Tree{S: fmt.Sprintf("Row(%v=%v)", fea.Field, fea.RowID)}
return features.RandomQuery(cfg)
}
r := cfg.Rnd.Intn(len(cfg.BitmapFunc))
@ -338,6 +513,14 @@ func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) {
numChild = 1
case "Difference":
numChild = 2
case "Distinct":
numChild = 1
// sometimes do a bare distinct without a filter
if cfg.Rnd.Intn(10) == 0 {
numChild = 0
}
r = cfg.Rnd.Intn(len(features.Distinctables))
tr.Args = append(tr.Args, fmt.Sprintf("field=%s", features.Distinctables[r].Field))
}
for i := 0; i < numChild; i++ {
tr.Chd = append(tr.Chd, cfg.GenTree(index, depth-1))
@ -356,11 +539,13 @@ func (tr *Tree) ToPQL() (s string) {
for _, c := range tr.Chd {
chds = append(chds, c.ToPQL())
}
// If we had no extra args, this does nothing.
chds = append(chds, tr.Args...)
all := strings.Join(chds, ", ")
return fmt.Sprintf("%v(%v)", tr.S, all)
}
func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowKey string, isRowKey bool) {
func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowKey string, isRowKey bool, hasTime bool) {
f, ok := cfg.IndexMap[index]
if !ok {
@ -373,5 +558,7 @@ func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowK
RowID: rowID,
RowKey: rowKey,
IsRowKey: isRowKey,
HasTime: hasTime,
})
f.SlcWeight++
}