From 48b97d6ad4028f1d9d738709dca76236c0d8cde9 Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 19 Nov 2020 12:52:47 -0600 Subject: [PATCH 1/4] report QPS after last query More generally, report QPS not at 0 queries, which is boring, but every 100 queries *and* after the last query if the last query wasn't at a multiple of 100 queries. Makes the output slightly more useful, I think. --- cmd/random-query/main.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 10fdfa6d5..7cfd0a636 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -133,6 +133,17 @@ func (cfg *RandomQueryConfig) Run() (err error) { loops := 0 t0 := time.Now() + report := func() { + dur := time.Since(t0) + if dur > 0 { + qps := 1e9 * float64(totalQ) / float64(dur) + AlwaysPrintf("totalQueries run: %v elapsed: %v qps: %0.02f", totalQ, dur, qps) + } else { + AlwaysPrintf("totalQueries run: %v elapsed: %v qps: N/A", totalQ, dur) + } + } + defer report() + NewSetup: err = cfg.Setup(cli) if err != nil { @@ -162,12 +173,8 @@ NewSetup: goto NewSetup } } - if totalQ%100 == 0 { - dur := time.Since(t0) - if dur > 0 { - qps := 1e9 * float64(totalQ) / float64(dur) - AlwaysPrintf("totalQueries run: %v elapsed: %v qps: %0.02f", totalQ, dur, qps) - } + if totalQ > 0 && totalQ%100 == 0 { + report() } index := indexes[rand.Intn(len(indexes))] @@ -190,6 +197,7 @@ NewSetup: } totalQ++ loops++ + } return nil From 2a2a8abbd4b043c814e199a3f2bbb5afaa461a75 Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 19 Nov 2020 14:03:39 -0600 Subject: [PATCH 2/4] support time fields For time fields, allow specifying a range of times, then 19/20 times, specify "from" and "to" times in that range when querying those fields, rather than just looking at the standard view all the time. --- cmd/random-query/main.go | 67 ++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 7cfd0a636..7af8dd6bc 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -36,6 +36,11 @@ type RandomQueryConfig struct { TreeDepth int // -d QueryCount int // -n Verbose bool // -v + TimeFromArg string // --time.from + TimeToArg string // --time.to + TimeFrom time.Time // parsed time + TimeTo time.Time // parsed time + TimeRange int64 // hours between parsed times IndexMap map[string]*Features @@ -76,12 +81,18 @@ func wrapApiToInternalClient(api *pilosa.API) *wrapper { return &wrapper{api: api} } +// These times are copied from the "kitchen sink" data generator to serve as defaults. +var defaultEndTime = time.Date(2020, time.May, 4, 12, 2, 28, 0, time.UTC) +var defaultStartTime = defaultEndTime.Add(-5 * 365 * 24 * time.Hour) + // call DefineFlags before myflags.Parse() func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.") fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.") fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") + fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") + fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") } // call c.ValidateConfig() after myflags.Parse() @@ -92,6 +103,20 @@ func (c *RandomQueryConfig) ValidateConfig() error { if c.QueryCount < 0 { return fmt.Errorf("-n count must be 0 or greater; saw %v", c.QueryCount) } + var err error + c.TimeFrom, err = time.Parse(time.RFC3339, c.TimeFromArg) + if err != nil { + return fmt.Errorf("-time.from value couldn't be parsed: %w", err) + } + c.TimeTo, err = time.Parse(time.RFC3339, c.TimeToArg) + if err != nil { + return fmt.Errorf("-time.to value couldn't be parsed: %w", err) + } + c.TimeRange = int64(c.TimeTo.Sub(c.TimeFrom).Hours()) + if c.TimeRange < 1 { + return fmt.Errorf("time.to (%s) should be at least one hour after time.from (%s)", + c.TimeToArg, c.TimeFromArg) + } return nil } @@ -219,6 +244,7 @@ type IndexFieldRow struct { RowID uint64 RowKey string IsRowKey bool + HasTime bool } // Run a RandomQuery takes a list of RowIDFeatures and ColumnKeyObjects @@ -235,7 +261,8 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { _ = i for k, fld := range ii.Fields { _ = k - if fld.Options.Type == "set" { + switch fld.Options.Type { + case "set", "mutex", "time": pql := fmt.Sprintf("Rows(%v)", fld.Name) res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql}) @@ -251,11 +278,17 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { switch x := res.Results[0].(type) { case *pilosa.RowIdentifiers: // internalClient gets this - cfg.AddResponse(ii.Name, fld.Name, x) + cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time") case pilosa.RowIdentifiers: // test gets this - cfg.AddResponse(ii.Name, fld.Name, &x) + cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time") } + case "int": + fmt.Printf("int field: details %#v\n", fld) + case "decimal": + fmt.Printf("decimal field: details %#v\n", fld) + default: + AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) } } } @@ -267,12 +300,12 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { return nil } -func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers) { +func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool) { for _, rowID := range x.Rows { - cfg.AddFeature(index, field, rowID, "", false) + cfg.AddFeature(index, field, rowID, "", false, hasTime) } for _, rowKey := range x.Keys { - cfg.AddFeature(index, field, 0, rowKey, true) + cfg.AddFeature(index, field, 0, rowKey, true, hasTime) } } @@ -323,16 +356,29 @@ func (tr *Tree) StringIndent(ind int) (s string) { return } +const pilosaTimeFmt = "2006-01-02T15:04" func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) { if depth == 0 { slc := cfg.IndexMap[index].Slc //vv("depth is 0, slc = '%#v'", slc) r := cfg.Rnd.Intn(len(slc)) fea := slc[r] - if fea.IsRowKey { - return &Tree{S: fmt.Sprintf("Row(%v='%v')", fea.Field, fea.RowKey)} + fromTo := "" + // 5% of queries on a time field will use the standard view + // anyway. + if slc[r].HasTime && rand.Int63n(20) != 0 { + startHours := (rand.Int63n(cfg.TimeRange - 1)) + endHours := rand.Int63n(cfg.TimeRange - startHours) + 1 + startHours + startTime := cfg.TimeFrom.Add(time.Duration(startHours) * time.Hour) + endTime := cfg.TimeFrom.Add(time.Duration(endHours) * time.Hour) + fromTo = fmt.Sprintf(", from=%s, to=%s", + startTime.Format(pilosaTimeFmt), + endTime.Format(pilosaTimeFmt)) } - return &Tree{S: fmt.Sprintf("Row(%v=%v)", fea.Field, fea.RowID)} + if fea.IsRowKey { + return &Tree{S: fmt.Sprintf("Row(%v='%v'%s)", fea.Field, fea.RowKey, fromTo)} + } + return &Tree{S: fmt.Sprintf("Row(%v=%v%s)", fea.Field, fea.RowID, fromTo)} } r := cfg.Rnd.Intn(len(cfg.BitmapFunc)) @@ -368,7 +414,7 @@ func (tr *Tree) ToPQL() (s string) { return fmt.Sprintf("%v(%v)", tr.S, all) } -func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowKey string, isRowKey bool) { +func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowKey string, isRowKey bool, hasTime bool) { f, ok := cfg.IndexMap[index] if !ok { @@ -381,5 +427,6 @@ func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowK RowID: rowID, RowKey: rowKey, IsRowKey: isRowKey, + HasTime: hasTime, }) } From 796d646a0ce4374c90839dcb80147b99e6a39137 Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 19 Nov 2020 16:14:23 -0600 Subject: [PATCH 3/4] handle integer fields Extend field handling to include int (and decimal) fields, allowing them to have range operations specified on them. --- cmd/random-query/main.go | 163 ++++++++++++++++++++++++++++++++------- 1 file changed, 136 insertions(+), 27 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 7af8dd6bc..8e80b9147 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -18,13 +18,16 @@ import ( "context" "flag" "fmt" + "math" "math/rand" nethttp "net/http" "os" + "strconv" "strings" "time" "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/http" ) @@ -36,6 +39,7 @@ type RandomQueryConfig struct { TreeDepth int // -d QueryCount int // -n Verbose bool // -v + VeryVerbose bool // -V TimeFromArg string // --time.from TimeToArg string // --time.to TimeFrom time.Time // parsed time @@ -91,6 +95,7 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.") fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") + fs.BoolVar(&cfg.VeryVerbose, "V", false, "show query results") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") } @@ -217,7 +222,7 @@ NewSetup: AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql) return err } - if cfg.Verbose { + if cfg.VeryVerbose { fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) } totalQ++ @@ -230,6 +235,20 @@ NewSetup: type Features struct { Slc []IndexFieldRow + Ranges []IndexFieldRange + SlcWeight int + RangeWeight int +} + +// Pick either a feature entry or a random query on a range, weighted +// by number of features and approximate weight of ranges +func (f *Features) RandomQuery(cfg *RandomQueryConfig) *Tree { + r := rand.Intn(f.SlcWeight + f.RangeWeight) + if r < f.SlcWeight { + return f.Slc[r].Query(cfg) + } + r = rand.Intn(len(f.Ranges)) + return f.Ranges[r].Query() } func NewRandomQueryConfig() *RandomQueryConfig { @@ -245,13 +264,81 @@ type IndexFieldRow struct { RowKey string IsRowKey bool HasTime bool + IsInt bool +} + +func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree { + fromTo := "" + // 5% of queries on a time field will use the standard view + // anyway. + if fea.HasTime && rand.Int63n(20) != 0 { + startHours := (rand.Int63n(cfg.TimeRange - 1)) + endHours := rand.Int63n(cfg.TimeRange - startHours) + 1 + startHours + startTime := cfg.TimeFrom.Add(time.Duration(startHours) * time.Hour) + endTime := cfg.TimeFrom.Add(time.Duration(endHours) * time.Hour) + fromTo = fmt.Sprintf(", from=%s, to=%s", + startTime.Format(pilosaTimeFmt), + endTime.Format(pilosaTimeFmt)) + } + if fea.IsRowKey { + return &Tree{S: fmt.Sprintf("Row(%v='%v'%s)", fea.Field, fea.RowKey, fromTo)} + } + return &Tree{S: fmt.Sprintf("Row(%v=%v%s)", fea.Field, fea.RowID, fromTo)} +} + +type IndexFieldRange struct { + Index string + Field string + Min, Max, Scale int64 + ScaleDiv float64 + Range uint64 +} + +// We want to pick one of (1) a single-operation filter, (2) a +// between-filter of some kind. +// So, that's one of <=, >=, ==, !=, >, <, or +// one of [<, <], [<, <=], [<=, <=], [<=, <]. +var binaryOps = []string{ + "<=", ">=", "==", "!=", "<", ">", +} + +func (i *IndexFieldRange) Query() *Tree { + r := rand.Int63n(10) + // this is unevenly weighted, but there's no Uint64N, and + // Int63n can't represent the whole range. + v1 := rand.Uint64() % i.Range + v2 := rand.Uint64() % i.Range + if v1 > v2 { + v1, v2 = v2, v1 + } + v1 = v1 + uint64(i.Min) + v2 = v2 + uint64(i.Min) + var v1s, v2s string + if i.Scale != 0 { + v1s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v1)) / i.ScaleDiv) + v2s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v2)) / i.ScaleDiv) + } else { + v1s = strconv.FormatInt(int64(v1), 10) + v2s = strconv.FormatInt(int64(v2), 10) + } + if r < 4 { + lte := "<=" + op1 := lte[:1+(r&1)] + op2 := lte[:1+((r>>1)&1)] + return &Tree{S: fmt.Sprintf("Row(%s %s %s %s %s)", + v1s, op1, i.Field, op2, v2s)} + } else { + if rand.Int63n(2) == 1 { + v1s = v2s + } + return &Tree{S: fmt.Sprintf("Row(%s %s %s)", i.Field, binaryOps[r - 4], v1s)} + } } // Run a RandomQuery takes a list of RowIDFeatures and ColumnKeyObjects // and spits back a PQL query // func (cfg *RandomQueryConfig) Setup(api API) (err error) { - ctx := context.Background() cfg.Info, err = api.Schema(ctx) if err != nil { @@ -267,7 +354,7 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql}) panicOn(err) - if cfg.Verbose { + if cfg.VeryVerbose { fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) } // if the option is set to use RowKeys, then must get the Keys instead of the Rows from the RowIdentifiers. @@ -283,10 +370,9 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { // test gets this cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time") } - case "int": - fmt.Printf("int field: details %#v\n", fld) - case "decimal": - fmt.Printf("decimal field: details %#v\n", fld) + case "int", "decimal": + // we'll ignore row keys and just use value ranges + cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale) default: AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) } @@ -309,6 +395,47 @@ func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIden } } +const maxEffectiveRange = 1000000 + +func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Decimal, scale int64) { + f, ok := cfg.IndexMap[index] + if !ok { + f = &Features{} + cfg.IndexMap[index] = f + } + if min.Scale != scale || max.Scale != scale { + panic(fmt.Sprintf("scale error; min scale %d, max scale %d, field scale %d, assumed they'd be equal", + min.Scale, max.Scale, scale)) + } + + effectiveRange := uint64(max.Value) - uint64(min.Value) + 1 + // if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which + // wraps to 0. Anything closer together will be fine. We accept the loss + // of accuracy in the range from not representing quite the full value + // in that edge case. + if effectiveRange == 0 { + effectiveRange-- + } + + // we assume that the Value of the field is already scaled, I guess? + f.Ranges = append(f.Ranges, IndexFieldRange{ + Index: index, + Field: field, + Min: min.Value, + Max: max.Value, + Scale: scale, + ScaleDiv: math.Pow(10, float64(scale)), + Range: effectiveRange, + }) + // We want to add more values for larger int fields, but the + // default KitchenSink field has a range of 1<<64 which would make + // it completely dominate weights, so... + if effectiveRange > maxEffectiveRange { + effectiveRange = maxEffectiveRange + } + f.RangeWeight += int(effectiveRange) +} + func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) { tree := cfg.GenTree(index, cfg.TreeDepth) @@ -359,26 +486,7 @@ func (tr *Tree) StringIndent(ind int) (s string) { const pilosaTimeFmt = "2006-01-02T15:04" func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) { if depth == 0 { - slc := cfg.IndexMap[index].Slc - //vv("depth is 0, slc = '%#v'", slc) - r := cfg.Rnd.Intn(len(slc)) - fea := slc[r] - fromTo := "" - // 5% of queries on a time field will use the standard view - // anyway. - if slc[r].HasTime && rand.Int63n(20) != 0 { - startHours := (rand.Int63n(cfg.TimeRange - 1)) - endHours := rand.Int63n(cfg.TimeRange - startHours) + 1 + startHours - startTime := cfg.TimeFrom.Add(time.Duration(startHours) * time.Hour) - endTime := cfg.TimeFrom.Add(time.Duration(endHours) * time.Hour) - fromTo = fmt.Sprintf(", from=%s, to=%s", - startTime.Format(pilosaTimeFmt), - endTime.Format(pilosaTimeFmt)) - } - if fea.IsRowKey { - return &Tree{S: fmt.Sprintf("Row(%v='%v'%s)", fea.Field, fea.RowKey, fromTo)} - } - return &Tree{S: fmt.Sprintf("Row(%v=%v%s)", fea.Field, fea.RowID, fromTo)} + return cfg.IndexMap[index].RandomQuery(cfg) } r := cfg.Rnd.Intn(len(cfg.BitmapFunc)) @@ -429,4 +537,5 @@ func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowK IsRowKey: isRowKey, HasTime: hasTime, }) + f.SlcWeight++ } From 3d802f428b577f46f90526c1edf26475ed3d9e18 Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 20 Nov 2020 11:40:38 -0600 Subject: [PATCH 4/4] add Distinct, be more consistent about using correct rand.Rand The RandomQueryConfig has its own seeded RNG, but we didn't always use it (especially in the last two commits, but also I think in one previous thing), so let's use it more consistently. We check for int fields (as opposed to decimal), and if we find them, we add Distinct to our list of potential queries to use if and only if we've got a depth of at least one so there'd be a child query under the current query, and if we do, grab one of the int (not decimal) fields and do a query on that. --- cmd/random-query/main.go | 61 +++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 8e80b9147..19ea4e5d1 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -207,7 +207,7 @@ NewSetup: report() } - index := indexes[rand.Intn(len(indexes))] + index := indexes[cfg.Rnd.Intn(len(indexes))] pql, err := cfg.GenQuery(index) panicOn(err) @@ -236,6 +236,7 @@ NewSetup: type Features struct { Slc []IndexFieldRow Ranges []IndexFieldRange + Distinctables []IndexFieldRange SlcWeight int RangeWeight int } @@ -243,12 +244,12 @@ type Features struct { // Pick either a feature entry or a random query on a range, weighted // by number of features and approximate weight of ranges func (f *Features) RandomQuery(cfg *RandomQueryConfig) *Tree { - r := rand.Intn(f.SlcWeight + f.RangeWeight) + r := cfg.Rnd.Intn(f.SlcWeight + f.RangeWeight) if r < f.SlcWeight { return f.Slc[r].Query(cfg) } - r = rand.Intn(len(f.Ranges)) - return f.Ranges[r].Query() + r = cfg.Rnd.Intn(len(f.Ranges)) + return f.Ranges[r].Query(cfg) } func NewRandomQueryConfig() *RandomQueryConfig { @@ -271,9 +272,9 @@ func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree { fromTo := "" // 5% of queries on a time field will use the standard view // anyway. - if fea.HasTime && rand.Int63n(20) != 0 { - startHours := (rand.Int63n(cfg.TimeRange - 1)) - endHours := rand.Int63n(cfg.TimeRange - startHours) + 1 + startHours + if fea.HasTime && cfg.Rnd.Int63n(20) != 0 { + startHours := (cfg.Rnd.Int63n(cfg.TimeRange - 1)) + endHours := cfg.Rnd.Int63n(cfg.TimeRange - startHours) + 1 + startHours startTime := cfg.TimeFrom.Add(time.Duration(startHours) * time.Hour) endTime := cfg.TimeFrom.Add(time.Duration(endHours) * time.Hour) fromTo = fmt.Sprintf(", from=%s, to=%s", @@ -302,12 +303,12 @@ var binaryOps = []string{ "<=", ">=", "==", "!=", "<", ">", } -func (i *IndexFieldRange) Query() *Tree { - r := rand.Int63n(10) +func (i *IndexFieldRange) Query(cfg *RandomQueryConfig) *Tree { + r := cfg.Rnd.Int63n(10) // this is unevenly weighted, but there's no Uint64N, and // Int63n can't represent the whole range. - v1 := rand.Uint64() % i.Range - v2 := rand.Uint64() % i.Range + v1 := cfg.Rnd.Uint64() % i.Range + v2 := cfg.Rnd.Uint64() % i.Range if v1 > v2 { v1, v2 = v2, v1 } @@ -328,7 +329,7 @@ func (i *IndexFieldRange) Query() *Tree { return &Tree{S: fmt.Sprintf("Row(%s %s %s %s %s)", v1s, op1, i.Field, op2, v2s)} } else { - if rand.Int63n(2) == 1 { + if cfg.Rnd.Int63n(2) == 1 { v1s = v2s } return &Tree{S: fmt.Sprintf("Row(%s %s %s)", i.Field, binaryOps[r - 4], v1s)} @@ -344,6 +345,7 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { if err != nil { return err } + foundIntField := false for i, ii := range cfg.Info { _ = i for k, fld := range ii.Fields { @@ -370,9 +372,12 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { // test gets this cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time") } - case "int", "decimal": + case "int": + foundIntField = true + fallthrough // I bet you thought you'd never see this used + case "decimal": // we'll ignore row keys and just use value ranges - cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale) + cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal") default: AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) } @@ -380,6 +385,9 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { } cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"} + if foundIntField { + cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct") + } seed := int64(42) cfg.Rnd = rand.New(rand.NewSource(seed)) @@ -397,7 +405,7 @@ func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIden const maxEffectiveRange = 1000000 -func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Decimal, scale int64) { +func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Decimal, scale int64, decimal bool) { f, ok := cfg.IndexMap[index] if !ok { f = &Features{} @@ -418,7 +426,7 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci } // we assume that the Value of the field is already scaled, I guess? - f.Ranges = append(f.Ranges, IndexFieldRange{ + newRange := IndexFieldRange{ Index: index, Field: field, Min: min.Value, @@ -426,7 +434,11 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci Scale: scale, ScaleDiv: math.Pow(10, float64(scale)), Range: effectiveRange, - }) + } + f.Ranges = append(f.Ranges, newRange) + if !decimal { + f.Distinctables = append(f.Distinctables, newRange) + } // We want to add more values for larger int fields, but the // default KitchenSink field has a range of 1<<64 which would make // it completely dominate weights, so... @@ -452,8 +464,8 @@ func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) { type Tree struct { Chd []*Tree - S string + Args []string // Extra args to pass after children, such as a field for Distinct. } func (tr *Tree) StringIndent(ind int) (s string) { @@ -485,8 +497,9 @@ func (tr *Tree) StringIndent(ind int) (s string) { const pilosaTimeFmt = "2006-01-02T15:04" func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) { + features := cfg.IndexMap[index] if depth == 0 { - return cfg.IndexMap[index].RandomQuery(cfg) + return features.RandomQuery(cfg) } r := cfg.Rnd.Intn(len(cfg.BitmapFunc)) @@ -500,6 +513,14 @@ func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) { numChild = 1 case "Difference": numChild = 2 + case "Distinct": + numChild = 1 + // sometimes do a bare distinct without a filter + if cfg.Rnd.Intn(10) == 0 { + numChild = 0 + } + r = cfg.Rnd.Intn(len(features.Distinctables)) + tr.Args = append(tr.Args, fmt.Sprintf("field=%s", features.Distinctables[r].Field)) } for i := 0; i < numChild; i++ { tr.Chd = append(tr.Chd, cfg.GenTree(index, depth-1)) @@ -518,6 +539,8 @@ func (tr *Tree) ToPQL() (s string) { for _, c := range tr.Chd { chds = append(chds, c.ToPQL()) } + // If we had no extra args, this does nothing. + chds = append(chds, tr.Args...) all := strings.Join(chds, ", ") return fmt.Sprintf("%v(%v)", tr.S, all) }