From 87ededd61e55ef88ba7973d050d1ae88679ec705 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 18 Nov 2021 16:23:12 -0600 Subject: [PATCH 01/14] Modified random query to use vegeta library --- .gitlab/cloud-init.sh | 6 +- cmd/random-query/main.go | 276 ++++++++++++++++++++-------------- cmd/random-query/main_test.go | 8 +- go.mod | 1 + go.sum | 41 +++++ 5 files changed, 210 insertions(+), 122 deletions(-) diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index 055d0f6aa..8d710d0b2 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -14,8 +14,8 @@ echo 'cat /proc/sys/fs/file-max' # yum install golang -y # latest verion in ec2 is 1.15.14 # install go 1.16.9 manually -curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz -tar xvf go1.16.9.linux-amd64.tar.gz +curl -O https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz +tar xvf go1.16.10.linux-amd64.tar.gz chown -R root:root ./go mv go /usr/local echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null @@ -23,4 +23,4 @@ source /etc/profile # install aws session manager pluggin curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" -yum install -y session-manager-plugin.rpm \ No newline at end of file +yum install -y session-manager-plugin.rpm diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 1faafad1e..5c3bc4ee0 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -18,34 +18,47 @@ import ( "context" "flag" "fmt" + "io/ioutil" "math" "math/rand" nethttp "net/http" + "os" "strconv" "strings" "time" - "github.com/molecula/featurebase/v2" + "github.com/gogo/protobuf/proto" + pilosa "github.com/molecula/featurebase/v2" + "github.com/molecula/featurebase/v2/client" "github.com/molecula/featurebase/v2/http" + "github.com/molecula/featurebase/v2/pb" "github.com/molecula/featurebase/v2/pql" + "github.com/molecula/featurebase/v2/vprint" . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck + "github.com/pkg/errors" + vegeta "github.com/tsenart/vegeta/v12/lib" ) // RandomQueryConfig type RandomQueryConfig struct { // user facing flags - HostPort string // -hostport - TreeDepth int // -d - QueryCount int // -n - Verbose bool // -v - VeryVerbose bool // -V + HostPort string // -hostport + TreeDepth int // -d + QueryCount int // -n + Verbose bool // -v + NumRuns int TimeFromArg string // --time.from TimeToArg string // --time.to TimeFrom time.Time // parsed time TimeTo time.Time // parsed time TimeRange int64 // hours between parsed times + Index string + QPS int + SrcFile string + Duration time.Duration + Target vegeta.Target IndexMap map[string]*Features @@ -93,12 +106,16 @@ var defaultStartTime = defaultEndTime.Add(-5 * 365 * 24 * time.Hour) // call DefineFlags before myflags.Parse() func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.") - fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.") - fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.") + fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.") + fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate") + fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") + fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") + fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") + fs.IntVar(&cfg.QPS, "qps", 10, "number of currernt requests per sec to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") - fs.BoolVar(&cfg.VeryVerbose, "V", false, "show query results") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") + fs.StringVar(&cfg.SrcFile, "query-file", "", "use pql contained in this file for query batch instead of generating") } // call c.ValidateConfig() after myflags.Parse() @@ -144,7 +161,6 @@ func main() { fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err) os.Exit(1) } - err = cfg.Run() if err != nil { @@ -159,76 +175,24 @@ func (cfg *RandomQueryConfig) Run() (err error) { if err != nil { return err } - ctx := context.Background() - totalQ := 0 - loops := 0 - t0 := time.Now() - - report := func() { - dur := time.Since(t0) - if dur > 0 { - qps := 1e9 * float64(totalQ) / float64(dur) - AlwaysPrintf("totalQueries run: %v elapsed: %v qps: %0.02f", totalQ, dur, qps) - } else { - AlwaysPrintf("totalQueries run: %v elapsed: %v qps: N/A", totalQ, dur) - } - } - defer report() - -NewSetup: err = cfg.Setup(cli) if err != nil { return err } + rate := vegeta.Rate{Freq: cfg.QPS, Per: time.Second} + duration := cfg.Duration + targeter := vegeta.NewStaticTargeter(cfg.Target) + attacker := vegeta.NewAttacker() - if len(cfg.IndexMap) == 0 { - return fmt.Errorf("no rows to query") - } - - var indexes []string - for index := range cfg.IndexMap { - indexes = append(indexes, index) - } - - for j := 0; ; j++ { - if cfg.QueryCount > 0 { - if j >= cfg.QueryCount { - break - } - } else { - // else keep doing queries forever... - if loops > 0 && loops%500 == 0 { - // ...but account for any new data arrived by getting - // the schema and rows again every so often. - loops++ - goto NewSetup - } + for i := 0; i < cfg.NumRuns; i++ { + vprint.VV("================") + var metrics vegeta.Metrics + for res := range attacker.Attack(targeter, rate, duration, "Big Bang!") { + metrics.Add(res) } - if totalQ > 0 && totalQ%100 == 0 { - report() - } - - index := indexes[cfg.Rnd.Intn(len(indexes))] - - pql, err := cfg.GenQuery(index) - PanicOn(err) - - if cfg.Verbose { - fmt.Printf("pql = '%v'\n", pql) - } - - // Query node0. - res, err := cli.Query(ctx, index, &pilosa.QueryRequest{Index: index, Query: pql}) - if err != nil { - AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql) - return err - } - if cfg.VeryVerbose { - fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) - } - totalQ++ - loops++ - + metrics.Close() + rpt := vegeta.NewTextReporter(&metrics) + rpt(os.Stdout) } return nil @@ -238,6 +202,7 @@ type Features struct { Slc []IndexFieldRow Ranges []IndexFieldRange Distinctables []IndexFieldRange + Stores []IndexFieldRow SlcWeight int RangeWeight int } @@ -283,7 +248,7 @@ func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree { endTime.Format(pilosaTimeFmt)) } if fea.IsRowKey { - return &Tree{S: fmt.Sprintf("Row(%v='%v'%s)", fea.Field, fea.RowKey, fromTo)} + return &Tree{S: fmt.Sprintf(`Row(%v="%v"%s)`, fea.Field, fea.RowKey, fromTo)} } return &Tree{S: fmt.Sprintf("Row(%v=%v%s)", fea.Field, fea.RowID, fromTo)} } @@ -341,6 +306,9 @@ func (i *IndexFieldRange) Query(cfg *RandomQueryConfig) *Tree { // and spits back a PQL query // func (cfg *RandomQueryConfig) Setup(api API) (err error) { + if cfg.SrcFile != "" { + return cfg.buildPayload() + } ctx := context.Background() cfg.Info, err = api.Schema(ctx) if err != nil { @@ -348,58 +316,114 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { } foundIntField := false for i, ii := range cfg.Info { - _ = i - for k, fld := range ii.Fields { - _ = k - switch fld.Options.Type { - case "set", "mutex", "time": - pql := fmt.Sprintf("Rows(%v)", fld.Name) + if ii.Name == cfg.Index { - res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql}) - PanicOn(err) - if cfg.VeryVerbose { - fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) - } - // if the option is set to use RowKeys, then must get the Keys instead of the Rows from the RowIdentifiers. - // e.g. - // success on pql = 'Rows(aba)'; res='&pilosa.RowIdentifiers{Rows:[]uint64(nil), Keys:[]string{"aba1", "aba2"} - // success on pql = 'Rows(f)'; res='pilosa.RowIdentifiers{Rows:[]uint64{0x1}, Keys:[]string(nil), field:"f"}' + _ = i + for k, fld := range ii.Fields { + _ = k + switch fld.Options.Type { + case "set", "mutex", "time": + pql := fmt.Sprintf("Rows(%v)", fld.Name) - switch x := res.Results[0].(type) { - case *pilosa.RowIdentifiers: - // internalClient gets this - cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time") - case pilosa.RowIdentifiers: - // test gets this - cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time") + res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql}) + PanicOn(err) + switch x := res.Results[0].(type) { + case *pilosa.RowIdentifiers: + cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time", fld.Options.Type == "set") + case pilosa.RowIdentifiers: + cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time", fld.Options.Type == "set") + } + case "int": + foundIntField = true + fallthrough + case "decimal": + cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal") + default: + AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) } - case "int": - foundIntField = true - fallthrough // I bet you thought you'd never see this used - case "decimal": - cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal") - default: - AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) } } } - cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"} if foundIntField { cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct") } seed := int64(42) cfg.Rnd = rand.New(rand.NewSource(seed)) - - return nil + return cfg.buildPayload() } -func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool) { +func (cfg *RandomQueryConfig) buildPayload() error { + var request strings.Builder + if cfg.SrcFile != "" { + b, err := ioutil.ReadFile(cfg.SrcFile) // just pass the file name + if err != nil { + return err + } + request.WriteString(string(b)) + } else { + + for i := 0; i < cfg.QueryCount; i++ { + pql, err := cfg.GenQuery(cfg.Index) + if err != nil { + return err + } + request.WriteString(pql) + } + } + //TODO (twg) tls support + path := fmt.Sprintf("http://%s/index/%s/query", cfg.HostPort, cfg.Index) + header := nethttp.Header{ + "Content-Type": []string{"application/x-protobuf"}, + "Accept": []string{"application/x-protobuf"}, + "PQL-Version": []string{client.PQLVersion}, + } + + req := &pb.QueryRequest{ + Query: request.String(), + } + vprint.VV("%v", request.String()) + payload, err := proto.Marshal(req) + if err != nil { + return errors.Wrap(err, "marshaling request to protobuf") + } + cfg.Target = vegeta.Target{ + Method: "POST", + URL: path, + Body: payload, + Header: header, + } + return nil +} +func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool, isSet bool) { + storeID := uint64(0) for _, rowID := range x.Rows { cfg.AddFeature(index, field, rowID, "", false, hasTime) + storeID = rowID } + storeKey := "" for _, rowKey := range x.Keys { cfg.AddFeature(index, field, 0, rowKey, true, hasTime) + storeKey = rowKey + } + idx := cfg.IndexMap[index] + if isSet { + if storeID > 0 { + idx.Stores = append(idx.Stores, IndexFieldRow{ + Index: index, + Field: field, + RowID: storeID + 1, + }) + } + if storeKey != "" { + idx.Stores = append(idx.Stores, IndexFieldRow{ + Index: index, + Field: field, + RowKey: storeKey + "_1", + IsRowKey: true, + }) + + } } } @@ -411,10 +435,12 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci f = &Features{} cfg.IndexMap[index] = f } - if min.Scale != scale || max.Scale != scale { - PanicOn(fmt.Sprintf("scale error; min scale %d, max scale %d, field scale %d, assumed they'd be equal", - min.Scale, max.Scale, scale)) - } + /* if min.Scale != scale || max.Scale != scale { + + PanicOn(fmt.Sprintf("scale error; %v:%v min scale %d, max scale %d, field scale %d, assumed they'd be equal", + index, field, min.Scale, max.Scale, scale)) + } + */ effectiveRange := uint64(max.Value) - uint64(min.Value) + 1 // if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which @@ -449,11 +475,32 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci } func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) { - tree := cfg.GenTree(index, cfg.TreeDepth) pql = tree.ToPQL() - // avoid using too much bandwidth, just count the final bitmap. + dice := cfg.Rnd.Intn(9) + if dice == 3 { // 1 in 9 of getting a store + idx := cfg.IndexMap[index] + if len(idx.Stores) > 0 { + i := cfg.Rnd.Intn(len(idx.Stores)) + fr := idx.Stores[i] + var key string + if fr.IsRowKey { + key = fmt.Sprintf(`%v="%v"`, fr.Field, fr.RowKey) + c := strings.LastIndex(fr.RowKey, "_") + n, err := strconv.Atoi(fr.RowKey[c+1:]) + PanicOn(err) + n += 1 + fr.RowKey = fmt.Sprintf("%v%v", fr.RowKey[:c+1], n) + } else { + key = fmt.Sprintf("%v=%v", fr.Field, fr.RowID) + fr.RowID = fr.RowID + 1 + } + idx.Stores[i] = fr + pql = fmt.Sprintf("Store(%v,%v)Count(Row(%v))", pql, key, key) + return + } + } pql = fmt.Sprintf("Count(%v)", pql) return } @@ -522,7 +569,6 @@ func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) { } func (tr *Tree) ToPQL() (s string) { - if len(tr.Chd) == 0 { // leaf return tr.S diff --git a/cmd/random-query/main_test.go b/cmd/random-query/main_test.go index eb6cc9040..7ad6936fa 100644 --- a/cmd/random-query/main_test.go +++ b/cmd/random-query/main_test.go @@ -20,7 +20,7 @@ import ( "strconv" "testing" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/boltdb" "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/server" @@ -65,8 +65,8 @@ func Test_RandomQuery(t *testing.T) { ctx := context.Background() - indexes := []string{"rick", "morty"} - fieldName := []string{"f", "flying_car"} + indexes := []string{"rick"} + fieldName := []string{"f"} idx := make([]*pilosa.Index, len(indexes)) field := make([]*pilosa.Field, len(indexes)) @@ -144,7 +144,7 @@ func Test_RandomQuery(t *testing.T) { //qcx.Reset() } // end of setup. - + cfg.Index = indexes[0] PanicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API))) for j := 0; j < 4; j++ { diff --git a/go.mod b/go.mod index 53a0860d3..add6082e3 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.7.0 + github.com/tsenart/vegeta/v12 v12.8.4 github.com/uber/jaeger-client-go v2.25.0+incompatible github.com/uber/jaeger-lib v2.4.0+incompatible // indirect github.com/zeebo/blake3 v0.1.1 diff --git a/go.sum b/go.sum index 0b9624b38..4224f965d 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,7 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/jsonschema v0.0.0-20180308105923-f2c93856175a/go.mod h1:qpebaTNSsyUn5rPSJMsfqEtDw71TTggXM6stUDI16HA= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -42,8 +43,11 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE= +github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -75,6 +79,9 @@ github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051 h1:ByJUvQYyTtNNCVfYNM48q6uYUT4fAlN0wNmd3th4BSo= +github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051/go.mod h1:qm+vckxRlDt0aOla0RYJJVeqHZlWfOm2UIxHaqPB46E= +github.com/dgryski/go-lttb v0.0.0-20180810165845-318fcdf10a77/go.mod h1:Va5MyIzkU0rAM92tn3hb3Anb7oz7KcnixF49+2wOMe4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= @@ -120,6 +127,24 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac h1:Q0Jsdxl5jbxouNs1TQYt0gxesYMU4VXRbsTlgDloZ50= +github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc= +github.com/gonum/diff v0.0.0-20181124234638-500114f11e71 h1:BE6g8oinc3Ek2elIHq+uDOiZgX3/ODi+EerJ48yrrKc= +github.com/gonum/diff v0.0.0-20181124234638-500114f11e71/go.mod h1:22dM4PLscQl+Nzf64qNBurVJvfyvZELT0iRW2l/NN70= +github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82 h1:EvokxLQsaaQjcWVWSV38221VAK7qc2zhaO17bKys/18= +github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82/go.mod h1:PxC8OnwL11+aosOB5+iEPoV3picfs8tUpkVd0pDo+Kg= +github.com/gonum/integrate v0.0.0-20181209220457-a422b5c0fdf2 h1:GUSkTcIe1SlregbHNUKbYDhBsS8lNgYfIp4S4cToUyU= +github.com/gonum/integrate v0.0.0-20181209220457-a422b5c0fdf2/go.mod h1:pDgmNM6seYpwvPos3q+zxlXMsbve6mOIPucUnUOrI7Y= +github.com/gonum/internal v0.0.0-20181124074243-f884aa714029 h1:8jtTdc+Nfj9AR+0soOeia9UZSvYBvETVHZrugUowJ7M= +github.com/gonum/internal v0.0.0-20181124074243-f884aa714029/go.mod h1:Pu4dmpkhSyOzRwuXkOgAvijx4o+4YMUJJo9OvPYMkks= +github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9 h1:7qnwS9+oeSiOIsiUMajT+0R7HR6hw5NegnKPmn/94oI= +github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9/go.mod h1:XA3DeT6rxh2EAE789SSiSJNqxPaC0aE9J8NTOI0Jo/A= +github.com/gonum/mathext v0.0.0-20181121095525-8a4bf007ea55 h1:Ajwn2ENgC/pKtVat0LEHEWNa4a4VGyYJ1feGSccOzFU= +github.com/gonum/mathext v0.0.0-20181121095525-8a4bf007ea55/go.mod h1:fmo8aiSEWkJeiGXUJf+sPvuDgEFgqIoZSs843ePKrGg= +github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9 h1:V2IgdyerlBa/MxaEFRbV5juy/C3MGdj4ePi+g6ePIp4= +github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9/go.mod h1:0EXg4mc1CNP0HCqCz+K4ts155PXIlUywf0wqN+GfPZw= +github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b h1:fbskpz/cPqWH8VqkQ7LJghFkl2KPAiIFUHrTJ2O3RGk= +github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b/go.mod h1:Z4GIJBJO3Wa4gD4vbwQxXXZ+WHmW6E9ixmNrwvs0iZs= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -184,6 +209,8 @@ github.com/improbable-eng/grpc-web v0.13.0 h1:7XqtaBWaOCH0cVGKHyvhtcuo6fgW32Y10y github.com/improbable-eng/grpc-web v0.13.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a h1:vMqgISSVkIqWxCIZs8m1L4096temR7IbYyNdMiBxSPA= +github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a/go.mod h1:9GkyshztGufsdPQWjH+ifgnIr3xNUL5syI70g2dzU1o= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -210,6 +237,8 @@ github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -217,6 +246,7 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.17/go.mod h1:WgzbA6oji13JREwiNsRDNfl7jYdPnmz+VEuLrA+/48M= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -321,6 +351,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 h1:7z3LSn867ex6VSaahyKadf4WtSsJIgne6A1WLOAGM8A= +github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -338,6 +370,9 @@ github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcy github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tsenart/go-tsz v0.0.0-20180814232043-cdeb9e1e981e/go.mod h1:SWZznP1z5Ki7hDT2ioqiFKEse8K9tU2OUvaRI0NeGQo= +github.com/tsenart/vegeta/v12 v12.8.4 h1:UQ7tG7WkDorKj0wjx78Z4/vsMBP8RJQMGJqRVrkvngg= +github.com/tsenart/vegeta/v12 v12.8.4/go.mod h1:ZiJtwLn/9M4fTPdMY7bdbIeyNeFVE8/AHbWFqCsUuho= github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ= @@ -367,6 +402,7 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -416,6 +452,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= @@ -446,6 +483,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -486,6 +524,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -559,6 +598,8 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +pgregory.net/rapid v0.3.3 h1:jCjBsY4ln4Atz78QoBWxUEvAHaFyNDQg9+WU62aCn1U= +pgregory.net/rapid v0.3.3/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From f6b3a81ac3694b87fe0b8f460c83f90576768e68 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 05:16:30 -0600 Subject: [PATCH 02/14] made duration minutes for query rate --- cmd/random-query/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 5c3bc4ee0..22f5209ad 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -55,7 +55,7 @@ type RandomQueryConfig struct { TimeTo time.Time // parsed time TimeRange int64 // hours between parsed times Index string - QPS int + QPM int SrcFile string Duration time.Duration Target vegeta.Target @@ -111,7 +111,7 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") - fs.IntVar(&cfg.QPS, "qps", 10, "number of currernt requests per sec to simulate, default 10") + fs.IntVar(&cfg.QPM, "qps", 10, "number of currernt requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") @@ -179,7 +179,7 @@ func (cfg *RandomQueryConfig) Run() (err error) { if err != nil { return err } - rate := vegeta.Rate{Freq: cfg.QPS, Per: time.Second} + rate := vegeta.Rate{Freq: cfg.QPM, Per: time.Second} duration := cfg.Duration targeter := vegeta.NewStaticTargeter(cfg.Target) attacker := vegeta.NewAttacker() From b3b11638a32d6b80cba4406e2c3d328170b6355c Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 9 Nov 2021 08:23:28 -0600 Subject: [PATCH 03/14] remove outdated files at top level we're no longer Apache 2.0 licensed, or open source, so LICENSE and CONTRIBUTING.MD are gone. We track the changelog elsewhere, so that can go, and I don't think anyone has looked at the NOTES file in 3 years. I modified the NOTICE not to refer to the Apache license any more. --- CHANGELOG.md | 715 ------------------------------------------------ CONTRIBUTING.md | 177 ------------ LICENSE | 202 -------------- NOTES | 26 -- 4 files changed, 1120 deletions(-) delete mode 100644 CHANGELOG.md delete mode 100644 CONTRIBUTING.md delete mode 100644 LICENSE delete mode 100644 NOTES diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 57de1f746..000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,715 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/) -and this project adheres to [Semantic Versioning](http://semver.org/). - -## [1.4.0] - 2019-09-17 - -This version contains 99 contributions from 11 contributors. There are 94 files changed; 9,453 insertions; and 6,121 deletions. - -**Attention**: Pilosa 1.4.0 changes the way that integer fields are stored. The upgrade from old format to new is handled automatically, however you will not be able to downgrade to 1.3 should you wish to do so. We *always* recommend taking a backup of your Pilosa data directory before upgrading Pilosa, but doubly so with this release. - -### Added -- Update "Getting Started" documentation ([#2028](https://github.com/pilosa/pilosa/pull/2028)) -- Add ability to disable tracing and use nopTracer ([#2029](https://github.com/pilosa/pilosa/pull/2029)) -- Add test for no containers ([#2016](https://github.com/pilosa/pilosa/pull/2016)) -- Add naive implementations of Roaring and fuzz test ([#2023](https://github.com/pilosa/pilosa/pull/2023)) -- Add fuzzing code and readme.md to explain the fuzzer ([#2004](https://github.com/pilosa/pilosa/pull/2004)) -- Add MinRow and MaxRow calls ([#1983](https://github.com/pilosa/pilosa/pull/1983)) -- Add Prometheus stats backend ([#1992](https://github.com/pilosa/pilosa/pull/1992)) -- Add extra tracing spans and metadata ([#1939](https://github.com/pilosa/pilosa/pull/1939)) -- Add more Debugf() statements to the holder open process ([#1950](https://github.com/pilosa/pilosa/pull/1950)) -- Add ability to post schema using holder.applySchema ([#1956](https://github.com/pilosa/pilosa/pull/1956)) - -### Changed -- Update CircleCI build with Go 1.13 and run enterprise tests ([#2064](https://github.com/pilosa/pilosa/pull/2064)) -- Update Alpine to 3.9.4 in Dockerfile ([#2001](https://github.com/pilosa/pilosa/pull/2001)) -- Add Prometheus tests, refactor http stats as middleware, minor fixes ([#1994](https://github.com/pilosa/pilosa/pull/1994)) -- Add confirmation logic to catch false nodeLeave events ([#1993](https://github.com/pilosa/pilosa/pull/1993)) -- Improve TopN() errors ([#1978](https://github.com/pilosa/pilosa/pull/1978)) -- Make integer fields unbounded by using sign+magnitude representation ([#1902](https://github.com/pilosa/pilosa/pull/1902)) -- Simplify contributing instructions by removing weird upstream thing ([#1966](https://github.com/pilosa/pilosa/pull/1966)) - -### Fixed -- Default BSI base value to min, max, or 0 depending on the min/max range ([#2050](https://github.com/pilosa/pilosa/pull/2050)) -- Add worker pool for query processing ([#2034](https://github.com/pilosa/pilosa/pull/2034)) -- Move Range deprecation message to higher level ([#2033](https://github.com/pilosa/pilosa/pull/2033)) -- Use lock in view.deleteFragment while altering fragments ([#2026](https://github.com/pilosa/pilosa/pull/2026)) -- Fix malformed offset bug in readOffsets and readWithRuns ([#2021](https://github.com/pilosa/pilosa/pull/2021)) -- Fix various container iteration bugs in Roaring ([#2019](https://github.com/pilosa/pilosa/pull/2019)) -- Fix malformed bitmap handling ([#2017](https://github.com/pilosa/pilosa/pull/2017)) -- Fix fuzzer errors in roaring ([#2012](https://github.com/pilosa/pilosa/pull/2012)) -- Save all state files atomically to avoid corruption ([#2000](https://github.com/pilosa/pilosa/pull/2000)) -- Fix slice container updates ([#1997](https://github.com/pilosa/pilosa/pull/1997)) -- Fix out of bounds panic to show error ([#1975](https://github.com/pilosa/pilosa/pull/1975)) -- Fix error message returned by regex on field and index names ([#1973](https://github.com/pilosa/pilosa/pull/1973)) -- Fix filter calls in GroupBy not being translated ([#1970](https://github.com/pilosa/pilosa/pull/1970)) -- Fix TranslateFile behavior when reopened ([#1954](https://github.com/pilosa/pilosa/pull/1954)) -- Remove buggy shard validation code ([#1951](https://github.com/pilosa/pilosa/pull/1951)) -- Fix some lint warnings raised in VS-Code ([#1947](https://github.com/pilosa/pilosa/pull/1947)) - -### Performance -- Address some startup speed and performance issues ([#1988](https://github.com/pilosa/pilosa/pull/1988)) -- Add a worker pool for importRoaring jobs ([#2048](https://github.com/pilosa/pilosa/pull/2048)) -- Use UnionInPlace for computing time rows which involve multiple views ([#2041](https://github.com/pilosa/pilosa/pull/2041)) -- Improve ingest performance with snapshot queue and unmarshaling improvements ([#2024](https://github.com/pilosa/pilosa/pull/2024)) -- Improve row cache ([#1974](https://github.com/pilosa/pilosa/pull/1974)) - -### Removed -- Remove extraneous stat tags to improve prometheus performance ([#1996](https://github.com/pilosa/pilosa/pull/1996)) - -## [1.3.1] - 2019-05-01 - -This version contains 1 contribution from 1 contributor. There are 6 files changed; 10 insertions; and 95 deletions. - -### Fixed -- Remove shard validation to fix bug where some nodes weren't loading their fragments. #1951 ([#1964](https://github.com/pilosa/pilosa/pull/1964)) - -## [1.3.0] - 2019-04-16 - -This version contains 98 contributions from 10 contributors. There are 144 files changed; 12,635 insertions; and 4,341 deletions. - -### Added -- Add license headers and CI check ([#1940](https://github.com/pilosa/pilosa/pull/1940)) -- Add support to modify shard width at build time ([#1921](https://github.com/pilosa/pilosa/pull/1921)) -- Add 'bench' Makefile target and run fewer concurrency level benchmarks ([#1915](https://github.com/pilosa/pilosa/pull/1915)) -- Add server stats to /info endpoint ([#1859](https://github.com/pilosa/pilosa/pull/1859)) -- Implement config options for block profile rate and mutex fraction ([#1910](https://github.com/pilosa/pilosa/pull/1910)) -- Implement global open file counter using syswrap (to scale past system open file limits) ([#1906](https://github.com/pilosa/pilosa/pull/1906)) -- Implement global mmap counter with fallback (to scale past system mmap limits) ([#1903](https://github.com/pilosa/pilosa/pull/1903)) -- Add shard width to index info in schema (allows client to get shard width at run time) ([#1881](https://github.com/pilosa/pilosa/pull/1881)) -- Add shift operator ([#1761](https://github.com/pilosa/pilosa/pull/1761)) -- Support advertise address and listen on 0.0.0.0 ([#1832](https://github.com/pilosa/pilosa/pull/1832)) -- Added convenience function to efficiently calculate size of a roaring bitmap in bytes ([#1839](https://github.com/pilosa/pilosa/pull/1839)) -- Make sure more tests and benchmarks can have their temp dir set by flag ([#1831](https://github.com/pilosa/pilosa/pull/1831)) -- Add sliceascending/slicedescending striped benchmarks ([#1763](https://github.com/pilosa/pilosa/pull/1763)) -- Add setValue test and benchmarks ([#1820](https://github.com/pilosa/pilosa/pull/1820)) -- Add a test for groupby filter with RangeLTLT ([#1818](https://github.com/pilosa/pilosa/pull/1818)) -- Add tests for GroupBy with keys; removes unused Bit message from proto ([#1811](https://github.com/pilosa/pilosa/pull/1811)) - -### Fixed -- Update to latest memberlist fork with race fixes ([#1944](https://github.com/pilosa/pilosa/pull/1944)) -- Return original error instead of cause in handler ([#1943](https://github.com/pilosa/pilosa/pull/1943)) -- Validate (and panic) on duplicate PQL arguments ([#1938](https://github.com/pilosa/pilosa/pull/1938)) -- Add correct content type to query responses Fixes #1873 ([#1936](https://github.com/pilosa/pilosa/pull/1936)) -- Address race condition by getting cluster nodes with lock ([#1931](https://github.com/pilosa/pilosa/pull/1931)) -- Make sure to unmap containers before modifying ([#1876](https://github.com/pilosa/pilosa/pull/1876)) -- Avoid probable race when creating fragments ([#1863](https://github.com/pilosa/pilosa/pull/1863)) -- Improve help strings for metrics options ([#1887](https://github.com/pilosa/pilosa/pull/1887)) -- Ensure ClearRow() arguments get translated ([#1848](https://github.com/pilosa/pilosa/pull/1848)) -- Prevent omitting zero ids on columnattrs ([#1846](https://github.com/pilosa/pilosa/pull/1846)) -- Set cache size to 0 if cache type is none ([#1842](https://github.com/pilosa/pilosa/pull/1842)) -- Prevent deadlock in replication logic on reopening a store ([#1834](https://github.com/pilosa/pilosa/pull/1834)) -- Pass loggers around properly in gossip ([#1835](https://github.com/pilosa/pilosa/pull/1835)) -- Include read lock in cluster.Nodes() ([#1836](https://github.com/pilosa/pilosa/pull/1836)) -- Raise an error on Rows() query against a time field with noStandardView: true ([#1826](https://github.com/pilosa/pilosa/pull/1826)) -- Don't delete test fragment data (part of repo) ([#1827](https://github.com/pilosa/pilosa/pull/1827)) -- Fix bug on upper end of bsi range queries ([#1822](https://github.com/pilosa/pilosa/pull/1822)) -- Group by fixes ([#1802](https://github.com/pilosa/pilosa/pull/1802)) - -### Changed -- Switch to GolangCI lint ([#1924](https://github.com/pilosa/pilosa/pull/1924)) -- Return empty result set when query empty ([#1937](https://github.com/pilosa/pilosa/pull/1937)) -- Add Go 1.12 to CircleCI ([#1909](https://github.com/pilosa/pilosa/pull/1909)) -- Ignore fragment files from shards node doesn't own ([#1900](https://github.com/pilosa/pilosa/pull/1900)) -- Go module support. Use Modules instead of dep for dependencies ([#1616](https://github.com/pilosa/pilosa/pull/1616)) -- Merge Range() into Row() call. ([#1804](https://github.com/pilosa/pilosa/pull/1804)) -- Add from/to range arguments to Rows() call ([#1851](https://github.com/pilosa/pilosa/pull/1851)) -- Fixes Store call error messages, Rows doesn't need field argument ([#1830](https://github.com/pilosa/pilosa/pull/1830)) - -### Performance -- BTree performance improvements ([#1916](https://github.com/pilosa/pilosa/pull/1916)) -- Make Containers smaller, especially when they have small contents ([#1901](https://github.com/pilosa/pilosa/pull/1901)) -- Address UnionInPlace performance regressions ([#1897](https://github.com/pilosa/pilosa/pull/1897)) -- Small write path for import-roaring. Makes small imports faster ([#1892](https://github.com/pilosa/pilosa/pull/1892)) -- Small write path for imports ([#1871](https://github.com/pilosa/pilosa/pull/1871)) -- Remove copy for pilosa roaring files ([#1865](https://github.com/pilosa/pilosa/pull/1865)) -- Disable anti-entropy if not using replication [performance] ([#1814](https://github.com/pilosa/pilosa/pull/1814)) -- Group By—skip 0 counts as early as possible ([#1803](https://github.com/pilosa/pilosa/pull/1803)) - -## [1.2.0] - 2018-12-20 - -This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions. - -### Added - -- Cancel queries on Context.Done() ([#1773](https://github.com/pilosa/pilosa/pull/1773)) -- Union In Place ([#1766](https://github.com/pilosa/pilosa/pull/1766), [#1774](https://github.com/pilosa/pilosa/pull/1774)) -- Import benchmarking ([#1771](https://github.com/pilosa/pilosa/pull/1771)) -- Add GroupBy() Filter ([#1753](https://github.com/pilosa/pilosa/pull/1753)) -- Add /internal/translate/keys endpoint ([#1751](https://github.com/pilosa/pilosa/pull/1751)) -- CircleCI: Add race detector to parallel build, default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756)) -- Add distributed tracing. ([#1684](https://github.com/pilosa/pilosa/pull/1684)) -- Add NoStandardView field option ([#1733](https://github.com/pilosa/pilosa/pull/1733)) -- Add some stat tracking to roaring implementation ([#1743](https://github.com/pilosa/pilosa/pull/1743)) -- Add cluster fault testing using docker-compose and pumba ([#1717](https://github.com/pilosa/pilosa/pull/1717)) -- Allow backslash, carriage return in PQL strings ([#1713](https://github.com/pilosa/pilosa/pull/1713)) -- Add base system, curl and jq for debug and checks ([#1707](https://github.com/pilosa/pilosa/pull/1707)) -- Add `Rows` and `GroupBy` functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647)) -- Add `clear` functional option for imports ([#1699](https://github.com/pilosa/pilosa/pull/1699)) -- Implement tracking of available shards to help support sparse datasets ([#1600](https://github.com/pilosa/pilosa/pull/1600), [#1695](https://github.com/pilosa/pilosa/pull/1695), [#1624](https://github.com/pilosa/pilosa/pull/1624), [#1663](https://github.com/pilosa/pilosa/pull/1663)) -- Add missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683)) -- Add Store() operation to PQL ([#1666](https://github.com/pilosa/pilosa/pull/1666)) -- Add diagnostics CPUArch field ([#1671](https://github.com/pilosa/pilosa/pull/1671)) -- Add CircleCI step to generate Docker image and push to Docker hub ([#1673](https://github.com/pilosa/pilosa/pull/1673)) -- Implement ClearRow() query ([#1645](https://github.com/pilosa/pilosa/pull/1645)) -- Add support for Bool fields ([#1658](https://github.com/pilosa/pilosa/pull/1658)) -- Make translate map size configurable ([#1653](https://github.com/pilosa/pilosa/pull/1653)) -- Add DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646)) -- Implement Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622), [#1738](https://github.com/pilosa/pilosa/pull/1738)) -- Add Not() query ([#1635](https://github.com/pilosa/pilosa/pull/1635)) -- Implement Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631)) -- Add field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625)) -- Implement column existence tracking ([#1788](https://github.com/pilosa/pilosa/pull/1788), [#1672](https://github.com/pilosa/pilosa/pull/1672), [#1628](https://github.com/pilosa/pilosa/pull/1628)) - -### Changed - -- Convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` ([#1780](https://github.com/pilosa/pilosa/pull/1780)) -- Simplify `require-*` logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755)) -- Cleanup logging ([#1748](https://github.com/pilosa/pilosa/pull/1748)) -- Remove TravisCI, add CircleCI shield ([#1740](https://github.com/pilosa/pilosa/pull/1740)) -- Upgrade Peg dependency and regenerate grammar ([#1725](https://github.com/pilosa/pilosa/pull/1725)) -- Upgrade to protoc 3.6.1 (also updated protoc-gen-gofast) ([#1724](https://github.com/pilosa/pilosa/pull/1724)) -- Move column attrs logic to executor ([#1677](https://github.com/pilosa/pilosa/pull/1677)) -- Shrink container bit count to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664)) - -### Performance - -- Remove bounds check ([#1619](https://github.com/pilosa/pilosa/pull/1619)) -- Improve benchmarking and performance ([#1741](https://github.com/pilosa/pilosa/pull/1741)) - -### Fixed - -- Ensure internal client closes all response bodies ([#1795](https://github.com/pilosa/pilosa/pull/1795)) -- Allow translate log entry buffer to grow ([#1787](https://github.com/pilosa/pilosa/pull/1787)) -- Add Gopkg.lock as a dependency for vendor target ([#1790](https://github.com/pilosa/pilosa/pull/1790)) -- Cluster resize fix ([#1785](https://github.com/pilosa/pilosa/pull/1785)) -- Attempt to fix deadlock by releasing view lock before broadcasting ([#1782](https://github.com/pilosa/pilosa/pull/1782)) -- Fix bug where cluster goes into RESIZING instead of NORMAL ([#1777](https://github.com/pilosa/pilosa/pull/1777)) -- Propogate updates to node details (not just additions and deletions) ([#1769](https://github.com/pilosa/pilosa/pull/1769)) -- Fix arm64 support ([#1764](https://github.com/pilosa/pilosa/pull/1764)) -- Fix data races ([#1750](https://github.com/pilosa/pilosa/pull/1750)) -- Fix fragment checksums race condition ([#1749](https://github.com/pilosa/pilosa/pull/1749)) -- Import cmd field type flag ([#1732](https://github.com/pilosa/pilosa/pull/1732)) -- Increase the translate file size for tests/benchmarks ([#1744](https://github.com/pilosa/pilosa/pull/1744)) -- Prevent panic in Bitmap.UnmarshalBinary when there is no data ([#1742](https://github.com/pilosa/pilosa/pull/1742)) -- Remove unused rule from peg grammar ([#1737](https://github.com/pilosa/pilosa/pull/1737)) -- Improve Internal Client errors ([#1729](https://github.com/pilosa/pilosa/pull/1729)) -- Forward imports to non-coordinator shards ([#1719](https://github.com/pilosa/pilosa/pull/1719)) -- Fix double escapes in PQL grammar ([#1727](https://github.com/pilosa/pilosa/pull/1727)) -- Ensure btree comparison doesn't fail for smallish N ([#1712](https://github.com/pilosa/pilosa/pull/1712)) -- Drop now-superfluous methodNotAllowedHandler ([#1711](https://github.com/pilosa/pilosa/pull/1711)) -- Use pilosa.Logger everywhere ([#1674](https://github.com/pilosa/pilosa/pull/1674)) -- Ensure view closes fragment on broadcast error ([#1675](https://github.com/pilosa/pilosa/pull/1675)) -- Prevent closing os.Stderr (used in verbose test logging) ([#1696](https://github.com/pilosa/pilosa/pull/1696)) -- Allow holder to close/open/close without panic on closing closed channel ([#1686](https://github.com/pilosa/pilosa/pull/1686)) -- Fix bug with Range() queries with field keys ([#1679](https://github.com/pilosa/pilosa/pull/1679)) -- Sync query validation for handlers ([#1676](https://github.com/pilosa/pilosa/pull/1676)) -- Wrap translation store errors, decrease test map size to prevent failure on 32-bit ([#1665](https://github.com/pilosa/pilosa/pull/1665)) -- Fix pass-by-value issue in proto decode ([#1662](https://github.com/pilosa/pilosa/pull/1662)) -- Do not run prerelease in CI if this is a pull request ([#1655](https://github.com/pilosa/pilosa/pull/1655)) -- Ensure mutex imports unset previous columns ([#1656](https://github.com/pilosa/pilosa/pull/1656)) -- Treat import timestamps as UTC ([#1651](https://github.com/pilosa/pilosa/pull/1651)) -- Remove unused log buffers from test cluster, fixes race ([#1612](https://github.com/pilosa/pilosa/pull/1612)) -- Add --field-keys and --index-keys options to pilosa import ([#1621](https://github.com/pilosa/pilosa/pull/1621)) -- Use passed stdin, stdout, and stderr in the cmd package ([#1620](https://github.com/pilosa/pilosa/pull/1620)) -- Update Go client sample to match latest master ([#1614](https://github.com/pilosa/pilosa/pull/1614)) - - -## [1.1.0] - 2018-08-21 - -This version contains 32 contributions from 5 contributors. There are 89 files changed; 2,752 insertions; and 1,013 deletions. - -### Added - -- Add CircleCI ([#1610](https://github.com/pilosa/pilosa/pull/1610)) -- Add key translation to exports ([#1608](https://github.com/pilosa/pilosa/pull/1608)) -- Support importing key values ([#1599](https://github.com/pilosa/pilosa/pull/1599), [#1601](https://github.com/pilosa/pilosa/pull/1601)) -- Treat coordinator as primary translate store ([#1582](https://github.com/pilosa/pilosa/pull/1582)) -- Add DEGRADED cluster state and handle gossip NodeLeave events correctly ([#1584](https://github.com/pilosa/pilosa/pull/1584)) -- Add linters to gometalinter and fix related issues ([#1544](https://github.com/pilosa/pilosa/pull/1544), [#1543](https://github.com/pilosa/pilosa/pull/1543), [#1540](https://github.com/pilosa/pilosa/pull/1540), [#1539](https://github.com/pilosa/pilosa/pull/1539), [#1537](https://github.com/pilosa/pilosa/pull/1537), [#1536](https://github.com/pilosa/pilosa/pull/1536), [#1535](https://github.com/pilosa/pilosa/pull/1535), [#1534](https://github.com/pilosa/pilosa/pull/1534), [#1530](https://github.com/pilosa/pilosa/pull/1530), [#1529](https://github.com/pilosa/pilosa/pull/1529), [#1528](https://github.com/pilosa/pilosa/pull/1528), [#1526](https://github.com/pilosa/pilosa/pull/1526), [#1527](https://github.com/pilosa/pilosa/pull/1527)) -- Add mutex field type ([#1524](https://github.com/pilosa/pilosa/pull/1524)) -- Fragment rows() and rowsForColumn() ([#1532](https://github.com/pilosa/pilosa/pull/1532)) - -### Fixed - -- Fix race on replicationClosing channel ([#1607](https://github.com/pilosa/pilosa/pull/1607)) -- Prevent anti-entropy and cluster resize from running simultaneously ([#1586](https://github.com/pilosa/pilosa/pull/1586)) -- Require a valid port that isn't greater than 65,535 ([#1603](https://github.com/pilosa/pilosa/pull/1603)) -- Add view parameter to sync logic for syncing time fields ([#1602](https://github.com/pilosa/pilosa/pull/1602)) -- Fix translator in cluster environment ([#1552](https://github.com/pilosa/pilosa/pull/1552)) -- Use string prefix instead of equality so json error message will pass on all Go versions ([#1558](https://github.com/pilosa/pilosa/pull/1558)) - -## [1.0.2] - 2018-08-01 - -This version contains 11 contributions from 3 contributors. There are 30 files changed; 1,569 insertions; and 1,215 deletions. - -### Fixed - -- Fix documentation ([#1503](https://github.com/pilosa/pilosa/pull/1503), [#1495](https://github.com/pilosa/pilosa/pull/1495), [#1551](https://github.com/pilosa/pilosa/pull/1551)) -- Fix places where empty IndexOptions were being used ([#1547](https://github.com/pilosa/pilosa/pull/1547)) -- Fix translator syncing bug in cluster environments ([#1552](https://github.com/pilosa/pilosa/pull/1552)) -- Fix race condition in translate_test ([#1541](https://github.com/pilosa/pilosa/pull/1541)) -- Add IndexOptions to IndexInfo json response ([#1542](https://github.com/pilosa/pilosa/pull/1542)) -- Add proper locking to cluster code to prevent races ([#1533](https://github.com/pilosa/pilosa/pull/1533)) -- Re-export erroneously unexported func Row.Intersect ([#1502](https://github.com/pilosa/pilosa/pull/1502)) -- Update parser to handle row keys on SetRowAttrs() ([#1555](https://github.com/pilosa/pilosa/pull/1555)) - -## [1.0.1] - 2018-07-11 - -This version contains 12 contributions from 4 contributors. There are 11 files changed; 133 insertions; and 39 deletions. - -### Fixed - -- Use `dep ensure -vendor-only` for build repeatability ([#1491](https://github.com/pilosa/pilosa/pull/1491)) -- Make sure time range views are calculated correctly across months ([#1485](https://github.com/pilosa/pilosa/pull/1485)) -- Fix up error handling, add a configurable timeout to http handler closing ([#1486](https://github.com/pilosa/pilosa/pull/1486)) -- Add gossip Closer ([#1483](https://github.com/pilosa/pilosa/pull/1483)) -- Update docs references to WebUI naming (console) and installation ([#1493](https://github.com/pilosa/pilosa/pull/1493)) - -## [1.0.0] - 2018-07-09 - -This version contains 218 contributions from 7 contributors. There are 184 files changed; 21,769 insertions; and 20,275 deletions. - -### Added - -- ID-Key Translation ([#1337](https://github.com/pilosa/pilosa/pull/1337)) -- Add CORS support to handler ([#1327](https://github.com/pilosa/pilosa/pull/1327)) - -### Changed - -- HTTP handler updates ([#1408](https://github.com/pilosa/pilosa/pull/1408), [#1399](https://github.com/pilosa/pilosa/pull/1399), [#1441](https://github.com/pilosa/pilosa/pull/1441), [#1375](https://github.com/pilosa/pilosa/pull/1375), [#1433](https://github.com/pilosa/pilosa/pull/1433), [#1444](https://github.com/pilosa/pilosa/pull/1444), [#1388](https://github.com/pilosa/pilosa/pull/1388), [#1309](https://github.com/pilosa/pilosa/pull/1309), [#1302](https://github.com/pilosa/pilosa/pull/1302), [#1304](https://github.com/pilosa/pilosa/pull/1304), [#1465](https://github.com/pilosa/pilosa/pull/1465), [#1466](https://github.com/pilosa/pilosa/pull/1466)) -- Refactor/improve tests ([#1437](https://github.com/pilosa/pilosa/pull/1437), [#1434](https://github.com/pilosa/pilosa/pull/1434), [#1435](https://github.com/pilosa/pilosa/pull/1435), [#1425](https://github.com/pilosa/pilosa/pull/1425), [#1418](https://github.com/pilosa/pilosa/pull/1418), [#1419](https://github.com/pilosa/pilosa/pull/1419), [#1413](https://github.com/pilosa/pilosa/pull/1413), [#1394](https://github.com/pilosa/pilosa/pull/1394), [#1387](https://github.com/pilosa/pilosa/pull/1387), [#1386](https://github.com/pilosa/pilosa/pull/1386), [#1378](https://github.com/pilosa/pilosa/pull/1378), [#1364](https://github.com/pilosa/pilosa/pull/1364), [#1348](https://github.com/pilosa/pilosa/pull/1348), [#1340](https://github.com/pilosa/pilosa/pull/1340), [#1297](https://github.com/pilosa/pilosa/pull/1297)) -- Simplify inter-node communication ([#1428](https://github.com/pilosa/pilosa/pull/1428), [#1427](https://github.com/pilosa/pilosa/pull/1427), [#1412](https://github.com/pilosa/pilosa/pull/1412), [#1398](https://github.com/pilosa/pilosa/pull/1398), [#1391](https://github.com/pilosa/pilosa/pull/1391), [#1389](https://github.com/pilosa/pilosa/pull/1389)) -- Make gossip's interface to Pilosa the API struct ([#1452](https://github.com/pilosa/pilosa/pull/1452)) -- Rename slice to shard ([#1426](https://github.com/pilosa/pilosa/pull/1426)) -- Clearbit for time fields ([#1424](https://github.com/pilosa/pilosa/pull/1424)) -- Update docs ([#1390](https://github.com/pilosa/pilosa/pull/1390), [#1329](https://github.com/pilosa/pilosa/pull/1329), [#1305](https://github.com/pilosa/pilosa/pull/1305), [#1296](https://github.com/pilosa/pilosa/pull/1296), [#1461](https://github.com/pilosa/pilosa/pull/1461)) -- Simplify server setup ([#1417](https://github.com/pilosa/pilosa/pull/1417), [#1393](https://github.com/pilosa/pilosa/pull/1393),[#1451](https://github.com/pilosa/pilosa/pull/1451)) -- Refactor API ([#1407](https://github.com/pilosa/pilosa/pull/1407)) -- Rewrite PQL parser and add various improvements/simplifications ([#1382](https://github.com/pilosa/pilosa/pull/1382), [#1402](https://github.com/pilosa/pilosa/pull/1402), [#1354](https://github.com/pilosa/pilosa/pull/1354), [#1463](https://github.com/pilosa/pilosa/pull/1463)) -- Rename "frame" to "field" ([#1395](https://github.com/pilosa/pilosa/pull/1395), [#1362](https://github.com/pilosa/pilosa/pull/1362), [#1360](https://github.com/pilosa/pilosa/pull/1360), [#1358](https://github.com/pilosa/pilosa/pull/1358), [#1357](https://github.com/pilosa/pilosa/pull/1357), [#1355](https://github.com/pilosa/pilosa/pull/1355)) -- Optimize count ([#1365](https://github.com/pilosa/pilosa/pull/1365)) -- Simplify bitmap max function ([#1333](https://github.com/pilosa/pilosa/pull/1333)) -- Rename "bit" to "column" for clarity ([#1326](https://github.com/pilosa/pilosa/pull/1326)) -- Rename pilosa.Bitmap to Row ([#1311](https://github.com/pilosa/pilosa/pull/1311)) -- Invert encoding/decoding and remove internal references ([#1454](https://github.com/pilosa/pilosa/pull/1454)) - -### Removed - -- Rename (unexport) many items to reduce public API footprint prior to 1.0 release ([#1470](https://github.com/pilosa/pilosa/pull/1470), [#1458](https://github.com/pilosa/pilosa/pull/1458), [#1450](https://github.com/pilosa/pilosa/pull/1450), [#1449](https://github.com/pilosa/pilosa/pull/1449), [#1448](https://github.com/pilosa/pilosa/pull/1448), [#1447](https://github.com/pilosa/pilosa/pull/1447), [#1446](https://github.com/pilosa/pilosa/pull/1446), [#1438](https://github.com/pilosa/pilosa/pull/1438), [#1443](https://github.com/pilosa/pilosa/pull/1443), [#1440](https://github.com/pilosa/pilosa/pull/1440), [#1439](https://github.com/pilosa/pilosa/pull/1439), [#1409](https://github.com/pilosa/pilosa/pull/1409), [#1392](https://github.com/pilosa/pilosa/pull/1392), [#1374](https://github.com/pilosa/pilosa/pull/1374), [#1372](https://github.com/pilosa/pilosa/pull/1372), [#1369](https://github.com/pilosa/pilosa/pull/1369), [#1367](https://github.com/pilosa/pilosa/pull/1367), [#1366](https://github.com/pilosa/pilosa/pull/1366), [#1351](https://github.com/pilosa/pilosa/pull/1351), [#1420](https://github.com/pilosa/pilosa/pull/1420), [#1416](https://github.com/pilosa/pilosa/pull/1416), [#1397](https://github.com/pilosa/pilosa/pull/1397)) -- Remove dead code ([#1432](https://github.com/pilosa/pilosa/pull/1432), [#1457](https://github.com/pilosa/pilosa/pull/1457), [#1421](https://github.com/pilosa/pilosa/pull/1421), [#1411](https://github.com/pilosa/pilosa/pull/1411), [#1377](https://github.com/pilosa/pilosa/pull/1377), [#1393](https://github.com/pilosa/pilosa/pull/1393), [#1462](https://github.com/pilosa/pilosa/pull/1462)) -- Remove view argument from Field.SetBit and Field.ClearBit ([#1396](https://github.com/pilosa/pilosa/pull/1396)) -- Remove WebUI (now contained in a separate package) ([#1363](https://github.com/pilosa/pilosa/pull/1363)) -- Remove bench command ([#1347](https://github.com/pilosa/pilosa/pull/1347)) -- Remove "view" from API, handler, docs ([#1346](https://github.com/pilosa/pilosa/pull/1346)) -- Remove backup/restore stuff ([#1339](https://github.com/pilosa/pilosa/pull/1339), [#1341](https://github.com/pilosa/pilosa/pull/1341)) -- Remove inverse frame functionality ([#1335](https://github.com/pilosa/pilosa/pull/1335)) -- Remove rangeEnabled option ([#1332](https://github.com/pilosa/pilosa/pull/1332)) -- Remove index and field MarshalJSON ([#1468](https://github.com/pilosa/pilosa/pull/1468)) - -### Fixed - -- Fix a few data races ([#1423](https://github.com/pilosa/pilosa/pull/1423)) -- Fix for crash while removing containers ([#1401](https://github.com/pilosa/pilosa/pull/1401)) -- Allow dashes in frame names ([#1415](https://github.com/pilosa/pilosa/pull/1415)) -- Fix generate-config command, use single toml lib ([#1350](https://github.com/pilosa/pilosa/pull/1350)) - -## [0.10.0] - 2018-05-15 - -This version contains 93 contributions from 8 contributors. There are 93 files changed; 4,495 insertions; and 5,392 deletions. - -### Added - -- Add B+Tree containers (Enterprise Edition) ([#1285](https://github.com/pilosa/pilosa/pull/1285)) -- Add /info endpoint ([#1236](https://github.com/pilosa/pilosa/pull/1236)) - -### Changed - -- Wrap errors ([#1271](https://github.com/pilosa/pilosa/pull/1271), [#1258](https://github.com/pilosa/pilosa/pull/1258), [#1274](https://github.com/pilosa/pilosa/pull/1274), [#1270](https://github.com/pilosa/pilosa/pull/1270), [#1273](https://github.com/pilosa/pilosa/pull/1273), [#1272](https://github.com/pilosa/pilosa/pull/1272), [#1260](https://github.com/pilosa/pilosa/pull/1260), [#1259](https://github.com/pilosa/pilosa/pull/1259), [#1256](https://github.com/pilosa/pilosa/pull/1256), [#1257](https://github.com/pilosa/pilosa/pull/1257), [#1261](https://github.com/pilosa/pilosa/pull/1261), [#1262](https://github.com/pilosa/pilosa/pull/1262), [#1263](https://github.com/pilosa/pilosa/pull/1263), [#1265](https://github.com/pilosa/pilosa/pull/1265)) - -### Removed - -- Remove unused code ([#1286](https://github.com/pilosa/pilosa/pull/1286)) -- Remove input definition, add install-stringer to Makefile ([#1284](https://github.com/pilosa/pilosa/pull/1284)) -- Remove /id and /hosts endpoints. Add local ID to /status ([#1238](https://github.com/pilosa/pilosa/pull/1238)) -- Remove API.URI ([#1255](https://github.com/pilosa/pilosa/pull/1255)) - -### Fixed - -- Assorted docs fixes ([#1281](https://github.com/pilosa/pilosa/pull/1281), [#1269](https://github.com/pilosa/pilosa/pull/1269)) -- Update PQL syntax in bench subcommand ([#1279](https://github.com/pilosa/pilosa/pull/1279)) -- Update help menu in WebUI ([#1278](https://github.com/pilosa/pilosa/pull/1278)) -- Fix dead lock ([#1268](https://github.com/pilosa/pilosa/pull/1268)) -- Make sure gossipMemberSet.Logger is set during server setup ([#1266](https://github.com/pilosa/pilosa/pull/1266)) -- Make sure ~ is expanded in NewServer; BroadcastReceiver uses temp path ([#1242](https://github.com/pilosa/pilosa/pull/1242)) -- Avoid creating a slice of nil timestamps on Import() ([#1234](https://github.com/pilosa/pilosa/pull/1234)) -- Fixup internal client ([#1253](https://github.com/pilosa/pilosa/pull/1253)) - -## [0.9.0] - 2018-05-04 - -This version contains 188 contributions from 12 contributors. There are 141 files changed; 17,832 insertions; and 7,503 deletions. - -*Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.* - -### Added - -- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908)) -- Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088)) -- Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133)) -- Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121)) -- Add validation around node-remove conditions ([#1138](https://github.com/pilosa/pilosa/pull/1138)) -- broadcast.SendSync field creation and deletion to all nodes ([#1132](https://github.com/pilosa/pilosa/pull/1132)) -- Spread recalculate caches to all nodes. Fixes #1069 ([#1109](https://github.com/pilosa/pilosa/pull/1109)) -- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064)) -- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063)) -- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048)) -- Add support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014)) -- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245)) -- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) -- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) -- Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191)) -- Log time/version to startup log ([#1246](https://github.com/pilosa/pilosa/pull/1246)) -- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972), [#1215](https://github.com/pilosa/pilosa/pull/1215), [#1213](https://github.com/pilosa/pilosa/pull/1213), [#1224](https://github.com/pilosa/pilosa/pull/1224), [#1250](https://github.com/pilosa/pilosa/pull/1250)) - -### Changed - -- Put Statik behind an interface ([#1163](https://github.com/pilosa/pilosa/pull/1163)) -- Refactor diagnostics, inject gopsutil dependency ([#1166](https://github.com/pilosa/pilosa/pull/1166)) -- Use boolean instead of address to configure coordinator ([#1158](https://github.com/pilosa/pilosa/pull/1158)) -- Put GCNotify behind an interface ([#1148](https://github.com/pilosa/pilosa/pull/1148)) -- Replace custom assembly bit functions with standard go ([#797](https://github.com/pilosa/pilosa/pull/797)) -- Improve roaring tests ([#1115](https://github.com/pilosa/pilosa/pull/1115)) -- Change configuration cluster.type (string) to cluster.disabled (bool) ([#1099](https://github.com/pilosa/pilosa/pull/1099)) -- Use NodeID instead of URI for node identification ([#1077](https://github.com/pilosa/pilosa/pull/1077)) -- Change gossip config from DefaultLocalConfig to DefaultWANConfig ([#1032](https://github.com/pilosa/pilosa/pull/1032)) -- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027)) -- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001)) -- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950)) -- Refactor HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) -- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) -- Refactor pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) -- Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223)) -- Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216)) -- Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221)) - -### Removed - -- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171)) -- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142)) -- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130)) -- Remove holder.Peek, combine with HasData, move server logic ([#1226](https://github.com/pilosa/pilosa/pull/1226)) -- Remove PATCH frame endpoint ([#1222](https://github.com/pilosa/pilosa/pull/1222)) -- Remove Index.MergeSchemas() method ([#1219](https://github.com/pilosa/pilosa/pull/1219)) -- Remove references to Input Definition from the docs ([#1212](https://github.com/pilosa/pilosa/pull/1212)) -- Remove Index.TimeQuantum ([#1209](https://github.com/pilosa/pilosa/pull/1209)) -- Remove SecurityManager. Implement api restrictions in api package. ([#1207](https://github.com/pilosa/pilosa/pull/1207)) - -### Fixed - -- Handle the scheme correctly in config.Bind ([#1143](https://github.com/pilosa/pilosa/pull/1143)) -- Prevent excessive sendSync (createView) messages. ([#1139](https://github.com/pilosa/pilosa/pull/1139)) -- Fix a shift logic bug in bitmapZeroRange ([#1110](https://github.com/pilosa/pilosa/pull/1110)) -- Fix node id validation on set-coordinator ([#1102](https://github.com/pilosa/pilosa/pull/1102)) -- Avoid overflow bug in differenceRunArray ([#1105](https://github.com/pilosa/pilosa/pull/1105)) -- Fix bug in NewServerCluster where each host was its own coordinator ([#1101](https://github.com/pilosa/pilosa/pull/1101)) -- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084)) -- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979)) -- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980)) -- Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233)) -- Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229)) -- Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228)) - -### Deprecated - -- Deprecate RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) - -### Performance - -- Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017)) - -## [0.8.8] - 2018-02-19 - -This version contains 1 contribution from 2 contributors. There are 4 files changed; 1,153 insertions; and 618 deletions. - -### Fixed - -- Bug fixes and improved test coverage in roaring ([#1118](https://github.com/pilosa/pilosa/pull/1118)) - -## [0.8.7] - 2018-02-12 - -This version contains 1 contribution from 1 contributors. There are 2 files changed; 84 insertions; and 4 deletions. - -### Fixed - -- Fix a shift logic bug in bitmapZeroRange ([#1111](https://github.com/pilosa/pilosa/pull/1111)) - -## [0.8.6] - 2018-02-09 - -This version contains 2 contributions from 2 contributors. There are 3 files changed; 171 insertions; and 6 deletions. - -### Fixed - -- Fix overflow bug in differenceRunArray [#1106](https://github.com/pilosa/pilosa/pull/1106) -- Fix bug where count and bitmap queries could return different numbers [#1083](https://github.com/pilosa/pilosa/pull/1083) - -## [0.8.5] - 2018-01-18 - -This version contains 1 contribution from 1 contributor. There is 1 file changed; 1 insertion, and 0 deletions. - -### Fixed - -- Bind Docker container on all interfaces ([#1061](https://github.com/pilosa/pilosa/pull/1061)) - -## [0.8.4] - 2018-01-10 - -This version contains 4 contributions from 3 contributors. There are 17 files changed; 974 insertions; and 221 deletions. - -### Fixed - -- Group the write operations in syncBlock by MaxWritesPerRequest ([#1038](https://github.com/pilosa/pilosa/pull/1038)) -- Change gossip config from memberlist.DefaultLocalConfig to memberlist.DefaultWANConfig ([#1033](https://github.com/pilosa/pilosa/pull/1033)) - -### Performance - -- Change AttrBlock handler calls to support protobuf instead of json ([#1046](https://github.com/pilosa/pilosa/pull/1046)) -- Use RLock instead of Lock in a few places ([#1042](https://github.com/pilosa/pilosa/pull/1042)) - -## [0.8.3] - 2017-12-12 - -This version contains 1 contribution from 1 contributor. There are 2 files changed; 59 insertions; and 42 deletions. - -### Fixed - -- Protect against accessing pointers to memory which was unmapped ([#1000](https://github.com/pilosa/pilosa/pull/1000)) - -## [0.8.2] - 2017-12-05 - -This version contains 1 contribution from 1 contributor. There are 15 files changed; 127 insertions; and 98 deletions. - -### Fixed - -- Modify initialization of HTTP client so only one instance is created ([#994](https://github.com/pilosa/pilosa/pull/994)) - -## [0.8.1] - 2017-11-15 - -This version contains 2 contributions from 2 contributors. There are 4 files changed; 27 insertions; and 14 deletions. - -### Fixed - -- Fix CountOpenFiles() fatal crash ([#969](https://github.com/pilosa/pilosa/pull/969)) -- Fix version check when local is greater than pilosa.com ([#968](https://github.com/pilosa/pilosa/pull/968)) - -## [0.8.0] - 2017-11-15 - -This version contains 31 contributions from 8 contributors. There are 84 files changed; 3,732 insertions; and 1,428 deletions. - -### Added - -- Diagnostics ([#895](https://github.com/pilosa/pilosa/pull/895)) -- Add docker-build make target for repeatable Docker-based builds ([#933](https://github.com/pilosa/pilosa/pull/933)) -- Add documentation on importing field values; fixes #924 ([#938](https://github.com/pilosa/pilosa/pull/938)) -- Add flag documentation and tests, remove "plugins.path" ([#942](https://github.com/pilosa/pilosa/pull/942)) -- Add TLS support ([#867](https://github.com/pilosa/pilosa/pull/867)) -- Add TLS cluster how to ([#898](https://github.com/pilosa/pilosa/pull/898)) -- Add support for gossip encryption ([#889](https://github.com/pilosa/pilosa/pull/889)) -- Add Recalculate Caches endpoint ([#881](https://github.com/pilosa/pilosa/pull/881)) -- Add search-friendly documentation for BSI range query syntax ([#955](https://github.com/pilosa/pilosa/pull/955)) - -### Changed - -- Remove unneeded Gopkg.toml constraints and update all dependencies ([#943](https://github.com/pilosa/pilosa/pull/943)) -- Remove row and column labels in webUI ([#884](https://github.com/pilosa/pilosa/pull/884)) -- Internal Client refactoring ([#892](https://github.com/pilosa/pilosa/pull/892)) -- Remove column/row labels for input definition ([#945](https://github.com/pilosa/pilosa/pull/945)) -- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878)) - -### Fixed - -- Skip permissions test when run as root. Fixes #940 ([#941](https://github.com/pilosa/pilosa/pull/941)) -- Address "connection reset" issues in client ([#934](https://github.com/pilosa/pilosa/pull/934)) -- Fix field value import: Use signed int and respect field minimum ([#919](https://github.com/pilosa/pilosa/pull/919)) -- Constrain BoltDB to version rather than specific revision ([#887](https://github.com/pilosa/pilosa/pull/887)) -- Fix bug in environment variable format ([#882](https://github.com/pilosa/pilosa/pull/882)) -- Fix overflow in differenceRunBitmap ([#949](https://github.com/pilosa/pilosa/pull/949)) - -### Performance - -- Use FieldNotNull to improve efficiency of BETWEEN queries ([#874](https://github.com/pilosa/pilosa/pull/874)) - -## [0.7.2] - 2017-11-15 - -This version contains 1 contribution from 1 contributor. There is 1 file changed; 16 insertions; and 1 deletion. - -### Changed - -- Bump HTTP client's MaxIdleConns and MaxIdleConnsPerHost ([#920](https://github.com/pilosa/pilosa/pull/920)) - -## [0.7.1] - 2017-10-09 - -This version contains 3 contributions from 3 contributors. There are 14 files changed; 221 insertions; and 52 deletions. - -### Changed - -- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878)) - -### Performance - -- Leverage not-null field to make BETWEEN queries more efficient ([#874](https://github.com/pilosa/pilosa/pull/874)) - -## [0.7.0] - 2017-10-03 - -This version contains 59 contributions from 9 contributors. There are 61 files changed; 5207 insertions; and 1054 deletions. - -### Added - -- Add HTTP API for fields ([#811](https://github.com/pilosa/pilosa/pull/811), [#856](https://github.com/pilosa/pilosa/pull/856)) -- Add HTTP API for delete views ([#785](https://github.com/pilosa/pilosa/pull/785)) -- Modify import endpoint to handle BSI field values ([#840](https://github.com/pilosa/pilosa/pull/840)) -- Add field Range() support to Executor ([#791](https://github.com/pilosa/pilosa/pull/791)) -- Support PQL Range() queries for fields ([#755](https://github.com/pilosa/pilosa/pull/755)) -- Add Sum() field query ([#778](https://github.com/pilosa/pilosa/pull/778)) -- Add documentation for BSI ([#861](https://github.com/pilosa/pilosa/pull/861)) -- Add BETWEEN for Range queries ([#847](https://github.com/pilosa/pilosa/pull/847)) -- Add Xor support for PQL ([#789](https://github.com/pilosa/pilosa/pull/789)) -- Enable auto-creating the schema on imports ([#837](https://github.com/pilosa/pilosa/pull/837)) -- Update client library docs ([#831](https://github.com/pilosa/pilosa/pull/831)) -- Handle SIGTERM signal ([#830](https://github.com/pilosa/pilosa/pull/830)) -- Add cluster config example to docs ([#806](https://github.com/pilosa/pilosa/pull/806)) -- Add ability to exclude attributes and bits in Bitmap queries ([#783](https://github.com/pilosa/pilosa/pull/783)) - -### Fixed - -- Fix panic when iterating over an empty run container ([#860](https://github.com/pilosa/pilosa/pull/860)) -- Fix row id zero bug ([#814](https://github.com/pilosa/pilosa/pull/814)) -- Fix cache invalidation bug ([#795](https://github.com/pilosa/pilosa/pull/795)) -- Set container.n in differenceRunRun ([#794](https://github.com/pilosa/pilosa/pull/794)) -- Fix infinite loop in bitmap-to-array conversion ([#779](https://github.com/pilosa/pilosa/pull/779)) -- Fix CountRange bug ([#773](https://github.com/pilosa/pilosa/pull/773)) - -### Deprecated - -- Remove support for row/column labels ([#839](https://github.com/pilosa/pilosa/pull/839)) - -### Performance - -- Refactor differenceRunArray ([#859](https://github.com/pilosa/pilosa/pull/859)) -- Update fragment.FieldSum to use roaring IntersectionCount() ([#841](https://github.com/pilosa/pilosa/pull/841)) -- Add roaring optimizations ([#842](https://github.com/pilosa/pilosa/pull/842)) -- Convert lock to read lock ([#848](https://github.com/pilosa/pilosa/pull/848)) -- Reduce Lock calls in executor ([#846](https://github.com/pilosa/pilosa/pull/846)) -- Implement container.flipBitmap() to improve differenceRunBitmap() ([#849](https://github.com/pilosa/pilosa/pull/849)) -- Reuse container storage on UnmarshalBinary to improve memory utilization ([#820](https://github.com/pilosa/pilosa/pull/820)) -- Improve WriteTo performance ([#812](https://github.com/pilosa/pilosa/pull/812)) - -## [0.6.0] - 2017-08-11 - -This version contains 14 contributions from 5 contributors. There are 28 files changed; 4,936 insertions; and 692 deletions. - -### Added - -- Add Run-length Encoding ([#758](https://github.com/pilosa/pilosa/pull/758)) - -### Changed - -- Make gossip the default broadcast type ([#750](https://github.com/pilosa/pilosa/pull/750)) - -### Fixed - -- Fix CountRange ([#759](https://github.com/pilosa/pilosa/pull/759)) -- Fix `differenceArrayRun` logic ([#674](https://github.com/pilosa/pilosa/pull/674)) - -## [0.5.0] - 2017-08-02 - -This version contains 65 contributions from 8 contributors (including 1 volunteer contributor). There are 79 files changed; 7,972 insertions; and 2,800 deletions. - -### Added - -- Set open file limit during Pilosa startup ([#748](https://github.com/pilosa/pilosa/pull/748)) -- Add Input Definition ([#646](https://github.com/pilosa/pilosa/pull/646)) -- Add cache type: None ([#745](https://github.com/pilosa/pilosa/pull/745)) -- Add panic recovery in top level HTTP handler ([#741](https://github.com/pilosa/pilosa/pull/741)) -- Count open file handles as a StatsD metric ([#636](https://github.com/pilosa/pilosa/pull/636)) -- Add coverage tools to Makefile ([#635](https://github.com/pilosa/pilosa/pull/635)) -- Add Holder test coverage ([#629](https://github.com/pilosa/pilosa/pull/629)) -- Add runtime memory metrics ([#600](https://github.com/pilosa/pilosa/pull/600)) -- Add sorting flag to import command ([#606](https://github.com/pilosa/pilosa/pull/606)) -- Add PQL support for field values (WIP) ([#721](https://github.com/pilosa/pilosa/pull/721)) -- Set and retrieve field values (WIP) ([#702](https://github.com/pilosa/pilosa/pull/702)) -- Add BSI range-encoding schema support (WIP) ([#670](https://github.com/pilosa/pilosa/pull/670)) - -### Changed - -- Move InternalPort config option to top-level ([#747](https://github.com/pilosa/pilosa/pull/747)) -- Switch from glide to dep for dependency management ([#744](https://github.com/pilosa/pilosa/pull/744)) -- Remove QueryRequest.Quantum since it is no longer used ([#699](https://github.com/pilosa/pilosa/pull/699)) -- Refactor test utilities into importable package ([#675](https://github.com/pilosa/pilosa/pull/675)) - -### Fixed - -- Add mutex for attribute cache ([#729](https://github.com/pilosa/pilosa/pull/729)) -- Use log-path flag to specify log file ([#678](https://github.com/pilosa/pilosa/pull/678)) - -## [0.4.0] - 2017-06-08 - -This version contains 53 contributions from 13 contributors (including 4 volunteer contributors). There are 96 files changed; 6373 insertions; and 770 deletions. - -*Note that data files created in Pilosa < 0.4.0 are not compatible with Pilosa 0.4.0 as a result of [#520](https://github.com/pilosa/pilosa/pull/520).* - -### Added -- Support metric reporting through StatsD protocol ([#468](https://github.com/pilosa/pilosa/pull/468), [#568](https://github.com/pilosa/pilosa/pull/568), [#580](https://github.com/pilosa/pilosa/pull/580)) -- Improve test coverage for ctl package ([#586](https://github.com/pilosa/pilosa/pull/586)) -- Add support for bit flip (negate) in roaring ([#592](https://github.com/pilosa/pilosa/pull/592)) -- Add xor support to roaring ([#571](https://github.com/pilosa/pilosa/pull/571)) -- Improve WebUI autocomplete ([#560](https://github.com/pilosa/pilosa/pull/560)) -- Add syntax hints tooltip to WebUI ([#537](https://github.com/pilosa/pilosa/pull/537)) -- Implement 'config' CLI command ([#541](https://github.com/pilosa/pilosa/pull/541)) -- Move docs into repo ([#563](https://github.com/pilosa/pilosa/pull/563)) -- Add inverse TopN() support ([#551](https://github.com/pilosa/pilosa/pull/551)) -- Add various Makefile updates ([#540](https://github.com/pilosa/pilosa/pull/540)) -- Provide details on Glide checksum mismatch ([#546](https://github.com/pilosa/pilosa/pull/546)) -- Add Docker multi-stage build ([#535](https://github.com/pilosa/pilosa/pull/535)) -- Support inverse Range() queries ([#533](https://github.com/pilosa/pilosa/pull/533)) -- Support colon commands in WebUI ([#529](https://github.com/pilosa/pilosa/pull/529), [#510](https://github.com/pilosa/pilosa/pull/510)) - -### Changed -- Increase default partition count from 16 to 256 (BREAKING CHANGE) ([#520](https://github.com/pilosa/pilosa/pull/520)) -- Validate unknown query params ([#578](https://github.com/pilosa/pilosa/pull/578)) -- Validate configuration file ([#573](https://github.com/pilosa/pilosa/pull/573)) -- Change default cache type to ranked ([#524](https://github.com/pilosa/pilosa/pull/524)) -- Add max-writes-per-requests limit ([#525](https://github.com/pilosa/pilosa/pull/525)) - -### Fixed -- Add "make test" to PHONY section of Makefile ([#605](https://github.com/pilosa/pilosa/pull/605)) -- Fix failing tests when IPv6 is disabled ([#594](https://github.com/pilosa/pilosa/pull/594)) -- Add minor docs fix, indent in JSON ([#599](https://github.com/pilosa/pilosa/pull/599)) -- Fix BroadcastHandler handle missing index error ([#597](https://github.com/pilosa/pilosa/pull/597)) -- Add WebUI fixes ([#589](https://github.com/pilosa/pilosa/pull/589)) -- Fix support for 32-bit Linux ([#549](https://github.com/pilosa/pilosa/pull/549), [#565](https://github.com/pilosa/pilosa/pull/565)) -- Fix 3 separate bugs in bitmapCountRange ([#559](https://github.com/pilosa/pilosa/pull/559)) -- Add client support for MaxInverseSliceByIndex ([#555](https://github.com/pilosa/pilosa/pull/555)) -- Fix bug in `handleGetSliceMax` ([#554](https://github.com/pilosa/pilosa/pull/554)) -- Default to `standard` view in export command ([#548](https://github.com/pilosa/pilosa/pull/548)) -- Fix vet issues with the assembly code in Roaring ([#528](https://github.com/pilosa/pilosa/pull/528)) -- Prevent row labels that match the column label ([#503](https://github.com/pilosa/pilosa/pull/503)) -- Fix roaring test: TestBitmap_Quick_Array1 ([#507](https://github.com/pilosa/pilosa/pull/507)) -- Don't try to create inverse views on Import() when inverseEnabled is false ([#462](https://github.com/pilosa/pilosa/pull/462)) - -### Performance -- Set n based on array length instead of incrementing repeatedly ([#590](https://github.com/pilosa/pilosa/pull/590)) -- Rewrite intersectCountArrayBitmap for perf test ([#577](https://github.com/pilosa/pilosa/pull/577)) -- Check for duplicate attributes under read lock on insert ([#562](https://github.com/pilosa/pilosa/pull/562)) - -[Unreleased]: https://github.com/pilosa/pilosa/compare/v1.2...HEAD -[0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4 -[0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5 -[0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6 -[0.7.0]: https://github.com/pilosa/pilosa/compare/v0.6...v0.7 -[0.8.0]: https://github.com/pilosa/pilosa/compare/v0.7...v0.8 -[0.9.0]: https://github.com/pilosa/pilosa/compare/v0.8...v0.9 -[0.10.0]: https://github.com/pilosa/pilosa/compare/v0.9...v0.10 -[1.0.0]: https://github.com/pilosa/pilosa/compare/v0.10...v1.0 -[1.1.0]: https://github.com/pilosa/pilosa/compare/v1.0...v1.1 -[1.2.0]: https://github.com/pilosa/pilosa/compare/v1.1...v1.2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 8e4b45e4e..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,177 +0,0 @@ -# Contributing to Pilosa - -The workflow components of these instructions apply to all Pilosa repositories. - -## Reporting a bug - -If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1]. - -## Submitting a feature request - -Feature requests are managed in Github issues, organized with [Zenhub](https://www.zenhub.com/), which is publicly available as a browser extension. New features typically go through a [Proposal Process][4] -which starts by [opening a new issue][1] that describes the new feature proposal. - -## Making code contributions - -Before you start working on new features, you should [open a new issue][1] to let others know what -you're doing, otherwise you run the risk of duplicating effort. This also -gives others an opportunity to provide input for your feature. - -If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6]. - - -### Development Environment - -- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [CircleCI config file](../master/.circleci/config.yml) for the most up-to-date information. - -- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH). - -- Fork the [Pilosa repository][2] to your own account. - -- It will be easier to follow these instructions if you: - - ```sh - export GH_USERNAME= - ``` - -- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone Pilosa: - - ```sh - mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ - git clone https://github.com/pilosa/pilosa.git - ``` - -- `cd` to your pilosa directory: - - ```sh - cd ${GOPATH}/src/github.com/pilosa/pilosa - ``` - -- Install Pilosa command line tools: - - ```sh - make install - ``` - - Running `pilosa` should now run a Pilosa instance. - -- The official Pilosa repository is your "origin" remote in git. Add your fork as your github username - - ```sh - cd ${GOPATH}/src/github.com/pilosa/pilosa - git remote add ${GH_USERNAME} git@github.com:${GH_USERNAME}/pilosa.git - ``` - -### Makefile - -Pilosa includes a Makefile that automates several tasks: - -- Install Pilosa: - - ```sh - make install - ``` - -- Install build dependencies: - - ```sh - make install-build-deps - ``` - -- Create the vendor directory: - - ```sh - make vendor - ``` - -- Run the test suite: - - ```sh - make test - ``` - -- View the coverage report: - - ```sh - make cover-viz - ``` - -- Clear the `vendor/` and `build/` directories: - - ```sh - make clean - ``` - -- Create release tarballs: - - ```sh - make release - ``` - -- Regenerate protocol buffer files in `internal/`: - - ```sh - make generate-protoc - ``` - -- Create tagged Docker image: - - ```sh - make docker - ``` - -- Run tests inside Docker container: - - ```sh - make docker-test - ``` - -Additional commands are available in the `Makefile`. - -### Submitting code changes - -- Before starting to work on a task, sync your branch with the upstream: - - ```sh - git checkout master - git pull - ``` - -- Create a local feature branch: - - ```sh - git checkout -b something-amazing - ``` - -- Commit your changes locally using `git add` and `git commit`. Please use [appropriate commit messages](https://chris.beams.io/posts/git-commit/). - -- Make sure that you've written tests for your new feature, and then run the tests: - - ```sh - make test - ``` - -- Verify that your pull request is applied to the latest version of code on github: - - ```sh - git checkout master - git pull - git checkout something-amazing - git rebase master - ``` - -- Push to your fork: - - ```sh - git push -u $GH_USERNAME something-amazing:something-amazing - ``` - -- Submit a [pull request][3] - - -[1]: https://github.com/pilosa/pilosa/issues/new -[2]: https://github.com/pilosa/pilosa -[3]: https://github.com/pilosa/pilosa/compare/ -[4]: https://github.com/pilosa/general/blob/master/proposal.md -[5]: https://github.com/pilosa/pilosa/issues -[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer diff --git a/LICENSE b/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/NOTES b/NOTES deleted file mode 100644 index 8da55fe11..000000000 --- a/NOTES +++ /dev/null @@ -1,26 +0,0 @@ - - Index Column - ┌───────────▼────────────────────────────┐ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - Row──▶0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │────────────────────────────────────────┤ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │────────────────────────────────────────┤ - F ▶│0000000000000000000000000000000000000000│ - i ││0000000000000000000000000000000000000000│ - e ││0000000000000000000000000000000000000000│ - l ││0000000000000000000000000000000000000000│ - d ▶│0000000000000000000000000000000000000000│ - └────────────────────────────────────────┘ - ▲───────────▲ - Shard - - -Fragment=intersection of field & shard From c72406819ee3eb46f056cf12c70429c42e2b675a Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 09:40:39 -0600 Subject: [PATCH 04/14] added seed param --- cmd/random-query/main.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 22f5209ad..7b01575ee 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -56,6 +56,7 @@ type RandomQueryConfig struct { TimeRange int64 // hours between parsed times Index string QPM int + Seed int SrcFile string Duration time.Duration Target vegeta.Target @@ -109,9 +110,10 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.") fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate") fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") + fs.IntVar(&cfg.Seed, "seed", 0, "RNG seed") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") - fs.IntVar(&cfg.QPM, "qps", 10, "number of currernt requests per minute to simulate, default 10") + fs.IntVar(&cfg.QPM, "qpm", 10, "number of currernt requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") @@ -179,7 +181,7 @@ func (cfg *RandomQueryConfig) Run() (err error) { if err != nil { return err } - rate := vegeta.Rate{Freq: cfg.QPM, Per: time.Second} + rate := vegeta.Rate{Freq: cfg.QPM, Per: time.Minute} duration := cfg.Duration targeter := vegeta.NewStaticTargeter(cfg.Target) attacker := vegeta.NewAttacker() @@ -348,8 +350,7 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { if foundIntField { cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct") } - seed := int64(42) - cfg.Rnd = rand.New(rand.NewSource(seed)) + cfg.Rnd = rand.New(rand.NewSource(int64(cfg.Seed))) return cfg.buildPayload() } From 10894352181a27addae7c0c87d2adb714581cbcc Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 09:48:45 -0600 Subject: [PATCH 05/14] add check for index presence --- cmd/random-query/main.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 7b01575ee..5fbe72290 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -113,7 +113,7 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.Seed, "seed", 0, "RNG seed") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") - fs.IntVar(&cfg.QPM, "qpm", 10, "number of currernt requests per minute to simulate, default 10") + fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") @@ -317,9 +317,10 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { return err } foundIntField := false + any := false for i, ii := range cfg.Info { if ii.Name == cfg.Index { - + any = true _ = i for k, fld := range ii.Fields { _ = k @@ -346,6 +347,9 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { } } } + if !any { + return errors.New(fmt.Sprintf("index %v not found", cfg.Index)) + } cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"} if foundIntField { cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct") From f7b4f621a15f3df150b3665b99787bddc20587b0 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 19 Nov 2021 10:38:06 -0600 Subject: [PATCH 06/14] remove references to LICENSE and checks for it in source files --- .circleci/config.yml | 11 --- .cloudbuild/Dockerfile | 1 - .cloudbuild/LICENSE | 202 ---------------------------------------- .cloudbuild/NOTICE | 13 +-- .gitlab/Dockerfile | 1 - Dockerfile | 1 - Dockerfile-clustertests | 1 - Dockerfile.pilosa | 1 - Makefile | 10 -- roaring/btree_test.go | 2 +- 10 files changed, 2 insertions(+), 241 deletions(-) delete mode 100644 .cloudbuild/LICENSE diff --git a/.circleci/config.yml b/.circleci/config.yml index 65534faa6..498c56ea3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,13 +63,6 @@ jobs: - checkout-plus - run: go mod download - save-mod-cache - check-license-headers: - executor: - name: golang - steps: - - checkout-plus - - skip-if-root-unchanged - - run: make check-license-headers linter: executor: name: golang @@ -239,10 +232,6 @@ workflows: context: molecula requires: - setup - - check-license-headers: - context: molecula - requires: - - setup - go-mod-tidy: context: molecula requires: diff --git a/.cloudbuild/Dockerfile b/.cloudbuild/Dockerfile index 407e5b994..fac3bf2fd 100644 --- a/.cloudbuild/Dockerfile +++ b/.cloudbuild/Dockerfile @@ -5,7 +5,6 @@ LABEL org.opencontainers.image.authors="dev@molecula.com" RUN apk add --no-cache curl jq -COPY LICENSE /LICENSE COPY NOTICE /NOTICE ARG SHORT_SHA diff --git a/.cloudbuild/LICENSE b/.cloudbuild/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/.cloudbuild/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/.cloudbuild/NOTICE b/.cloudbuild/NOTICE index 594272a27..5d88ea9c0 100644 --- a/.cloudbuild/NOTICE +++ b/.cloudbuild/NOTICE @@ -1,18 +1,7 @@ Software license ================ -Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright (C) 2017-2021 Molecula Corp. All rights reserved. Third-party software licenses ============================= diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index b22392ce1..b041f58ce 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -7,7 +7,6 @@ WORKDIR /featurebase RUN apk add --no-cache curl jq -COPY LICENSE . COPY NOTICE . COPY featurebase_linux_amd64 . diff --git a/Dockerfile b/Dockerfile index 115855034..25ea9c3f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,6 @@ RUN apk add --no-cache curl jq COPY --from=pilosa-builder /pilosa/build/featurebase / -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests index 8c60f11bf..4a3a0c196 100644 --- a/Dockerfile-clustertests +++ b/Dockerfile-clustertests @@ -20,7 +20,6 @@ RUN apt install -y docker.io RUN cp /go/bin/featurebase /featurebase -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 diff --git a/Dockerfile.pilosa b/Dockerfile.pilosa index a24c70ec7..6ee2e0bbf 100644 --- a/Dockerfile.pilosa +++ b/Dockerfile.pilosa @@ -24,7 +24,6 @@ RUN apk add --no-cache curl jq COPY --from=pilosa-builder /pilosa/build/featurebase / -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 diff --git a/Makefile b/Makefile index 55d1332ba..f72517684 100644 --- a/Makefile +++ b/Makefile @@ -18,10 +18,6 @@ GO_VERSION=1.16.10 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release BUILD_TAGS += shardwidth$(SHARD_WIDTH) TEST_TAGS = roaringparanoia -define LICENSE_HASH_CODE - head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/' -e 's/Pilosa Corp\./Molecula Corp./' | shasum | cut -f 1 -d " " -endef -LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go)) UNAME := $(shell uname -s) ifeq ($(UNAME), Darwin) IS_MACOS:=1 @@ -313,12 +309,6 @@ gometalinter: require-gometalinter vendor --exclude "^pql/pql.peg.go" \ ./... -# Verify that all Go files have license header -check-license-headers: SHELL:=/bin/bash -check-license-headers: - @! find . -path ./vendor -prune -o -name '*.go' -print | grep -v -F -f license.exceptions | while read fn;\ - do [[ `$(call LICENSE_HASH_CODE, $$fn)` == $(LICENSE_HASH) ]] || echo $$fn; done | grep '.' - ###################### # Build dependencies # ###################### diff --git a/roaring/btree_test.go b/roaring/btree_test.go index 7a952a937..ee51aeb0e 100644 --- a/roaring/btree_test.go +++ b/roaring/btree_test.go @@ -1,6 +1,6 @@ // Copyright 2014 The b Authors. All rights reserved. // Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. +// license that can be found in the NOTICE file. package roaring From b12f00db590dfe05549483219a0d087016d7ff43 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 10:57:36 -0600 Subject: [PATCH 07/14] changed default seed to current time --- cmd/random-query/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 5fbe72290..475ba9e60 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -110,7 +110,7 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.") fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate") fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") - fs.IntVar(&cfg.Seed, "seed", 0, "RNG seed") + fs.IntVar(&cfg.Seed, "seed", int(time.Now().Unix()), "RNG seed, defaults to currentime") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") @@ -440,6 +440,7 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci f = &Features{} cfg.IndexMap[index] = f } + vprint.VV("Add: %s %v %v %v", field, min, max, scale) /* if min.Scale != scale || max.Scale != scale { PanicOn(fmt.Sprintf("scale error; %v:%v min scale %d, max scale %d, field scale %d, assumed they'd be equal", From 25e5c6f5fbc1b9d4729cf5dcd6c98812e2366c65 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 19 Nov 2021 11:22:15 -0600 Subject: [PATCH 08/14] remove references to CONTRIBUTING and CHANGELOG --- .github/PULL_REQUEST_TEMPLATE.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 2c1f4979d..f579dc42f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,13 +6,10 @@ Fixes # ## Pull request checklist -- [ ] I have read the [contributing guide](https://github.com/molecula/featurebase/blob/master/CONTRIBUTING.md). -- [ ] I have agreed to the [Contributor License Agreement](https://cla-assistant.io/pilosa/pilosa). - [ ] I have updated the [documentation](https://github.com/molecula/docs). - [ ] I have resolved any merge conflicts. - [ ] I have included tests that cover my changes. - [ ] All new and existing tests pass. -- [ ] Make sure PR title conforms to convention in CHANGELOG.md. - [ ] Add appropriate changelog label to PR (if applicable). ## Code review checklist @@ -24,5 +21,4 @@ This is the checklist that the reviewer will follow while reviewing your pull re - [ ] Check that tests have been written and that they cover the new functionality. - [ ] Run tests and ensure they pass. - [ ] Build and run the code, performing any applicable integration testing. -- [ ] Make sure PR title conforms to convention in CHANGELOG.md. - [ ] Make sure PR is tagged with appropriate changelog label. From 989f9c3f48c480f58be9251c884131c5bdcae8c9 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 11:29:17 -0600 Subject: [PATCH 09/14] applied suggestions --- cmd/random-query/main.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 475ba9e60..0fcd71ce9 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -42,18 +42,16 @@ import ( // RandomQueryConfig type RandomQueryConfig struct { - - // user facing flags - HostPort string // -hostport - TreeDepth int // -d - QueryCount int // -n - Verbose bool // -v + HostPort string + TreeDepth int + QueryCount int + Verbose bool NumRuns int - TimeFromArg string // --time.from - TimeToArg string // --time.to - TimeFrom time.Time // parsed time - TimeTo time.Time // parsed time - TimeRange int64 // hours between parsed times + TimeFromArg string + TimeToArg string + TimeFrom time.Time + TimeTo time.Time + TimeRange int64 Index string QPM int Seed int @@ -110,10 +108,10 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.") fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate") fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") - fs.IntVar(&cfg.Seed, "seed", int(time.Now().Unix()), "RNG seed, defaults to currentime") + fs.IntVar(&cfg.Seed, "seed", int(time.Now().Unix()), "RNG seed, defaults to current time") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") - fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") + fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") @@ -142,6 +140,9 @@ func (c *RandomQueryConfig) ValidateConfig() error { return fmt.Errorf("time.to (%s) should be at least one hour after time.from (%s)", c.TimeToArg, c.TimeFromArg) } + if c.QPM <= 0 { + return fmt.Errorf("-qpm must be positive") + } return nil } From f4a40e79b976512113993c1b3e371624e01ce041 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 12:08:24 -0600 Subject: [PATCH 10/14] final cleanup --- cmd/random-query/main.go | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 0fcd71ce9..bef9a92c2 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -42,22 +42,23 @@ import ( // RandomQueryConfig type RandomQueryConfig struct { - HostPort string - TreeDepth int - QueryCount int - Verbose bool - NumRuns int - TimeFromArg string - TimeToArg string - TimeFrom time.Time - TimeTo time.Time - TimeRange int64 - Index string - QPM int - Seed int - SrcFile string - Duration time.Duration - Target vegeta.Target + HostPort string + TreeDepth int + QueryCount int + Verbose bool + GenerateOnly bool + NumRuns int + TimeFromArg string + TimeToArg string + TimeFrom time.Time + TimeTo time.Time + TimeRange int64 + Index string + QPM int + Seed int + SrcFile string + Duration time.Duration + Target vegeta.Target IndexMap map[string]*Features @@ -441,13 +442,6 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci f = &Features{} cfg.IndexMap[index] = f } - vprint.VV("Add: %s %v %v %v", field, min, max, scale) - /* if min.Scale != scale || max.Scale != scale { - - PanicOn(fmt.Sprintf("scale error; %v:%v min scale %d, max scale %d, field scale %d, assumed they'd be equal", - index, field, min.Scale, max.Scale, scale)) - } - */ effectiveRange := uint64(max.Value) - uint64(min.Value) + 1 // if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which From 5c5e16af33a7aeb1fb3f889ecbe72334b9dcb8ad Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 12:26:23 -0600 Subject: [PATCH 11/14] added a generate only flag --- cmd/random-query/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index bef9a92c2..e9dce0354 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -114,6 +114,7 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") + fs.BoolVar(&cfg.GenerateOnly, "generate-only", false, "only generate do not run package") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.SrcFile, "query-file", "", "use pql contained in this file for query batch instead of generating") @@ -390,6 +391,10 @@ func (cfg *RandomQueryConfig) buildPayload() error { Query: request.String(), } vprint.VV("%v", request.String()) + if cfg.GenerateOnly { + // just output the PQL and exit + os.Exit(0) + } payload, err := proto.Marshal(req) if err != nil { return errors.Wrap(err, "marshaling request to protobuf") From a02a8a94b36c2ad62edc89c4de02cc85db17edd8 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 19 Nov 2021 12:39:37 -0600 Subject: [PATCH 12/14] remove unused .cloudbuild directory (was for GCP CI stuff) --- .cloudbuild/Dockerfile | 23 ----- .cloudbuild/NOTICE | 109 -------------------- .cloudbuild/cloudbuild.yml | 202 ------------------------------------- 3 files changed, 334 deletions(-) delete mode 100644 .cloudbuild/Dockerfile delete mode 100644 .cloudbuild/NOTICE delete mode 100644 .cloudbuild/cloudbuild.yml diff --git a/.cloudbuild/Dockerfile b/.cloudbuild/Dockerfile deleted file mode 100644 index fac3bf2fd..000000000 --- a/.cloudbuild/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM alpine:3.14.2 - -LABEL maintainer "dev@molecula.com" -LABEL org.opencontainers.image.authors="dev@molecula.com" - -RUN apk add --no-cache curl jq - -COPY NOTICE /NOTICE -ARG SHORT_SHA - -COPY .cloudbuild/featurebase_linux_amd64 / - -RUN mv /featurebase_linux_amd64 /pilosa -RUN chmod ugo+x /pilosa -EXPOSE 10101 -VOLUME /data - -ENV PILOSA_DATA_DIR /data -ENV PILOSA_BIND 0.0.0.0:10101 -ENV PILOSA_BIND_GRPC 0.0.0.0:20101 - -ENTRYPOINT ["/pilosa"] -CMD ["server"] diff --git a/.cloudbuild/NOTICE b/.cloudbuild/NOTICE deleted file mode 100644 index 5d88ea9c0..000000000 --- a/.cloudbuild/NOTICE +++ /dev/null @@ -1,109 +0,0 @@ -Software license -================ - -Copyright (C) 2017-2021 Molecula Corp. All rights reserved. - -Third-party software licenses -============================= - -The file /pilosa/lru/lru.go contains a redistribution of lru -(github.com/golang/groupcache/lru); the license follows: - - Copyright 2013 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -The file /roaring/btree.go contains a modified redistribution of b -(https://github.com/cznic/b); the license follows: - - Copyright (c) 2014 The b Authors. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - * Neither the names of the authors nor the names of the - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The file /server/tlsconfig.go contains a modified redistribution of bridge -(https://github.com/robustirc/bridge); the license follows: - - Copyright © 2014-2015 The RobustIRC Authors. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of RobustIRC nor the names of contributors may be used - to endorse or promote products derived from this software without - specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows: - - The MIT License (MIT) - - Copyright (c) 2015 Nick Galbreath - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. diff --git a/.cloudbuild/cloudbuild.yml b/.cloudbuild/cloudbuild.yml deleted file mode 100644 index 4ec21187a..000000000 --- a/.cloudbuild/cloudbuild.yml +++ /dev/null @@ -1,202 +0,0 @@ -steps: - - name: gcr.io/cloud-builders/yarn - args: - - install - dir: lattice/ - id: Install Lattice - waitFor: - - '-' - - name: gcr.io/cloud-builders/yarn - args: - - build - dir: lattice/ - id: Build Lattice - waitFor: - - Install Lattice - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - tar - - '-c' - - '-z' - - '-v' - - '-f' - - lattice.tar.gz - - lattice/build - id: Package Lattice Build - waitFor: - - Build Lattice - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - mv - - lattice.tar.gz - - /cache/ - waitFor: - - Package Lattice Build - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - go - - test - - '-covermode=atomic' - - '-coverprofile=/cache/coverage.out' - id: Generate code coverage report - waitFor: - - '-' - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - '-c' - - | - go test -json > /cache/test-report.out - id: Generate test report - waitFor: - - '-' - entrypoint: bash - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - go - - tool - - cover - - '-html' - - /cache/coverage.out - - '-o' - - /cache/coverage.html - id: Calculate Coverage - waitFor: - - Generate code coverage report - - name: 'sonarsource/sonar-scanner-cli:4.6' - args: - - '-Dsonar.projectKey=molecula_featurebase' - - '-Dsonar.organization=molecula' - - '-Dsonar.sources=.' - - '-Dsonar.host.url=https://sonarcloud.io' - - '-Dsonar.go.coverage.reportPaths=/cache/coverage.out' - - '-Dsonar.go.tests.reportPaths=/cache/test-report.out' - id: SonarCloud - waitFor: - - Calculate Coverage - secretEnv: - - SONAR_TOKEN - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - jfrog - - config - - add - - molecula - - '--interactive=false' - - '--artifactory-url=https://molecula.jfrog.io/' - - '--access-token=$$JFROG_API_TOKEN' - secretEnv: - - JFROG_API_TOKEN - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - env: - - GOOS=linux - - GOARCH=amd64 - args: - - go - - build - - '-o' - - /cache/featurebase_linux_amd64 - - ./cmd/featurebase - id: Build for Linux amd64 - waitFor: - - '-' - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - env: - - GOOS=darwin - - GOARCH=amd64 - args: - - go - - build - - '-o' - - /cache/featurebase_darwin_amd64 - - ./cmd/featurebase - id: Build for Darwin amd64 - waitFor: - - '-' - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - env: - - GOOS=linux - - GOARCH=arm64 - args: - - go - - build - - '-o' - - /cache/featurebase_linux_arm64 - - ./cmd/featurebase - id: Build for Linux arm64 - waitFor: - - '-' - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - env: - - GOOS=darwin - - GOARCH=arm64 - args: - - go - - build - - '-o' - - /cache/featurebase_darwin_arm64 - - ./cmd/featurebase - id: Build for Darwin arm64 - waitFor: - - '-' - - name: gcr.io/cloud-builders/gsutil - args: - - '-c' - - > - gsutil -m cp -r '/cache/*' - "gs://molecula-pipeline/featurebase/dev/$SHORT_SHA/" - id: Upload Binaries by Short SHA - waitFor: - - Package Lattice Build - - Calculate Coverage - - Build for Linux amd64 - - Build for Darwin amd64 - - Build for Linux arm64 - - Build for Darwin arm64 - entrypoint: bash - - name: gcr.io/cloud-builders/gsutil - args: - - '-c' - - > - gsutil -m cp -r '/cache/*' - "gs://molecula-pipeline/featurebase/dev/$BRANCH_NAME/" - id: Upload Binaries by Branch Name - waitFor: - - Package Lattice Build - - Calculate Coverage - - Build for Linux amd64 - - Build for Darwin amd64 - - Build for Linux arm64 - - Build for Darwin arm64 - entrypoint: bash - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - > - cp -v /cache/* .cloudbuild/ && docker build --build-arg - SHORT_SHA=$SHORT_SHA -t - northamerica-northeast1-docker.pkg.dev/featurebase-pipeline-dev/featurebase/main:$SHORT_SHA - -f .cloudbuild/Dockerfile . - id: Build Linux amd Container - entrypoint: bash - - name: gcr.io/cloud-builders/docker - args: - - push - - >- - northamerica-northeast1-docker.pkg.dev/featurebase-pipeline-dev/featurebase/main:$SHORT_SHA - waitFor: - - Build Linux amd Container -timeout: 1200s -logsBucket: 'gs://molecula-pipeline/logs' -options: - volumes: - - name: data - path: /cache - pool: - name: >- - projects/featurebase-pipeline-dev/locations/us-central1/workerPools/test-pool-00 -availableSecrets: - secretManager: - - versionName: projects/903994035405/secrets/jfrog-admin-token/versions/1 - env: JFROG_API_TOKEN - - versionName: projects/903994035405/secrets/sonarcloud-featurebase-token/versions/1 - env: SONAR_TOKEN From c83170f1f025a8dd33d6b7d6c0370ac40f3884fc Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 19 Nov 2021 13:16:48 -0600 Subject: [PATCH 13/14] decrease sleep time --- .gitlab/.gitlab-ci.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 0dda19409..0dc7a60c5 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -205,7 +205,7 @@ deploy node for linux amd64: INSTANCE: "t3a.large" SECURITY_GROUP: "" SUBNET_ID: "" - PUBLIC_IP: "" + IP: "" PROFILE: "default" AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY before_script: @@ -229,11 +229,8 @@ deploy node for linux amd64: - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') - echo $INSTANCE_ID > linux-amd64-instance.txt - - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data - - PUBLIC_IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - - echo $PUBLIC_IP - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. + - while [ -z "$IP" ]; do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE); sleep 5; done + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$IP:. - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 needs: - job: build for linux amd64 From 2cde034e8287a1169d3e6e47de5e03c0ef30a865 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 19 Nov 2021 14:40:37 -0600 Subject: [PATCH 14/14] testing reviewer's suggestion --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 0dc7a60c5..fddcb9be6 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -229,7 +229,7 @@ deploy node for linux amd64: - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') - echo $INSTANCE_ID > linux-amd64-instance.txt - - while [ -z "$IP" ]; do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE); sleep 5; done + - for i in {0..24}; do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE); if [ -n "$IP" ]; then break; fi; sleep 5; done - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$IP:. - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 needs: