From 87ededd61e55ef88ba7973d050d1ae88679ec705 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 18 Nov 2021 16:23:12 -0600 Subject: [PATCH 01/12] Modified random query to use vegeta library --- .gitlab/cloud-init.sh | 6 +- cmd/random-query/main.go | 276 ++++++++++++++++++++-------------- cmd/random-query/main_test.go | 8 +- go.mod | 1 + go.sum | 41 +++++ 5 files changed, 210 insertions(+), 122 deletions(-) diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index 055d0f6aa..8d710d0b2 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -14,8 +14,8 @@ echo 'cat /proc/sys/fs/file-max' # yum install golang -y # latest verion in ec2 is 1.15.14 # install go 1.16.9 manually -curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz -tar xvf go1.16.9.linux-amd64.tar.gz +curl -O https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz +tar xvf go1.16.10.linux-amd64.tar.gz chown -R root:root ./go mv go /usr/local echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null @@ -23,4 +23,4 @@ source /etc/profile # install aws session manager pluggin curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" -yum install -y session-manager-plugin.rpm \ No newline at end of file +yum install -y session-manager-plugin.rpm diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 1faafad1e..5c3bc4ee0 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -18,34 +18,47 @@ import ( "context" "flag" "fmt" + "io/ioutil" "math" "math/rand" nethttp "net/http" + "os" "strconv" "strings" "time" - "github.com/molecula/featurebase/v2" + "github.com/gogo/protobuf/proto" + pilosa "github.com/molecula/featurebase/v2" + "github.com/molecula/featurebase/v2/client" "github.com/molecula/featurebase/v2/http" + "github.com/molecula/featurebase/v2/pb" "github.com/molecula/featurebase/v2/pql" + "github.com/molecula/featurebase/v2/vprint" . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck + "github.com/pkg/errors" + vegeta "github.com/tsenart/vegeta/v12/lib" ) // RandomQueryConfig type RandomQueryConfig struct { // user facing flags - HostPort string // -hostport - TreeDepth int // -d - QueryCount int // -n - Verbose bool // -v - VeryVerbose bool // -V + HostPort string // -hostport + TreeDepth int // -d + QueryCount int // -n + Verbose bool // -v + NumRuns int TimeFromArg string // --time.from TimeToArg string // --time.to TimeFrom time.Time // parsed time TimeTo time.Time // parsed time TimeRange int64 // hours between parsed times + Index string + QPS int + SrcFile string + Duration time.Duration + Target vegeta.Target IndexMap map[string]*Features @@ -93,12 +106,16 @@ var defaultStartTime = defaultEndTime.Add(-5 * 365 * 24 * time.Hour) // call DefineFlags before myflags.Parse() func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.") - fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.") - fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.") + fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.") + fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate") + fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") + fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") + fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") + fs.IntVar(&cfg.QPS, "qps", 10, "number of currernt requests per sec to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") - fs.BoolVar(&cfg.VeryVerbose, "V", false, "show query results") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") + fs.StringVar(&cfg.SrcFile, "query-file", "", "use pql contained in this file for query batch instead of generating") } // call c.ValidateConfig() after myflags.Parse() @@ -144,7 +161,6 @@ func main() { fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err) os.Exit(1) } - err = cfg.Run() if err != nil { @@ -159,76 +175,24 @@ func (cfg *RandomQueryConfig) Run() (err error) { if err != nil { return err } - ctx := context.Background() - totalQ := 0 - loops := 0 - t0 := time.Now() - - report := func() { - dur := time.Since(t0) - if dur > 0 { - qps := 1e9 * float64(totalQ) / float64(dur) - AlwaysPrintf("totalQueries run: %v elapsed: %v qps: %0.02f", totalQ, dur, qps) - } else { - AlwaysPrintf("totalQueries run: %v elapsed: %v qps: N/A", totalQ, dur) - } - } - defer report() - -NewSetup: err = cfg.Setup(cli) if err != nil { return err } + rate := vegeta.Rate{Freq: cfg.QPS, Per: time.Second} + duration := cfg.Duration + targeter := vegeta.NewStaticTargeter(cfg.Target) + attacker := vegeta.NewAttacker() - if len(cfg.IndexMap) == 0 { - return fmt.Errorf("no rows to query") - } - - var indexes []string - for index := range cfg.IndexMap { - indexes = append(indexes, index) - } - - for j := 0; ; j++ { - if cfg.QueryCount > 0 { - if j >= cfg.QueryCount { - break - } - } else { - // else keep doing queries forever... - if loops > 0 && loops%500 == 0 { - // ...but account for any new data arrived by getting - // the schema and rows again every so often. - loops++ - goto NewSetup - } + for i := 0; i < cfg.NumRuns; i++ { + vprint.VV("================") + var metrics vegeta.Metrics + for res := range attacker.Attack(targeter, rate, duration, "Big Bang!") { + metrics.Add(res) } - if totalQ > 0 && totalQ%100 == 0 { - report() - } - - index := indexes[cfg.Rnd.Intn(len(indexes))] - - pql, err := cfg.GenQuery(index) - PanicOn(err) - - if cfg.Verbose { - fmt.Printf("pql = '%v'\n", pql) - } - - // Query node0. - res, err := cli.Query(ctx, index, &pilosa.QueryRequest{Index: index, Query: pql}) - if err != nil { - AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql) - return err - } - if cfg.VeryVerbose { - fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) - } - totalQ++ - loops++ - + metrics.Close() + rpt := vegeta.NewTextReporter(&metrics) + rpt(os.Stdout) } return nil @@ -238,6 +202,7 @@ type Features struct { Slc []IndexFieldRow Ranges []IndexFieldRange Distinctables []IndexFieldRange + Stores []IndexFieldRow SlcWeight int RangeWeight int } @@ -283,7 +248,7 @@ func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree { endTime.Format(pilosaTimeFmt)) } if fea.IsRowKey { - return &Tree{S: fmt.Sprintf("Row(%v='%v'%s)", fea.Field, fea.RowKey, fromTo)} + return &Tree{S: fmt.Sprintf(`Row(%v="%v"%s)`, fea.Field, fea.RowKey, fromTo)} } return &Tree{S: fmt.Sprintf("Row(%v=%v%s)", fea.Field, fea.RowID, fromTo)} } @@ -341,6 +306,9 @@ func (i *IndexFieldRange) Query(cfg *RandomQueryConfig) *Tree { // and spits back a PQL query // func (cfg *RandomQueryConfig) Setup(api API) (err error) { + if cfg.SrcFile != "" { + return cfg.buildPayload() + } ctx := context.Background() cfg.Info, err = api.Schema(ctx) if err != nil { @@ -348,58 +316,114 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { } foundIntField := false for i, ii := range cfg.Info { - _ = i - for k, fld := range ii.Fields { - _ = k - switch fld.Options.Type { - case "set", "mutex", "time": - pql := fmt.Sprintf("Rows(%v)", fld.Name) + if ii.Name == cfg.Index { - res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql}) - PanicOn(err) - if cfg.VeryVerbose { - fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) - } - // if the option is set to use RowKeys, then must get the Keys instead of the Rows from the RowIdentifiers. - // e.g. - // success on pql = 'Rows(aba)'; res='&pilosa.RowIdentifiers{Rows:[]uint64(nil), Keys:[]string{"aba1", "aba2"} - // success on pql = 'Rows(f)'; res='pilosa.RowIdentifiers{Rows:[]uint64{0x1}, Keys:[]string(nil), field:"f"}' + _ = i + for k, fld := range ii.Fields { + _ = k + switch fld.Options.Type { + case "set", "mutex", "time": + pql := fmt.Sprintf("Rows(%v)", fld.Name) - switch x := res.Results[0].(type) { - case *pilosa.RowIdentifiers: - // internalClient gets this - cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time") - case pilosa.RowIdentifiers: - // test gets this - cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time") + res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql}) + PanicOn(err) + switch x := res.Results[0].(type) { + case *pilosa.RowIdentifiers: + cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time", fld.Options.Type == "set") + case pilosa.RowIdentifiers: + cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time", fld.Options.Type == "set") + } + case "int": + foundIntField = true + fallthrough + case "decimal": + cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal") + default: + AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) } - case "int": - foundIntField = true - fallthrough // I bet you thought you'd never see this used - case "decimal": - cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal") - default: - AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) } } } - cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"} if foundIntField { cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct") } seed := int64(42) cfg.Rnd = rand.New(rand.NewSource(seed)) - - return nil + return cfg.buildPayload() } -func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool) { +func (cfg *RandomQueryConfig) buildPayload() error { + var request strings.Builder + if cfg.SrcFile != "" { + b, err := ioutil.ReadFile(cfg.SrcFile) // just pass the file name + if err != nil { + return err + } + request.WriteString(string(b)) + } else { + + for i := 0; i < cfg.QueryCount; i++ { + pql, err := cfg.GenQuery(cfg.Index) + if err != nil { + return err + } + request.WriteString(pql) + } + } + //TODO (twg) tls support + path := fmt.Sprintf("http://%s/index/%s/query", cfg.HostPort, cfg.Index) + header := nethttp.Header{ + "Content-Type": []string{"application/x-protobuf"}, + "Accept": []string{"application/x-protobuf"}, + "PQL-Version": []string{client.PQLVersion}, + } + + req := &pb.QueryRequest{ + Query: request.String(), + } + vprint.VV("%v", request.String()) + payload, err := proto.Marshal(req) + if err != nil { + return errors.Wrap(err, "marshaling request to protobuf") + } + cfg.Target = vegeta.Target{ + Method: "POST", + URL: path, + Body: payload, + Header: header, + } + return nil +} +func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool, isSet bool) { + storeID := uint64(0) for _, rowID := range x.Rows { cfg.AddFeature(index, field, rowID, "", false, hasTime) + storeID = rowID } + storeKey := "" for _, rowKey := range x.Keys { cfg.AddFeature(index, field, 0, rowKey, true, hasTime) + storeKey = rowKey + } + idx := cfg.IndexMap[index] + if isSet { + if storeID > 0 { + idx.Stores = append(idx.Stores, IndexFieldRow{ + Index: index, + Field: field, + RowID: storeID + 1, + }) + } + if storeKey != "" { + idx.Stores = append(idx.Stores, IndexFieldRow{ + Index: index, + Field: field, + RowKey: storeKey + "_1", + IsRowKey: true, + }) + + } } } @@ -411,10 +435,12 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci f = &Features{} cfg.IndexMap[index] = f } - if min.Scale != scale || max.Scale != scale { - PanicOn(fmt.Sprintf("scale error; min scale %d, max scale %d, field scale %d, assumed they'd be equal", - min.Scale, max.Scale, scale)) - } + /* if min.Scale != scale || max.Scale != scale { + + PanicOn(fmt.Sprintf("scale error; %v:%v min scale %d, max scale %d, field scale %d, assumed they'd be equal", + index, field, min.Scale, max.Scale, scale)) + } + */ effectiveRange := uint64(max.Value) - uint64(min.Value) + 1 // if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which @@ -449,11 +475,32 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci } func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) { - tree := cfg.GenTree(index, cfg.TreeDepth) pql = tree.ToPQL() - // avoid using too much bandwidth, just count the final bitmap. + dice := cfg.Rnd.Intn(9) + if dice == 3 { // 1 in 9 of getting a store + idx := cfg.IndexMap[index] + if len(idx.Stores) > 0 { + i := cfg.Rnd.Intn(len(idx.Stores)) + fr := idx.Stores[i] + var key string + if fr.IsRowKey { + key = fmt.Sprintf(`%v="%v"`, fr.Field, fr.RowKey) + c := strings.LastIndex(fr.RowKey, "_") + n, err := strconv.Atoi(fr.RowKey[c+1:]) + PanicOn(err) + n += 1 + fr.RowKey = fmt.Sprintf("%v%v", fr.RowKey[:c+1], n) + } else { + key = fmt.Sprintf("%v=%v", fr.Field, fr.RowID) + fr.RowID = fr.RowID + 1 + } + idx.Stores[i] = fr + pql = fmt.Sprintf("Store(%v,%v)Count(Row(%v))", pql, key, key) + return + } + } pql = fmt.Sprintf("Count(%v)", pql) return } @@ -522,7 +569,6 @@ func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) { } func (tr *Tree) ToPQL() (s string) { - if len(tr.Chd) == 0 { // leaf return tr.S diff --git a/cmd/random-query/main_test.go b/cmd/random-query/main_test.go index eb6cc9040..7ad6936fa 100644 --- a/cmd/random-query/main_test.go +++ b/cmd/random-query/main_test.go @@ -20,7 +20,7 @@ import ( "strconv" "testing" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/boltdb" "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/server" @@ -65,8 +65,8 @@ func Test_RandomQuery(t *testing.T) { ctx := context.Background() - indexes := []string{"rick", "morty"} - fieldName := []string{"f", "flying_car"} + indexes := []string{"rick"} + fieldName := []string{"f"} idx := make([]*pilosa.Index, len(indexes)) field := make([]*pilosa.Field, len(indexes)) @@ -144,7 +144,7 @@ func Test_RandomQuery(t *testing.T) { //qcx.Reset() } // end of setup. - + cfg.Index = indexes[0] PanicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API))) for j := 0; j < 4; j++ { diff --git a/go.mod b/go.mod index 53a0860d3..add6082e3 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.7.0 + github.com/tsenart/vegeta/v12 v12.8.4 github.com/uber/jaeger-client-go v2.25.0+incompatible github.com/uber/jaeger-lib v2.4.0+incompatible // indirect github.com/zeebo/blake3 v0.1.1 diff --git a/go.sum b/go.sum index 0b9624b38..4224f965d 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,7 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/jsonschema v0.0.0-20180308105923-f2c93856175a/go.mod h1:qpebaTNSsyUn5rPSJMsfqEtDw71TTggXM6stUDI16HA= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -42,8 +43,11 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE= +github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -75,6 +79,9 @@ github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051 h1:ByJUvQYyTtNNCVfYNM48q6uYUT4fAlN0wNmd3th4BSo= +github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051/go.mod h1:qm+vckxRlDt0aOla0RYJJVeqHZlWfOm2UIxHaqPB46E= +github.com/dgryski/go-lttb v0.0.0-20180810165845-318fcdf10a77/go.mod h1:Va5MyIzkU0rAM92tn3hb3Anb7oz7KcnixF49+2wOMe4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= @@ -120,6 +127,24 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac h1:Q0Jsdxl5jbxouNs1TQYt0gxesYMU4VXRbsTlgDloZ50= +github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc= +github.com/gonum/diff v0.0.0-20181124234638-500114f11e71 h1:BE6g8oinc3Ek2elIHq+uDOiZgX3/ODi+EerJ48yrrKc= +github.com/gonum/diff v0.0.0-20181124234638-500114f11e71/go.mod h1:22dM4PLscQl+Nzf64qNBurVJvfyvZELT0iRW2l/NN70= +github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82 h1:EvokxLQsaaQjcWVWSV38221VAK7qc2zhaO17bKys/18= +github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82/go.mod h1:PxC8OnwL11+aosOB5+iEPoV3picfs8tUpkVd0pDo+Kg= +github.com/gonum/integrate v0.0.0-20181209220457-a422b5c0fdf2 h1:GUSkTcIe1SlregbHNUKbYDhBsS8lNgYfIp4S4cToUyU= +github.com/gonum/integrate v0.0.0-20181209220457-a422b5c0fdf2/go.mod h1:pDgmNM6seYpwvPos3q+zxlXMsbve6mOIPucUnUOrI7Y= +github.com/gonum/internal v0.0.0-20181124074243-f884aa714029 h1:8jtTdc+Nfj9AR+0soOeia9UZSvYBvETVHZrugUowJ7M= +github.com/gonum/internal v0.0.0-20181124074243-f884aa714029/go.mod h1:Pu4dmpkhSyOzRwuXkOgAvijx4o+4YMUJJo9OvPYMkks= +github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9 h1:7qnwS9+oeSiOIsiUMajT+0R7HR6hw5NegnKPmn/94oI= +github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9/go.mod h1:XA3DeT6rxh2EAE789SSiSJNqxPaC0aE9J8NTOI0Jo/A= +github.com/gonum/mathext v0.0.0-20181121095525-8a4bf007ea55 h1:Ajwn2ENgC/pKtVat0LEHEWNa4a4VGyYJ1feGSccOzFU= +github.com/gonum/mathext v0.0.0-20181121095525-8a4bf007ea55/go.mod h1:fmo8aiSEWkJeiGXUJf+sPvuDgEFgqIoZSs843ePKrGg= +github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9 h1:V2IgdyerlBa/MxaEFRbV5juy/C3MGdj4ePi+g6ePIp4= +github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9/go.mod h1:0EXg4mc1CNP0HCqCz+K4ts155PXIlUywf0wqN+GfPZw= +github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b h1:fbskpz/cPqWH8VqkQ7LJghFkl2KPAiIFUHrTJ2O3RGk= +github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b/go.mod h1:Z4GIJBJO3Wa4gD4vbwQxXXZ+WHmW6E9ixmNrwvs0iZs= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -184,6 +209,8 @@ github.com/improbable-eng/grpc-web v0.13.0 h1:7XqtaBWaOCH0cVGKHyvhtcuo6fgW32Y10y github.com/improbable-eng/grpc-web v0.13.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a h1:vMqgISSVkIqWxCIZs8m1L4096temR7IbYyNdMiBxSPA= +github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a/go.mod h1:9GkyshztGufsdPQWjH+ifgnIr3xNUL5syI70g2dzU1o= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -210,6 +237,8 @@ github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -217,6 +246,7 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.17/go.mod h1:WgzbA6oji13JREwiNsRDNfl7jYdPnmz+VEuLrA+/48M= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -321,6 +351,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 h1:7z3LSn867ex6VSaahyKadf4WtSsJIgne6A1WLOAGM8A= +github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -338,6 +370,9 @@ github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcy github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tsenart/go-tsz v0.0.0-20180814232043-cdeb9e1e981e/go.mod h1:SWZznP1z5Ki7hDT2ioqiFKEse8K9tU2OUvaRI0NeGQo= +github.com/tsenart/vegeta/v12 v12.8.4 h1:UQ7tG7WkDorKj0wjx78Z4/vsMBP8RJQMGJqRVrkvngg= +github.com/tsenart/vegeta/v12 v12.8.4/go.mod h1:ZiJtwLn/9M4fTPdMY7bdbIeyNeFVE8/AHbWFqCsUuho= github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ= @@ -367,6 +402,7 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -416,6 +452,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= @@ -446,6 +483,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -486,6 +524,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -559,6 +598,8 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +pgregory.net/rapid v0.3.3 h1:jCjBsY4ln4Atz78QoBWxUEvAHaFyNDQg9+WU62aCn1U= +pgregory.net/rapid v0.3.3/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 3b92f9493d5e3ce62ff53a91c49579db01e68ad1 Mon Sep 17 00:00:00 2001 From: nm Date: Tue, 9 Nov 2021 04:30:44 +0300 Subject: [PATCH 02/12] add config files for release --- Makefile | 13 +- NOTICE | 15 +- install/featurebase.conf | 373 +++++++++++++++++++++++++++ install/featurebase.debian.service | 13 + install/featurebase.redhat.service | 12 + install/test_installation.Dockerfile | 18 ++ install/test_installation.sh | 29 +++ 7 files changed, 458 insertions(+), 15 deletions(-) create mode 100644 install/featurebase.conf create mode 100644 install/featurebase.debian.service create mode 100644 install/featurebase.redhat.service create mode 100644 install/test_installation.Dockerfile create mode 100644 install/test_installation.sh diff --git a/Makefile b/Makefile index e2f659cf2..55d1332ba 100644 --- a/Makefile +++ b/Makefile @@ -104,10 +104,19 @@ build: # Create a single release build under the build directory release-build: $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/featurebase-$(VERSION_ID)/featurebase" - cp NOTICE README.md LICENSE build/featurebase$(VERSION_ID) + cp NOTICE install/featurebase.conf install/featurebase*.service build/featurebase-$(VERSION_ID) tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/ @echo Created release build: build/featurebase-$(VERSION_ID).tar.gz +test-release-build: docker-build + mv build/featurebase-$(VERSION_ID).tar.gz install/ + cd install && docker build -t featurebase:test_installation \ + -f test_installation.Dockerfile \ + --build-arg release_tarball=featurebase-$(VERSION_ID).tar.gz . + mv install/featurebase-$(VERSION_ID).tar.gz build/ + docker run -it -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + featurebase:test_installation + # Error out if there are untracked changes in Git check-clean: ifndef SKIP_CHECK_CLEAN @@ -220,7 +229,7 @@ docker-build: vendor docker create --name featurebase-build featurebase:build mkdir -p build/featurebase-$(VERSION_ID) docker cp featurebase-build:/pilosa/build/. ./build/featurebase-$(VERSION_ID) - cp NOTICE LICENSE ./build/featurebase-$(VERSION_ID) + cp NOTICE install/featurebase.conf install/featurebase*.service ./build/featurebase-$(VERSION_ID) docker rm featurebase-build tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/ diff --git a/NOTICE b/NOTICE index 594272a27..cb62dfdca 100644 --- a/NOTICE +++ b/NOTICE @@ -1,23 +1,12 @@ Software license ================ -Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright (C) 2017-2021 Molecula Corp. All rights reserved. Third-party software licenses ============================= -The file /pilosa/lru/lru.go contains a redistribution of lru +The file /lru/lru.go contains a redistribution of lru (github.com/golang/groupcache/lru); the license follows: Copyright 2013 Google Inc. diff --git a/install/featurebase.conf b/install/featurebase.conf new file mode 100644 index 000000000..73895ed6d --- /dev/null +++ b/install/featurebase.conf @@ -0,0 +1,373 @@ +# FEATUREBASE HOST CONFIGURATION +# +# Uncomment when/where appropriate + +# ============================================================================== +# Use advertise to specify the address advertised by the server to other nodes +# in the cluster and to clients via /status endpoint. Host defaults to IP +# address represented by bind parameter with network port. +# +# advertise = :10101 +# advertise-grpc = :20101 + + + +# "long-query-time" represents duration of time that will trigger log and stat +# message for queries longer than X time. Ex. "1m30s" 1 minute 30 seconds +# +# long-query-time = "10s" + + + +# Unique name for node in cluster. This is just a human-readable label for +# convenience and not used by any underlying logic. +# +# name = "featurebase1" + + + +# Host:Port where Featurebase server listens for HTTP requests. +# Default is localhost:10101 +# +# bind = "localhost:10101" + + + +# The address and port featurebase will listen to for all GRPC connections +# Ex. python-molecula, grafana for queries, etc. +# +# bind-grpc = "0.0.0.0:20101" + + + +# Directory to store Featurebase data files +data-dir = "/var/lib/molecula" + + +# ============================================================================== +# CORS (Cross-Origin Resource Sharing) Allowed Origins +# List of allowed origin URIs for CORS +# +# [handler] +# allowed-origins = ["https://myapp.com", "https://myapp.org"] + + + +# Path to the log file +log-path = "/var/log/molecula/featurebase.log" + + + +# Verbose - Enable verbose logging. Valid options are true or false. +# Set to true only when debugging as directed by Molecula engineers. +# +# verbose = true + + + +# Soft limit on max number of files featurebase will keep open simultaneously. +# When past this limit, featurebase will only keep files open for as long as is +# needed to write updates. +# +# max-file-count = 900000 + + + +# Maximum number of active memory maps featurebase will use for fragment files. +# Actual total usage may be slightly higher. +# Best practice is to set this to ~10% lower than your system's max map count. +# See sysctl vm.max_map_count in Linux. +# +# max-map-count = 900000 + + + +# Max Writes Per Request - Max number of mutating commands allowed per request. +# This includes Set, Clear, ClearRow, and Store +# +# max-writes-per-request = 5000 + + + +# The following option sets the maximum number of queries that are maintained +# for the /query-history endpoint. +# This parameter is per-node, and the result combines the history from all nodes. +# +# query-history-length = 100 + + + +# External database to connect to for `ExternalLookup` queries. +# lookup-db-dsn = "postgres://localhost:5432/db" + + + +# ============================================================================== +# For cluster stanza, "name" represents name for cluster. Must be same on all +# nodes in cluster. "replicas" represents number of hosts each piece of data +# should be stored on. Must be greater than or equal to 1 & less than or equal +# to number of nodes in cluster. +# [cluster] +# name = "cluster1" +# replicas = 1 + + + +# ============================================================================== +# [etcd] +# etcd is the tool Featurebase uses for node-to-node, intra-cluster +# communication. etcd is embedded in the featurebase cluster rather than +# running as a separate instance. +# It's important to configure this correctly for your network and nodes, and +# that it is consistent across all nodes. +# +# The easiest setup can be used when all nodes can reach all other nodes via a +# local subnet: +# listen-peer-address = advertise-peer-address +# = (what's in the initial-cluster-list) +# = the nodes ip address (which can be reached by every +# other node +# (localhost:10401 would not work for this, as each node can't reach that) +# +# If each node is separated by a proxy, or must be reached via url / dns, you +# will need to use a more complicated setup: +# listen-peer-address = the nodes local ip address +# (specific ip, localhost, or 0.0.0.0 for all +# local ip's) +# advertise-peer-address = the nodes ip address, reachable by all other nodes +# (This address should also be included in +# initital-cluster-list) +# in this case, you specify a different url/ip for listen-peer and +# advertise-peer. E.g. you specify 0.0.0.0 for listen, or (like in their case) +# you use a url for advertise. In each of these cases, you should set listen +# to the local ip, and you set advertise = to how each other node connects to +# this node, and you also use this same address in the initial cluster. +# The key here is that initial-cluster has to include the same node name and +# advertise-peer address as the node it's on (edited) + + + +# for additional assistance, and for help with config issues, +# see https://etcd.io/docs/v3.5/faq/ cluster-url - URL of existing cluster +# that a new node should join when adding nodes to cluster. +# +# cluster-url = "http://localhost:10401" + + + +# Address and port to bind to for client communication +# listen-client-address = "http://localhost:10401" + + + +# Address and port to bind to for peer communication +# listen-peer-address = "http://localhost:10301" + + + +# Comma-separated list of node=address pairs that makes up initial cluster when +# first started. In each pair, "node" value (left side of = ) should match +# name of node specified by "name" configuration parameter +# +# initial-cluster = "featurebase1=http://localhost:10301" + + + +# ============================================================================== +# Profile Block Rate - Block Rate is passed directly to Go's +# runtime.SetBlockProfileRate. Goroutine blocking events will be sampled at 1 +# per rate nanoseconds. A value of "1" samples every event, and 0 disables +# profiling. +# +# block-rate = 10000000 + +# Profile Mutex Fraction - Mutex Fraction is passed directly to Go's +# runtime.SetMutexProfileFraction. 1/ fraction of events will be sampled. +# +# mutex-fraction = 100 + + + +# ============================================================================== +# PostgreSQL Section +# [postgres] +# +# Endpoint Bind - Address to bind a PostgreSQL wire protocol endpoint. +# No PostgreSQL endpoint will be exposed unless a bind address is specified. +# Requires Molecula v3.0 or newer. +# +# bind = "localhost:55432" + + + +# The PostgreSQL endpoint has support for a connection limit. +# This is generally not necessary, so it is disabled by default. +# +# connection-limit = 10000 + + + +# PostgreSQL Max Startup Packet Size - By default, the postgres endpoint +# uses an 8 MiB limit on incoming PostgreSQL startup packets. This should +# typically be sufficient, but may be exceeded if a client sends an unusually +# large amount of configuration data. Oversized startup packets are typically +# caused by connecting with a different protocol, e.g. HTTP. +# +# max-startup-size = 10000000 + + + +# PostgreSQL Timeouts +# In order to detect stalled clients, the PostgreSQL endpoint has connection +# read and write timeouts. There is also a startup timeout, which is used for +# connection setup. The read timeout does not impact idle connections. Idle +# connections will only be closed by the server if TCP keepalive reports a +# break in the connection. TCP keepalives use the default configuration +# provided by the host. +# Caution: Due to a limitation of the PostgreSQL wire protocol, +# raising the write timeout may delay the shutdown of a featurebase node. +# +# startup-timeout = "20s" +# read-timeout = "20s" +# write-timeout - "20s" + + + +# Postgres Endpoint TLS - TLS configuration for the PostgreSQL endpoint is +# structured the same as the TLS configuration for Featurebase's other endpoints, +# but placed under [postgres.tls]. If TLS is configured on the postgres endpoint, +# Featurebase will reject unsecured connections. +# [postgres.tls] +# certificate = "/srv/pilosa/certs/server.crt" +# key = "/srv/pilosa/certs/server.key" +# ca-certificate = "/srv/pilosa/certs/ca.crt" +# enable-client-verification = true + + + +# ============================================================================== +# Usage Duty Cycle - Featurebase maintains a disk/memory usage cache that is +# calculated periodically in the background and accessed by the UI/usage +# endpoint. Since this disk scan can take a long and unpredictable amount of +# time, its timing behavior is specified in a relative, rather than absolute +# sense. That is, the duty cycle sets the percentage of time that is spent +# recalculating this cache. This setting affects the results received from +# the "/ui/usage" http endpoint, as well as all data file and memory usage +# values and graphs on the webui "tables" page + +# Special considerations: +# * If disk usage can be calculated quickly (less than 5 seconds), fresh +# results will be calculated when accessed +# * When disk usage takes longer to calculate, there is a minimum of one +# hour wait between cache recalculations +# Setting this value to 0 will completely disable the calculation of disk usage +# +# usage-duty-cycle = 20 + + + +# ============================================================================== +# Use [metric] stanza to define attributes for monitoring. +# [metric] +# Specify which service to use for collecting metrics. Valid options are: +# "statsd", "expvar", "prometheus", "none" +# +# service = "prometheus" + + + +# Remote host to send statsd metrics to. +# host = "localhost:8125" + + + +# The interval to send statsd metrics. +# poll-interval = "10s" + + + +# Debugging flag to enable to send diagnostic information to Featurebase +# developers. +# +# diagnostics = false + + + +# ============================================================================== +# TLS Certificate Section - Path to TLC certificate used for service HTTPS. +# Suffix should contain .crt or .pem +# +# [tls] +# certificate = "/srv/pilosa/certs/server.crt" +# TLS Certificate Key - Path to TLS certificate key for HTTPS. Suffix should +# be .key +# +# key = "/srv/pilosa/certs/server.key" + + + +# ============================================================================== +# Tracing Section +# [tracing] +# +# Jaeger sampler type. Valid options are: "const, "probabilistic", "ratelimiting", +# or "remote". Set to 'off' to disable tracing completely. +# +# sampler-type = "remote" + + +# Jaeger sampler parameter (number) +# sampler-param = 0.001 + + + +# Tracing Agent Host:Port +# agent-host-port = "localhost:6831" + + + +# ============================================================================== +# Configuration for the RBF storage format. +# [rbf] +# Maximum size for each RBF database file. +# Allocates virtual memory but does not preallocate physical disk space. +# If you get into the range where you have 16000 shards on a single node +# (across all indexes), you will need to lower this in order to not run out of +# virtual address space. +# +# max-db-size = 4294967296 + + + +# Maximum size for each RBF WAL file. +# Allocates virtual memory but does not preallocate physical disk space. +# This is the same as max-db-size, but for the write-ahead log. If you set it +# smaller, set max-wal-checkpoint-size to 1/2 of this (we will likely +# condense these options in the future). +# +# max-wal-size = 4294967296 + + + +# Minimum WAL size before WAL pages can be copied to the main database file. +# min-wal-checkpoint-size = 1048576 + + + +# Maximum WAL size before transactions are halted to copy WAL pages to the +# main database file. +# +# max-wal-checkpoint-size = 2147483648 + + +# ============================================================================== +# [storage] +# Sync all changes to the file system. +# Should not be changed in production systems unless you know what you are +# doing - Should always be on unless testing or possibly while performing a +# bulk import and you are not worried about data loss +# +# fsync = true + + +# ============================================================================== diff --git a/install/featurebase.debian.service b/install/featurebase.debian.service new file mode 100644 index 000000000..162264106 --- /dev/null +++ b/install/featurebase.debian.service @@ -0,0 +1,13 @@ +[Unit] +Description="Service for FeatureBase" +After=network.target + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] +WantedBy=multi-user.target diff --git a/install/featurebase.redhat.service b/install/featurebase.redhat.service new file mode 100644 index 000000000..cf507d8b0 --- /dev/null +++ b/install/featurebase.redhat.service @@ -0,0 +1,12 @@ +[Unit] +Description="Service for FeatureBase" + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] +WantedBy=multi-user.target diff --git a/install/test_installation.Dockerfile b/install/test_installation.Dockerfile new file mode 100644 index 000000000..5d9a93253 --- /dev/null +++ b/install/test_installation.Dockerfile @@ -0,0 +1,18 @@ +FROM fedora:rawhide + +RUN yum -y install systemd procps + +ARG release_tarball + +COPY $release_tarball . + +COPY test_installation.sh test_installation.sh + +RUN mkdir install && \ + tar -xf $release_tarball -C install --strip-components 1 && \ + cd install && \ + cp featurebase /usr/local/bin/featurebase && \ + cp featurebase.redhat.service /etc/systemd/system/featurebase.service && \ + cp featurebase.conf /etc/featurebase.conf + +CMD ["/bin/bash", "test_installation.sh"] diff --git a/install/test_installation.sh b/install/test_installation.sh new file mode 100644 index 000000000..4a301ea48 --- /dev/null +++ b/install/test_installation.sh @@ -0,0 +1,29 @@ +#!/bin/bash +tests_failed=0 + +# test validity of featurebase.service +systemd-analyze verify /etc/systemd/system/featurebase.service +exit_code=$? +if [ $exit_code -ne 0 ]; then + echo 'featurebase.redhat.service invalid' + tests_failed=1 +fi + +# test validity of featurebase.conf +mkdir /var/log/molecula +featurebase_bin='/usr/local/bin/featurebase' +config_file='/etc/featurebase.conf' +if ! $featurebase_bin -c $config_file holder /dev/null 2>&1; then + echo 'featurebase.conf is invalid' + tests_failed=1 +fi + +# print success if both passed +if [ $tests_failed -eq 0 ]; then + echo 'featurebase.redhat.service is valid' + echo 'featurebase.conf is valid' +fi + +# tests_failed set to 0 if none of the tests failed +# otherwise set to non-zero +exit $tests_failed From f6b3a81ac3694b87fe0b8f460c83f90576768e68 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 05:16:30 -0600 Subject: [PATCH 03/12] made duration minutes for query rate --- cmd/random-query/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 5c3bc4ee0..22f5209ad 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -55,7 +55,7 @@ type RandomQueryConfig struct { TimeTo time.Time // parsed time TimeRange int64 // hours between parsed times Index string - QPS int + QPM int SrcFile string Duration time.Duration Target vegeta.Target @@ -111,7 +111,7 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") - fs.IntVar(&cfg.QPS, "qps", 10, "number of currernt requests per sec to simulate, default 10") + fs.IntVar(&cfg.QPM, "qps", 10, "number of currernt requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") @@ -179,7 +179,7 @@ func (cfg *RandomQueryConfig) Run() (err error) { if err != nil { return err } - rate := vegeta.Rate{Freq: cfg.QPS, Per: time.Second} + rate := vegeta.Rate{Freq: cfg.QPM, Per: time.Second} duration := cfg.Duration targeter := vegeta.NewStaticTargeter(cfg.Target) attacker := vegeta.NewAttacker() From b3b11638a32d6b80cba4406e2c3d328170b6355c Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 9 Nov 2021 08:23:28 -0600 Subject: [PATCH 04/12] remove outdated files at top level we're no longer Apache 2.0 licensed, or open source, so LICENSE and CONTRIBUTING.MD are gone. We track the changelog elsewhere, so that can go, and I don't think anyone has looked at the NOTES file in 3 years. I modified the NOTICE not to refer to the Apache license any more. --- CHANGELOG.md | 715 ------------------------------------------------ CONTRIBUTING.md | 177 ------------ LICENSE | 202 -------------- NOTES | 26 -- 4 files changed, 1120 deletions(-) delete mode 100644 CHANGELOG.md delete mode 100644 CONTRIBUTING.md delete mode 100644 LICENSE delete mode 100644 NOTES diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 57de1f746..000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,715 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/) -and this project adheres to [Semantic Versioning](http://semver.org/). - -## [1.4.0] - 2019-09-17 - -This version contains 99 contributions from 11 contributors. There are 94 files changed; 9,453 insertions; and 6,121 deletions. - -**Attention**: Pilosa 1.4.0 changes the way that integer fields are stored. The upgrade from old format to new is handled automatically, however you will not be able to downgrade to 1.3 should you wish to do so. We *always* recommend taking a backup of your Pilosa data directory before upgrading Pilosa, but doubly so with this release. - -### Added -- Update "Getting Started" documentation ([#2028](https://github.com/pilosa/pilosa/pull/2028)) -- Add ability to disable tracing and use nopTracer ([#2029](https://github.com/pilosa/pilosa/pull/2029)) -- Add test for no containers ([#2016](https://github.com/pilosa/pilosa/pull/2016)) -- Add naive implementations of Roaring and fuzz test ([#2023](https://github.com/pilosa/pilosa/pull/2023)) -- Add fuzzing code and readme.md to explain the fuzzer ([#2004](https://github.com/pilosa/pilosa/pull/2004)) -- Add MinRow and MaxRow calls ([#1983](https://github.com/pilosa/pilosa/pull/1983)) -- Add Prometheus stats backend ([#1992](https://github.com/pilosa/pilosa/pull/1992)) -- Add extra tracing spans and metadata ([#1939](https://github.com/pilosa/pilosa/pull/1939)) -- Add more Debugf() statements to the holder open process ([#1950](https://github.com/pilosa/pilosa/pull/1950)) -- Add ability to post schema using holder.applySchema ([#1956](https://github.com/pilosa/pilosa/pull/1956)) - -### Changed -- Update CircleCI build with Go 1.13 and run enterprise tests ([#2064](https://github.com/pilosa/pilosa/pull/2064)) -- Update Alpine to 3.9.4 in Dockerfile ([#2001](https://github.com/pilosa/pilosa/pull/2001)) -- Add Prometheus tests, refactor http stats as middleware, minor fixes ([#1994](https://github.com/pilosa/pilosa/pull/1994)) -- Add confirmation logic to catch false nodeLeave events ([#1993](https://github.com/pilosa/pilosa/pull/1993)) -- Improve TopN() errors ([#1978](https://github.com/pilosa/pilosa/pull/1978)) -- Make integer fields unbounded by using sign+magnitude representation ([#1902](https://github.com/pilosa/pilosa/pull/1902)) -- Simplify contributing instructions by removing weird upstream thing ([#1966](https://github.com/pilosa/pilosa/pull/1966)) - -### Fixed -- Default BSI base value to min, max, or 0 depending on the min/max range ([#2050](https://github.com/pilosa/pilosa/pull/2050)) -- Add worker pool for query processing ([#2034](https://github.com/pilosa/pilosa/pull/2034)) -- Move Range deprecation message to higher level ([#2033](https://github.com/pilosa/pilosa/pull/2033)) -- Use lock in view.deleteFragment while altering fragments ([#2026](https://github.com/pilosa/pilosa/pull/2026)) -- Fix malformed offset bug in readOffsets and readWithRuns ([#2021](https://github.com/pilosa/pilosa/pull/2021)) -- Fix various container iteration bugs in Roaring ([#2019](https://github.com/pilosa/pilosa/pull/2019)) -- Fix malformed bitmap handling ([#2017](https://github.com/pilosa/pilosa/pull/2017)) -- Fix fuzzer errors in roaring ([#2012](https://github.com/pilosa/pilosa/pull/2012)) -- Save all state files atomically to avoid corruption ([#2000](https://github.com/pilosa/pilosa/pull/2000)) -- Fix slice container updates ([#1997](https://github.com/pilosa/pilosa/pull/1997)) -- Fix out of bounds panic to show error ([#1975](https://github.com/pilosa/pilosa/pull/1975)) -- Fix error message returned by regex on field and index names ([#1973](https://github.com/pilosa/pilosa/pull/1973)) -- Fix filter calls in GroupBy not being translated ([#1970](https://github.com/pilosa/pilosa/pull/1970)) -- Fix TranslateFile behavior when reopened ([#1954](https://github.com/pilosa/pilosa/pull/1954)) -- Remove buggy shard validation code ([#1951](https://github.com/pilosa/pilosa/pull/1951)) -- Fix some lint warnings raised in VS-Code ([#1947](https://github.com/pilosa/pilosa/pull/1947)) - -### Performance -- Address some startup speed and performance issues ([#1988](https://github.com/pilosa/pilosa/pull/1988)) -- Add a worker pool for importRoaring jobs ([#2048](https://github.com/pilosa/pilosa/pull/2048)) -- Use UnionInPlace for computing time rows which involve multiple views ([#2041](https://github.com/pilosa/pilosa/pull/2041)) -- Improve ingest performance with snapshot queue and unmarshaling improvements ([#2024](https://github.com/pilosa/pilosa/pull/2024)) -- Improve row cache ([#1974](https://github.com/pilosa/pilosa/pull/1974)) - -### Removed -- Remove extraneous stat tags to improve prometheus performance ([#1996](https://github.com/pilosa/pilosa/pull/1996)) - -## [1.3.1] - 2019-05-01 - -This version contains 1 contribution from 1 contributor. There are 6 files changed; 10 insertions; and 95 deletions. - -### Fixed -- Remove shard validation to fix bug where some nodes weren't loading their fragments. #1951 ([#1964](https://github.com/pilosa/pilosa/pull/1964)) - -## [1.3.0] - 2019-04-16 - -This version contains 98 contributions from 10 contributors. There are 144 files changed; 12,635 insertions; and 4,341 deletions. - -### Added -- Add license headers and CI check ([#1940](https://github.com/pilosa/pilosa/pull/1940)) -- Add support to modify shard width at build time ([#1921](https://github.com/pilosa/pilosa/pull/1921)) -- Add 'bench' Makefile target and run fewer concurrency level benchmarks ([#1915](https://github.com/pilosa/pilosa/pull/1915)) -- Add server stats to /info endpoint ([#1859](https://github.com/pilosa/pilosa/pull/1859)) -- Implement config options for block profile rate and mutex fraction ([#1910](https://github.com/pilosa/pilosa/pull/1910)) -- Implement global open file counter using syswrap (to scale past system open file limits) ([#1906](https://github.com/pilosa/pilosa/pull/1906)) -- Implement global mmap counter with fallback (to scale past system mmap limits) ([#1903](https://github.com/pilosa/pilosa/pull/1903)) -- Add shard width to index info in schema (allows client to get shard width at run time) ([#1881](https://github.com/pilosa/pilosa/pull/1881)) -- Add shift operator ([#1761](https://github.com/pilosa/pilosa/pull/1761)) -- Support advertise address and listen on 0.0.0.0 ([#1832](https://github.com/pilosa/pilosa/pull/1832)) -- Added convenience function to efficiently calculate size of a roaring bitmap in bytes ([#1839](https://github.com/pilosa/pilosa/pull/1839)) -- Make sure more tests and benchmarks can have their temp dir set by flag ([#1831](https://github.com/pilosa/pilosa/pull/1831)) -- Add sliceascending/slicedescending striped benchmarks ([#1763](https://github.com/pilosa/pilosa/pull/1763)) -- Add setValue test and benchmarks ([#1820](https://github.com/pilosa/pilosa/pull/1820)) -- Add a test for groupby filter with RangeLTLT ([#1818](https://github.com/pilosa/pilosa/pull/1818)) -- Add tests for GroupBy with keys; removes unused Bit message from proto ([#1811](https://github.com/pilosa/pilosa/pull/1811)) - -### Fixed -- Update to latest memberlist fork with race fixes ([#1944](https://github.com/pilosa/pilosa/pull/1944)) -- Return original error instead of cause in handler ([#1943](https://github.com/pilosa/pilosa/pull/1943)) -- Validate (and panic) on duplicate PQL arguments ([#1938](https://github.com/pilosa/pilosa/pull/1938)) -- Add correct content type to query responses Fixes #1873 ([#1936](https://github.com/pilosa/pilosa/pull/1936)) -- Address race condition by getting cluster nodes with lock ([#1931](https://github.com/pilosa/pilosa/pull/1931)) -- Make sure to unmap containers before modifying ([#1876](https://github.com/pilosa/pilosa/pull/1876)) -- Avoid probable race when creating fragments ([#1863](https://github.com/pilosa/pilosa/pull/1863)) -- Improve help strings for metrics options ([#1887](https://github.com/pilosa/pilosa/pull/1887)) -- Ensure ClearRow() arguments get translated ([#1848](https://github.com/pilosa/pilosa/pull/1848)) -- Prevent omitting zero ids on columnattrs ([#1846](https://github.com/pilosa/pilosa/pull/1846)) -- Set cache size to 0 if cache type is none ([#1842](https://github.com/pilosa/pilosa/pull/1842)) -- Prevent deadlock in replication logic on reopening a store ([#1834](https://github.com/pilosa/pilosa/pull/1834)) -- Pass loggers around properly in gossip ([#1835](https://github.com/pilosa/pilosa/pull/1835)) -- Include read lock in cluster.Nodes() ([#1836](https://github.com/pilosa/pilosa/pull/1836)) -- Raise an error on Rows() query against a time field with noStandardView: true ([#1826](https://github.com/pilosa/pilosa/pull/1826)) -- Don't delete test fragment data (part of repo) ([#1827](https://github.com/pilosa/pilosa/pull/1827)) -- Fix bug on upper end of bsi range queries ([#1822](https://github.com/pilosa/pilosa/pull/1822)) -- Group by fixes ([#1802](https://github.com/pilosa/pilosa/pull/1802)) - -### Changed -- Switch to GolangCI lint ([#1924](https://github.com/pilosa/pilosa/pull/1924)) -- Return empty result set when query empty ([#1937](https://github.com/pilosa/pilosa/pull/1937)) -- Add Go 1.12 to CircleCI ([#1909](https://github.com/pilosa/pilosa/pull/1909)) -- Ignore fragment files from shards node doesn't own ([#1900](https://github.com/pilosa/pilosa/pull/1900)) -- Go module support. Use Modules instead of dep for dependencies ([#1616](https://github.com/pilosa/pilosa/pull/1616)) -- Merge Range() into Row() call. ([#1804](https://github.com/pilosa/pilosa/pull/1804)) -- Add from/to range arguments to Rows() call ([#1851](https://github.com/pilosa/pilosa/pull/1851)) -- Fixes Store call error messages, Rows doesn't need field argument ([#1830](https://github.com/pilosa/pilosa/pull/1830)) - -### Performance -- BTree performance improvements ([#1916](https://github.com/pilosa/pilosa/pull/1916)) -- Make Containers smaller, especially when they have small contents ([#1901](https://github.com/pilosa/pilosa/pull/1901)) -- Address UnionInPlace performance regressions ([#1897](https://github.com/pilosa/pilosa/pull/1897)) -- Small write path for import-roaring. Makes small imports faster ([#1892](https://github.com/pilosa/pilosa/pull/1892)) -- Small write path for imports ([#1871](https://github.com/pilosa/pilosa/pull/1871)) -- Remove copy for pilosa roaring files ([#1865](https://github.com/pilosa/pilosa/pull/1865)) -- Disable anti-entropy if not using replication [performance] ([#1814](https://github.com/pilosa/pilosa/pull/1814)) -- Group By—skip 0 counts as early as possible ([#1803](https://github.com/pilosa/pilosa/pull/1803)) - -## [1.2.0] - 2018-12-20 - -This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions. - -### Added - -- Cancel queries on Context.Done() ([#1773](https://github.com/pilosa/pilosa/pull/1773)) -- Union In Place ([#1766](https://github.com/pilosa/pilosa/pull/1766), [#1774](https://github.com/pilosa/pilosa/pull/1774)) -- Import benchmarking ([#1771](https://github.com/pilosa/pilosa/pull/1771)) -- Add GroupBy() Filter ([#1753](https://github.com/pilosa/pilosa/pull/1753)) -- Add /internal/translate/keys endpoint ([#1751](https://github.com/pilosa/pilosa/pull/1751)) -- CircleCI: Add race detector to parallel build, default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756)) -- Add distributed tracing. ([#1684](https://github.com/pilosa/pilosa/pull/1684)) -- Add NoStandardView field option ([#1733](https://github.com/pilosa/pilosa/pull/1733)) -- Add some stat tracking to roaring implementation ([#1743](https://github.com/pilosa/pilosa/pull/1743)) -- Add cluster fault testing using docker-compose and pumba ([#1717](https://github.com/pilosa/pilosa/pull/1717)) -- Allow backslash, carriage return in PQL strings ([#1713](https://github.com/pilosa/pilosa/pull/1713)) -- Add base system, curl and jq for debug and checks ([#1707](https://github.com/pilosa/pilosa/pull/1707)) -- Add `Rows` and `GroupBy` functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647)) -- Add `clear` functional option for imports ([#1699](https://github.com/pilosa/pilosa/pull/1699)) -- Implement tracking of available shards to help support sparse datasets ([#1600](https://github.com/pilosa/pilosa/pull/1600), [#1695](https://github.com/pilosa/pilosa/pull/1695), [#1624](https://github.com/pilosa/pilosa/pull/1624), [#1663](https://github.com/pilosa/pilosa/pull/1663)) -- Add missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683)) -- Add Store() operation to PQL ([#1666](https://github.com/pilosa/pilosa/pull/1666)) -- Add diagnostics CPUArch field ([#1671](https://github.com/pilosa/pilosa/pull/1671)) -- Add CircleCI step to generate Docker image and push to Docker hub ([#1673](https://github.com/pilosa/pilosa/pull/1673)) -- Implement ClearRow() query ([#1645](https://github.com/pilosa/pilosa/pull/1645)) -- Add support for Bool fields ([#1658](https://github.com/pilosa/pilosa/pull/1658)) -- Make translate map size configurable ([#1653](https://github.com/pilosa/pilosa/pull/1653)) -- Add DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646)) -- Implement Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622), [#1738](https://github.com/pilosa/pilosa/pull/1738)) -- Add Not() query ([#1635](https://github.com/pilosa/pilosa/pull/1635)) -- Implement Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631)) -- Add field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625)) -- Implement column existence tracking ([#1788](https://github.com/pilosa/pilosa/pull/1788), [#1672](https://github.com/pilosa/pilosa/pull/1672), [#1628](https://github.com/pilosa/pilosa/pull/1628)) - -### Changed - -- Convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` ([#1780](https://github.com/pilosa/pilosa/pull/1780)) -- Simplify `require-*` logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755)) -- Cleanup logging ([#1748](https://github.com/pilosa/pilosa/pull/1748)) -- Remove TravisCI, add CircleCI shield ([#1740](https://github.com/pilosa/pilosa/pull/1740)) -- Upgrade Peg dependency and regenerate grammar ([#1725](https://github.com/pilosa/pilosa/pull/1725)) -- Upgrade to protoc 3.6.1 (also updated protoc-gen-gofast) ([#1724](https://github.com/pilosa/pilosa/pull/1724)) -- Move column attrs logic to executor ([#1677](https://github.com/pilosa/pilosa/pull/1677)) -- Shrink container bit count to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664)) - -### Performance - -- Remove bounds check ([#1619](https://github.com/pilosa/pilosa/pull/1619)) -- Improve benchmarking and performance ([#1741](https://github.com/pilosa/pilosa/pull/1741)) - -### Fixed - -- Ensure internal client closes all response bodies ([#1795](https://github.com/pilosa/pilosa/pull/1795)) -- Allow translate log entry buffer to grow ([#1787](https://github.com/pilosa/pilosa/pull/1787)) -- Add Gopkg.lock as a dependency for vendor target ([#1790](https://github.com/pilosa/pilosa/pull/1790)) -- Cluster resize fix ([#1785](https://github.com/pilosa/pilosa/pull/1785)) -- Attempt to fix deadlock by releasing view lock before broadcasting ([#1782](https://github.com/pilosa/pilosa/pull/1782)) -- Fix bug where cluster goes into RESIZING instead of NORMAL ([#1777](https://github.com/pilosa/pilosa/pull/1777)) -- Propogate updates to node details (not just additions and deletions) ([#1769](https://github.com/pilosa/pilosa/pull/1769)) -- Fix arm64 support ([#1764](https://github.com/pilosa/pilosa/pull/1764)) -- Fix data races ([#1750](https://github.com/pilosa/pilosa/pull/1750)) -- Fix fragment checksums race condition ([#1749](https://github.com/pilosa/pilosa/pull/1749)) -- Import cmd field type flag ([#1732](https://github.com/pilosa/pilosa/pull/1732)) -- Increase the translate file size for tests/benchmarks ([#1744](https://github.com/pilosa/pilosa/pull/1744)) -- Prevent panic in Bitmap.UnmarshalBinary when there is no data ([#1742](https://github.com/pilosa/pilosa/pull/1742)) -- Remove unused rule from peg grammar ([#1737](https://github.com/pilosa/pilosa/pull/1737)) -- Improve Internal Client errors ([#1729](https://github.com/pilosa/pilosa/pull/1729)) -- Forward imports to non-coordinator shards ([#1719](https://github.com/pilosa/pilosa/pull/1719)) -- Fix double escapes in PQL grammar ([#1727](https://github.com/pilosa/pilosa/pull/1727)) -- Ensure btree comparison doesn't fail for smallish N ([#1712](https://github.com/pilosa/pilosa/pull/1712)) -- Drop now-superfluous methodNotAllowedHandler ([#1711](https://github.com/pilosa/pilosa/pull/1711)) -- Use pilosa.Logger everywhere ([#1674](https://github.com/pilosa/pilosa/pull/1674)) -- Ensure view closes fragment on broadcast error ([#1675](https://github.com/pilosa/pilosa/pull/1675)) -- Prevent closing os.Stderr (used in verbose test logging) ([#1696](https://github.com/pilosa/pilosa/pull/1696)) -- Allow holder to close/open/close without panic on closing closed channel ([#1686](https://github.com/pilosa/pilosa/pull/1686)) -- Fix bug with Range() queries with field keys ([#1679](https://github.com/pilosa/pilosa/pull/1679)) -- Sync query validation for handlers ([#1676](https://github.com/pilosa/pilosa/pull/1676)) -- Wrap translation store errors, decrease test map size to prevent failure on 32-bit ([#1665](https://github.com/pilosa/pilosa/pull/1665)) -- Fix pass-by-value issue in proto decode ([#1662](https://github.com/pilosa/pilosa/pull/1662)) -- Do not run prerelease in CI if this is a pull request ([#1655](https://github.com/pilosa/pilosa/pull/1655)) -- Ensure mutex imports unset previous columns ([#1656](https://github.com/pilosa/pilosa/pull/1656)) -- Treat import timestamps as UTC ([#1651](https://github.com/pilosa/pilosa/pull/1651)) -- Remove unused log buffers from test cluster, fixes race ([#1612](https://github.com/pilosa/pilosa/pull/1612)) -- Add --field-keys and --index-keys options to pilosa import ([#1621](https://github.com/pilosa/pilosa/pull/1621)) -- Use passed stdin, stdout, and stderr in the cmd package ([#1620](https://github.com/pilosa/pilosa/pull/1620)) -- Update Go client sample to match latest master ([#1614](https://github.com/pilosa/pilosa/pull/1614)) - - -## [1.1.0] - 2018-08-21 - -This version contains 32 contributions from 5 contributors. There are 89 files changed; 2,752 insertions; and 1,013 deletions. - -### Added - -- Add CircleCI ([#1610](https://github.com/pilosa/pilosa/pull/1610)) -- Add key translation to exports ([#1608](https://github.com/pilosa/pilosa/pull/1608)) -- Support importing key values ([#1599](https://github.com/pilosa/pilosa/pull/1599), [#1601](https://github.com/pilosa/pilosa/pull/1601)) -- Treat coordinator as primary translate store ([#1582](https://github.com/pilosa/pilosa/pull/1582)) -- Add DEGRADED cluster state and handle gossip NodeLeave events correctly ([#1584](https://github.com/pilosa/pilosa/pull/1584)) -- Add linters to gometalinter and fix related issues ([#1544](https://github.com/pilosa/pilosa/pull/1544), [#1543](https://github.com/pilosa/pilosa/pull/1543), [#1540](https://github.com/pilosa/pilosa/pull/1540), [#1539](https://github.com/pilosa/pilosa/pull/1539), [#1537](https://github.com/pilosa/pilosa/pull/1537), [#1536](https://github.com/pilosa/pilosa/pull/1536), [#1535](https://github.com/pilosa/pilosa/pull/1535), [#1534](https://github.com/pilosa/pilosa/pull/1534), [#1530](https://github.com/pilosa/pilosa/pull/1530), [#1529](https://github.com/pilosa/pilosa/pull/1529), [#1528](https://github.com/pilosa/pilosa/pull/1528), [#1526](https://github.com/pilosa/pilosa/pull/1526), [#1527](https://github.com/pilosa/pilosa/pull/1527)) -- Add mutex field type ([#1524](https://github.com/pilosa/pilosa/pull/1524)) -- Fragment rows() and rowsForColumn() ([#1532](https://github.com/pilosa/pilosa/pull/1532)) - -### Fixed - -- Fix race on replicationClosing channel ([#1607](https://github.com/pilosa/pilosa/pull/1607)) -- Prevent anti-entropy and cluster resize from running simultaneously ([#1586](https://github.com/pilosa/pilosa/pull/1586)) -- Require a valid port that isn't greater than 65,535 ([#1603](https://github.com/pilosa/pilosa/pull/1603)) -- Add view parameter to sync logic for syncing time fields ([#1602](https://github.com/pilosa/pilosa/pull/1602)) -- Fix translator in cluster environment ([#1552](https://github.com/pilosa/pilosa/pull/1552)) -- Use string prefix instead of equality so json error message will pass on all Go versions ([#1558](https://github.com/pilosa/pilosa/pull/1558)) - -## [1.0.2] - 2018-08-01 - -This version contains 11 contributions from 3 contributors. There are 30 files changed; 1,569 insertions; and 1,215 deletions. - -### Fixed - -- Fix documentation ([#1503](https://github.com/pilosa/pilosa/pull/1503), [#1495](https://github.com/pilosa/pilosa/pull/1495), [#1551](https://github.com/pilosa/pilosa/pull/1551)) -- Fix places where empty IndexOptions were being used ([#1547](https://github.com/pilosa/pilosa/pull/1547)) -- Fix translator syncing bug in cluster environments ([#1552](https://github.com/pilosa/pilosa/pull/1552)) -- Fix race condition in translate_test ([#1541](https://github.com/pilosa/pilosa/pull/1541)) -- Add IndexOptions to IndexInfo json response ([#1542](https://github.com/pilosa/pilosa/pull/1542)) -- Add proper locking to cluster code to prevent races ([#1533](https://github.com/pilosa/pilosa/pull/1533)) -- Re-export erroneously unexported func Row.Intersect ([#1502](https://github.com/pilosa/pilosa/pull/1502)) -- Update parser to handle row keys on SetRowAttrs() ([#1555](https://github.com/pilosa/pilosa/pull/1555)) - -## [1.0.1] - 2018-07-11 - -This version contains 12 contributions from 4 contributors. There are 11 files changed; 133 insertions; and 39 deletions. - -### Fixed - -- Use `dep ensure -vendor-only` for build repeatability ([#1491](https://github.com/pilosa/pilosa/pull/1491)) -- Make sure time range views are calculated correctly across months ([#1485](https://github.com/pilosa/pilosa/pull/1485)) -- Fix up error handling, add a configurable timeout to http handler closing ([#1486](https://github.com/pilosa/pilosa/pull/1486)) -- Add gossip Closer ([#1483](https://github.com/pilosa/pilosa/pull/1483)) -- Update docs references to WebUI naming (console) and installation ([#1493](https://github.com/pilosa/pilosa/pull/1493)) - -## [1.0.0] - 2018-07-09 - -This version contains 218 contributions from 7 contributors. There are 184 files changed; 21,769 insertions; and 20,275 deletions. - -### Added - -- ID-Key Translation ([#1337](https://github.com/pilosa/pilosa/pull/1337)) -- Add CORS support to handler ([#1327](https://github.com/pilosa/pilosa/pull/1327)) - -### Changed - -- HTTP handler updates ([#1408](https://github.com/pilosa/pilosa/pull/1408), [#1399](https://github.com/pilosa/pilosa/pull/1399), [#1441](https://github.com/pilosa/pilosa/pull/1441), [#1375](https://github.com/pilosa/pilosa/pull/1375), [#1433](https://github.com/pilosa/pilosa/pull/1433), [#1444](https://github.com/pilosa/pilosa/pull/1444), [#1388](https://github.com/pilosa/pilosa/pull/1388), [#1309](https://github.com/pilosa/pilosa/pull/1309), [#1302](https://github.com/pilosa/pilosa/pull/1302), [#1304](https://github.com/pilosa/pilosa/pull/1304), [#1465](https://github.com/pilosa/pilosa/pull/1465), [#1466](https://github.com/pilosa/pilosa/pull/1466)) -- Refactor/improve tests ([#1437](https://github.com/pilosa/pilosa/pull/1437), [#1434](https://github.com/pilosa/pilosa/pull/1434), [#1435](https://github.com/pilosa/pilosa/pull/1435), [#1425](https://github.com/pilosa/pilosa/pull/1425), [#1418](https://github.com/pilosa/pilosa/pull/1418), [#1419](https://github.com/pilosa/pilosa/pull/1419), [#1413](https://github.com/pilosa/pilosa/pull/1413), [#1394](https://github.com/pilosa/pilosa/pull/1394), [#1387](https://github.com/pilosa/pilosa/pull/1387), [#1386](https://github.com/pilosa/pilosa/pull/1386), [#1378](https://github.com/pilosa/pilosa/pull/1378), [#1364](https://github.com/pilosa/pilosa/pull/1364), [#1348](https://github.com/pilosa/pilosa/pull/1348), [#1340](https://github.com/pilosa/pilosa/pull/1340), [#1297](https://github.com/pilosa/pilosa/pull/1297)) -- Simplify inter-node communication ([#1428](https://github.com/pilosa/pilosa/pull/1428), [#1427](https://github.com/pilosa/pilosa/pull/1427), [#1412](https://github.com/pilosa/pilosa/pull/1412), [#1398](https://github.com/pilosa/pilosa/pull/1398), [#1391](https://github.com/pilosa/pilosa/pull/1391), [#1389](https://github.com/pilosa/pilosa/pull/1389)) -- Make gossip's interface to Pilosa the API struct ([#1452](https://github.com/pilosa/pilosa/pull/1452)) -- Rename slice to shard ([#1426](https://github.com/pilosa/pilosa/pull/1426)) -- Clearbit for time fields ([#1424](https://github.com/pilosa/pilosa/pull/1424)) -- Update docs ([#1390](https://github.com/pilosa/pilosa/pull/1390), [#1329](https://github.com/pilosa/pilosa/pull/1329), [#1305](https://github.com/pilosa/pilosa/pull/1305), [#1296](https://github.com/pilosa/pilosa/pull/1296), [#1461](https://github.com/pilosa/pilosa/pull/1461)) -- Simplify server setup ([#1417](https://github.com/pilosa/pilosa/pull/1417), [#1393](https://github.com/pilosa/pilosa/pull/1393),[#1451](https://github.com/pilosa/pilosa/pull/1451)) -- Refactor API ([#1407](https://github.com/pilosa/pilosa/pull/1407)) -- Rewrite PQL parser and add various improvements/simplifications ([#1382](https://github.com/pilosa/pilosa/pull/1382), [#1402](https://github.com/pilosa/pilosa/pull/1402), [#1354](https://github.com/pilosa/pilosa/pull/1354), [#1463](https://github.com/pilosa/pilosa/pull/1463)) -- Rename "frame" to "field" ([#1395](https://github.com/pilosa/pilosa/pull/1395), [#1362](https://github.com/pilosa/pilosa/pull/1362), [#1360](https://github.com/pilosa/pilosa/pull/1360), [#1358](https://github.com/pilosa/pilosa/pull/1358), [#1357](https://github.com/pilosa/pilosa/pull/1357), [#1355](https://github.com/pilosa/pilosa/pull/1355)) -- Optimize count ([#1365](https://github.com/pilosa/pilosa/pull/1365)) -- Simplify bitmap max function ([#1333](https://github.com/pilosa/pilosa/pull/1333)) -- Rename "bit" to "column" for clarity ([#1326](https://github.com/pilosa/pilosa/pull/1326)) -- Rename pilosa.Bitmap to Row ([#1311](https://github.com/pilosa/pilosa/pull/1311)) -- Invert encoding/decoding and remove internal references ([#1454](https://github.com/pilosa/pilosa/pull/1454)) - -### Removed - -- Rename (unexport) many items to reduce public API footprint prior to 1.0 release ([#1470](https://github.com/pilosa/pilosa/pull/1470), [#1458](https://github.com/pilosa/pilosa/pull/1458), [#1450](https://github.com/pilosa/pilosa/pull/1450), [#1449](https://github.com/pilosa/pilosa/pull/1449), [#1448](https://github.com/pilosa/pilosa/pull/1448), [#1447](https://github.com/pilosa/pilosa/pull/1447), [#1446](https://github.com/pilosa/pilosa/pull/1446), [#1438](https://github.com/pilosa/pilosa/pull/1438), [#1443](https://github.com/pilosa/pilosa/pull/1443), [#1440](https://github.com/pilosa/pilosa/pull/1440), [#1439](https://github.com/pilosa/pilosa/pull/1439), [#1409](https://github.com/pilosa/pilosa/pull/1409), [#1392](https://github.com/pilosa/pilosa/pull/1392), [#1374](https://github.com/pilosa/pilosa/pull/1374), [#1372](https://github.com/pilosa/pilosa/pull/1372), [#1369](https://github.com/pilosa/pilosa/pull/1369), [#1367](https://github.com/pilosa/pilosa/pull/1367), [#1366](https://github.com/pilosa/pilosa/pull/1366), [#1351](https://github.com/pilosa/pilosa/pull/1351), [#1420](https://github.com/pilosa/pilosa/pull/1420), [#1416](https://github.com/pilosa/pilosa/pull/1416), [#1397](https://github.com/pilosa/pilosa/pull/1397)) -- Remove dead code ([#1432](https://github.com/pilosa/pilosa/pull/1432), [#1457](https://github.com/pilosa/pilosa/pull/1457), [#1421](https://github.com/pilosa/pilosa/pull/1421), [#1411](https://github.com/pilosa/pilosa/pull/1411), [#1377](https://github.com/pilosa/pilosa/pull/1377), [#1393](https://github.com/pilosa/pilosa/pull/1393), [#1462](https://github.com/pilosa/pilosa/pull/1462)) -- Remove view argument from Field.SetBit and Field.ClearBit ([#1396](https://github.com/pilosa/pilosa/pull/1396)) -- Remove WebUI (now contained in a separate package) ([#1363](https://github.com/pilosa/pilosa/pull/1363)) -- Remove bench command ([#1347](https://github.com/pilosa/pilosa/pull/1347)) -- Remove "view" from API, handler, docs ([#1346](https://github.com/pilosa/pilosa/pull/1346)) -- Remove backup/restore stuff ([#1339](https://github.com/pilosa/pilosa/pull/1339), [#1341](https://github.com/pilosa/pilosa/pull/1341)) -- Remove inverse frame functionality ([#1335](https://github.com/pilosa/pilosa/pull/1335)) -- Remove rangeEnabled option ([#1332](https://github.com/pilosa/pilosa/pull/1332)) -- Remove index and field MarshalJSON ([#1468](https://github.com/pilosa/pilosa/pull/1468)) - -### Fixed - -- Fix a few data races ([#1423](https://github.com/pilosa/pilosa/pull/1423)) -- Fix for crash while removing containers ([#1401](https://github.com/pilosa/pilosa/pull/1401)) -- Allow dashes in frame names ([#1415](https://github.com/pilosa/pilosa/pull/1415)) -- Fix generate-config command, use single toml lib ([#1350](https://github.com/pilosa/pilosa/pull/1350)) - -## [0.10.0] - 2018-05-15 - -This version contains 93 contributions from 8 contributors. There are 93 files changed; 4,495 insertions; and 5,392 deletions. - -### Added - -- Add B+Tree containers (Enterprise Edition) ([#1285](https://github.com/pilosa/pilosa/pull/1285)) -- Add /info endpoint ([#1236](https://github.com/pilosa/pilosa/pull/1236)) - -### Changed - -- Wrap errors ([#1271](https://github.com/pilosa/pilosa/pull/1271), [#1258](https://github.com/pilosa/pilosa/pull/1258), [#1274](https://github.com/pilosa/pilosa/pull/1274), [#1270](https://github.com/pilosa/pilosa/pull/1270), [#1273](https://github.com/pilosa/pilosa/pull/1273), [#1272](https://github.com/pilosa/pilosa/pull/1272), [#1260](https://github.com/pilosa/pilosa/pull/1260), [#1259](https://github.com/pilosa/pilosa/pull/1259), [#1256](https://github.com/pilosa/pilosa/pull/1256), [#1257](https://github.com/pilosa/pilosa/pull/1257), [#1261](https://github.com/pilosa/pilosa/pull/1261), [#1262](https://github.com/pilosa/pilosa/pull/1262), [#1263](https://github.com/pilosa/pilosa/pull/1263), [#1265](https://github.com/pilosa/pilosa/pull/1265)) - -### Removed - -- Remove unused code ([#1286](https://github.com/pilosa/pilosa/pull/1286)) -- Remove input definition, add install-stringer to Makefile ([#1284](https://github.com/pilosa/pilosa/pull/1284)) -- Remove /id and /hosts endpoints. Add local ID to /status ([#1238](https://github.com/pilosa/pilosa/pull/1238)) -- Remove API.URI ([#1255](https://github.com/pilosa/pilosa/pull/1255)) - -### Fixed - -- Assorted docs fixes ([#1281](https://github.com/pilosa/pilosa/pull/1281), [#1269](https://github.com/pilosa/pilosa/pull/1269)) -- Update PQL syntax in bench subcommand ([#1279](https://github.com/pilosa/pilosa/pull/1279)) -- Update help menu in WebUI ([#1278](https://github.com/pilosa/pilosa/pull/1278)) -- Fix dead lock ([#1268](https://github.com/pilosa/pilosa/pull/1268)) -- Make sure gossipMemberSet.Logger is set during server setup ([#1266](https://github.com/pilosa/pilosa/pull/1266)) -- Make sure ~ is expanded in NewServer; BroadcastReceiver uses temp path ([#1242](https://github.com/pilosa/pilosa/pull/1242)) -- Avoid creating a slice of nil timestamps on Import() ([#1234](https://github.com/pilosa/pilosa/pull/1234)) -- Fixup internal client ([#1253](https://github.com/pilosa/pilosa/pull/1253)) - -## [0.9.0] - 2018-05-04 - -This version contains 188 contributions from 12 contributors. There are 141 files changed; 17,832 insertions; and 7,503 deletions. - -*Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.* - -### Added - -- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908)) -- Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088)) -- Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133)) -- Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121)) -- Add validation around node-remove conditions ([#1138](https://github.com/pilosa/pilosa/pull/1138)) -- broadcast.SendSync field creation and deletion to all nodes ([#1132](https://github.com/pilosa/pilosa/pull/1132)) -- Spread recalculate caches to all nodes. Fixes #1069 ([#1109](https://github.com/pilosa/pilosa/pull/1109)) -- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064)) -- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063)) -- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048)) -- Add support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014)) -- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245)) -- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) -- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) -- Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191)) -- Log time/version to startup log ([#1246](https://github.com/pilosa/pilosa/pull/1246)) -- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972), [#1215](https://github.com/pilosa/pilosa/pull/1215), [#1213](https://github.com/pilosa/pilosa/pull/1213), [#1224](https://github.com/pilosa/pilosa/pull/1224), [#1250](https://github.com/pilosa/pilosa/pull/1250)) - -### Changed - -- Put Statik behind an interface ([#1163](https://github.com/pilosa/pilosa/pull/1163)) -- Refactor diagnostics, inject gopsutil dependency ([#1166](https://github.com/pilosa/pilosa/pull/1166)) -- Use boolean instead of address to configure coordinator ([#1158](https://github.com/pilosa/pilosa/pull/1158)) -- Put GCNotify behind an interface ([#1148](https://github.com/pilosa/pilosa/pull/1148)) -- Replace custom assembly bit functions with standard go ([#797](https://github.com/pilosa/pilosa/pull/797)) -- Improve roaring tests ([#1115](https://github.com/pilosa/pilosa/pull/1115)) -- Change configuration cluster.type (string) to cluster.disabled (bool) ([#1099](https://github.com/pilosa/pilosa/pull/1099)) -- Use NodeID instead of URI for node identification ([#1077](https://github.com/pilosa/pilosa/pull/1077)) -- Change gossip config from DefaultLocalConfig to DefaultWANConfig ([#1032](https://github.com/pilosa/pilosa/pull/1032)) -- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027)) -- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001)) -- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950)) -- Refactor HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) -- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) -- Refactor pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) -- Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223)) -- Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216)) -- Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221)) - -### Removed - -- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171)) -- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142)) -- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130)) -- Remove holder.Peek, combine with HasData, move server logic ([#1226](https://github.com/pilosa/pilosa/pull/1226)) -- Remove PATCH frame endpoint ([#1222](https://github.com/pilosa/pilosa/pull/1222)) -- Remove Index.MergeSchemas() method ([#1219](https://github.com/pilosa/pilosa/pull/1219)) -- Remove references to Input Definition from the docs ([#1212](https://github.com/pilosa/pilosa/pull/1212)) -- Remove Index.TimeQuantum ([#1209](https://github.com/pilosa/pilosa/pull/1209)) -- Remove SecurityManager. Implement api restrictions in api package. ([#1207](https://github.com/pilosa/pilosa/pull/1207)) - -### Fixed - -- Handle the scheme correctly in config.Bind ([#1143](https://github.com/pilosa/pilosa/pull/1143)) -- Prevent excessive sendSync (createView) messages. ([#1139](https://github.com/pilosa/pilosa/pull/1139)) -- Fix a shift logic bug in bitmapZeroRange ([#1110](https://github.com/pilosa/pilosa/pull/1110)) -- Fix node id validation on set-coordinator ([#1102](https://github.com/pilosa/pilosa/pull/1102)) -- Avoid overflow bug in differenceRunArray ([#1105](https://github.com/pilosa/pilosa/pull/1105)) -- Fix bug in NewServerCluster where each host was its own coordinator ([#1101](https://github.com/pilosa/pilosa/pull/1101)) -- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084)) -- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979)) -- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980)) -- Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233)) -- Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229)) -- Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228)) - -### Deprecated - -- Deprecate RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) - -### Performance - -- Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017)) - -## [0.8.8] - 2018-02-19 - -This version contains 1 contribution from 2 contributors. There are 4 files changed; 1,153 insertions; and 618 deletions. - -### Fixed - -- Bug fixes and improved test coverage in roaring ([#1118](https://github.com/pilosa/pilosa/pull/1118)) - -## [0.8.7] - 2018-02-12 - -This version contains 1 contribution from 1 contributors. There are 2 files changed; 84 insertions; and 4 deletions. - -### Fixed - -- Fix a shift logic bug in bitmapZeroRange ([#1111](https://github.com/pilosa/pilosa/pull/1111)) - -## [0.8.6] - 2018-02-09 - -This version contains 2 contributions from 2 contributors. There are 3 files changed; 171 insertions; and 6 deletions. - -### Fixed - -- Fix overflow bug in differenceRunArray [#1106](https://github.com/pilosa/pilosa/pull/1106) -- Fix bug where count and bitmap queries could return different numbers [#1083](https://github.com/pilosa/pilosa/pull/1083) - -## [0.8.5] - 2018-01-18 - -This version contains 1 contribution from 1 contributor. There is 1 file changed; 1 insertion, and 0 deletions. - -### Fixed - -- Bind Docker container on all interfaces ([#1061](https://github.com/pilosa/pilosa/pull/1061)) - -## [0.8.4] - 2018-01-10 - -This version contains 4 contributions from 3 contributors. There are 17 files changed; 974 insertions; and 221 deletions. - -### Fixed - -- Group the write operations in syncBlock by MaxWritesPerRequest ([#1038](https://github.com/pilosa/pilosa/pull/1038)) -- Change gossip config from memberlist.DefaultLocalConfig to memberlist.DefaultWANConfig ([#1033](https://github.com/pilosa/pilosa/pull/1033)) - -### Performance - -- Change AttrBlock handler calls to support protobuf instead of json ([#1046](https://github.com/pilosa/pilosa/pull/1046)) -- Use RLock instead of Lock in a few places ([#1042](https://github.com/pilosa/pilosa/pull/1042)) - -## [0.8.3] - 2017-12-12 - -This version contains 1 contribution from 1 contributor. There are 2 files changed; 59 insertions; and 42 deletions. - -### Fixed - -- Protect against accessing pointers to memory which was unmapped ([#1000](https://github.com/pilosa/pilosa/pull/1000)) - -## [0.8.2] - 2017-12-05 - -This version contains 1 contribution from 1 contributor. There are 15 files changed; 127 insertions; and 98 deletions. - -### Fixed - -- Modify initialization of HTTP client so only one instance is created ([#994](https://github.com/pilosa/pilosa/pull/994)) - -## [0.8.1] - 2017-11-15 - -This version contains 2 contributions from 2 contributors. There are 4 files changed; 27 insertions; and 14 deletions. - -### Fixed - -- Fix CountOpenFiles() fatal crash ([#969](https://github.com/pilosa/pilosa/pull/969)) -- Fix version check when local is greater than pilosa.com ([#968](https://github.com/pilosa/pilosa/pull/968)) - -## [0.8.0] - 2017-11-15 - -This version contains 31 contributions from 8 contributors. There are 84 files changed; 3,732 insertions; and 1,428 deletions. - -### Added - -- Diagnostics ([#895](https://github.com/pilosa/pilosa/pull/895)) -- Add docker-build make target for repeatable Docker-based builds ([#933](https://github.com/pilosa/pilosa/pull/933)) -- Add documentation on importing field values; fixes #924 ([#938](https://github.com/pilosa/pilosa/pull/938)) -- Add flag documentation and tests, remove "plugins.path" ([#942](https://github.com/pilosa/pilosa/pull/942)) -- Add TLS support ([#867](https://github.com/pilosa/pilosa/pull/867)) -- Add TLS cluster how to ([#898](https://github.com/pilosa/pilosa/pull/898)) -- Add support for gossip encryption ([#889](https://github.com/pilosa/pilosa/pull/889)) -- Add Recalculate Caches endpoint ([#881](https://github.com/pilosa/pilosa/pull/881)) -- Add search-friendly documentation for BSI range query syntax ([#955](https://github.com/pilosa/pilosa/pull/955)) - -### Changed - -- Remove unneeded Gopkg.toml constraints and update all dependencies ([#943](https://github.com/pilosa/pilosa/pull/943)) -- Remove row and column labels in webUI ([#884](https://github.com/pilosa/pilosa/pull/884)) -- Internal Client refactoring ([#892](https://github.com/pilosa/pilosa/pull/892)) -- Remove column/row labels for input definition ([#945](https://github.com/pilosa/pilosa/pull/945)) -- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878)) - -### Fixed - -- Skip permissions test when run as root. Fixes #940 ([#941](https://github.com/pilosa/pilosa/pull/941)) -- Address "connection reset" issues in client ([#934](https://github.com/pilosa/pilosa/pull/934)) -- Fix field value import: Use signed int and respect field minimum ([#919](https://github.com/pilosa/pilosa/pull/919)) -- Constrain BoltDB to version rather than specific revision ([#887](https://github.com/pilosa/pilosa/pull/887)) -- Fix bug in environment variable format ([#882](https://github.com/pilosa/pilosa/pull/882)) -- Fix overflow in differenceRunBitmap ([#949](https://github.com/pilosa/pilosa/pull/949)) - -### Performance - -- Use FieldNotNull to improve efficiency of BETWEEN queries ([#874](https://github.com/pilosa/pilosa/pull/874)) - -## [0.7.2] - 2017-11-15 - -This version contains 1 contribution from 1 contributor. There is 1 file changed; 16 insertions; and 1 deletion. - -### Changed - -- Bump HTTP client's MaxIdleConns and MaxIdleConnsPerHost ([#920](https://github.com/pilosa/pilosa/pull/920)) - -## [0.7.1] - 2017-10-09 - -This version contains 3 contributions from 3 contributors. There are 14 files changed; 221 insertions; and 52 deletions. - -### Changed - -- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878)) - -### Performance - -- Leverage not-null field to make BETWEEN queries more efficient ([#874](https://github.com/pilosa/pilosa/pull/874)) - -## [0.7.0] - 2017-10-03 - -This version contains 59 contributions from 9 contributors. There are 61 files changed; 5207 insertions; and 1054 deletions. - -### Added - -- Add HTTP API for fields ([#811](https://github.com/pilosa/pilosa/pull/811), [#856](https://github.com/pilosa/pilosa/pull/856)) -- Add HTTP API for delete views ([#785](https://github.com/pilosa/pilosa/pull/785)) -- Modify import endpoint to handle BSI field values ([#840](https://github.com/pilosa/pilosa/pull/840)) -- Add field Range() support to Executor ([#791](https://github.com/pilosa/pilosa/pull/791)) -- Support PQL Range() queries for fields ([#755](https://github.com/pilosa/pilosa/pull/755)) -- Add Sum() field query ([#778](https://github.com/pilosa/pilosa/pull/778)) -- Add documentation for BSI ([#861](https://github.com/pilosa/pilosa/pull/861)) -- Add BETWEEN for Range queries ([#847](https://github.com/pilosa/pilosa/pull/847)) -- Add Xor support for PQL ([#789](https://github.com/pilosa/pilosa/pull/789)) -- Enable auto-creating the schema on imports ([#837](https://github.com/pilosa/pilosa/pull/837)) -- Update client library docs ([#831](https://github.com/pilosa/pilosa/pull/831)) -- Handle SIGTERM signal ([#830](https://github.com/pilosa/pilosa/pull/830)) -- Add cluster config example to docs ([#806](https://github.com/pilosa/pilosa/pull/806)) -- Add ability to exclude attributes and bits in Bitmap queries ([#783](https://github.com/pilosa/pilosa/pull/783)) - -### Fixed - -- Fix panic when iterating over an empty run container ([#860](https://github.com/pilosa/pilosa/pull/860)) -- Fix row id zero bug ([#814](https://github.com/pilosa/pilosa/pull/814)) -- Fix cache invalidation bug ([#795](https://github.com/pilosa/pilosa/pull/795)) -- Set container.n in differenceRunRun ([#794](https://github.com/pilosa/pilosa/pull/794)) -- Fix infinite loop in bitmap-to-array conversion ([#779](https://github.com/pilosa/pilosa/pull/779)) -- Fix CountRange bug ([#773](https://github.com/pilosa/pilosa/pull/773)) - -### Deprecated - -- Remove support for row/column labels ([#839](https://github.com/pilosa/pilosa/pull/839)) - -### Performance - -- Refactor differenceRunArray ([#859](https://github.com/pilosa/pilosa/pull/859)) -- Update fragment.FieldSum to use roaring IntersectionCount() ([#841](https://github.com/pilosa/pilosa/pull/841)) -- Add roaring optimizations ([#842](https://github.com/pilosa/pilosa/pull/842)) -- Convert lock to read lock ([#848](https://github.com/pilosa/pilosa/pull/848)) -- Reduce Lock calls in executor ([#846](https://github.com/pilosa/pilosa/pull/846)) -- Implement container.flipBitmap() to improve differenceRunBitmap() ([#849](https://github.com/pilosa/pilosa/pull/849)) -- Reuse container storage on UnmarshalBinary to improve memory utilization ([#820](https://github.com/pilosa/pilosa/pull/820)) -- Improve WriteTo performance ([#812](https://github.com/pilosa/pilosa/pull/812)) - -## [0.6.0] - 2017-08-11 - -This version contains 14 contributions from 5 contributors. There are 28 files changed; 4,936 insertions; and 692 deletions. - -### Added - -- Add Run-length Encoding ([#758](https://github.com/pilosa/pilosa/pull/758)) - -### Changed - -- Make gossip the default broadcast type ([#750](https://github.com/pilosa/pilosa/pull/750)) - -### Fixed - -- Fix CountRange ([#759](https://github.com/pilosa/pilosa/pull/759)) -- Fix `differenceArrayRun` logic ([#674](https://github.com/pilosa/pilosa/pull/674)) - -## [0.5.0] - 2017-08-02 - -This version contains 65 contributions from 8 contributors (including 1 volunteer contributor). There are 79 files changed; 7,972 insertions; and 2,800 deletions. - -### Added - -- Set open file limit during Pilosa startup ([#748](https://github.com/pilosa/pilosa/pull/748)) -- Add Input Definition ([#646](https://github.com/pilosa/pilosa/pull/646)) -- Add cache type: None ([#745](https://github.com/pilosa/pilosa/pull/745)) -- Add panic recovery in top level HTTP handler ([#741](https://github.com/pilosa/pilosa/pull/741)) -- Count open file handles as a StatsD metric ([#636](https://github.com/pilosa/pilosa/pull/636)) -- Add coverage tools to Makefile ([#635](https://github.com/pilosa/pilosa/pull/635)) -- Add Holder test coverage ([#629](https://github.com/pilosa/pilosa/pull/629)) -- Add runtime memory metrics ([#600](https://github.com/pilosa/pilosa/pull/600)) -- Add sorting flag to import command ([#606](https://github.com/pilosa/pilosa/pull/606)) -- Add PQL support for field values (WIP) ([#721](https://github.com/pilosa/pilosa/pull/721)) -- Set and retrieve field values (WIP) ([#702](https://github.com/pilosa/pilosa/pull/702)) -- Add BSI range-encoding schema support (WIP) ([#670](https://github.com/pilosa/pilosa/pull/670)) - -### Changed - -- Move InternalPort config option to top-level ([#747](https://github.com/pilosa/pilosa/pull/747)) -- Switch from glide to dep for dependency management ([#744](https://github.com/pilosa/pilosa/pull/744)) -- Remove QueryRequest.Quantum since it is no longer used ([#699](https://github.com/pilosa/pilosa/pull/699)) -- Refactor test utilities into importable package ([#675](https://github.com/pilosa/pilosa/pull/675)) - -### Fixed - -- Add mutex for attribute cache ([#729](https://github.com/pilosa/pilosa/pull/729)) -- Use log-path flag to specify log file ([#678](https://github.com/pilosa/pilosa/pull/678)) - -## [0.4.0] - 2017-06-08 - -This version contains 53 contributions from 13 contributors (including 4 volunteer contributors). There are 96 files changed; 6373 insertions; and 770 deletions. - -*Note that data files created in Pilosa < 0.4.0 are not compatible with Pilosa 0.4.0 as a result of [#520](https://github.com/pilosa/pilosa/pull/520).* - -### Added -- Support metric reporting through StatsD protocol ([#468](https://github.com/pilosa/pilosa/pull/468), [#568](https://github.com/pilosa/pilosa/pull/568), [#580](https://github.com/pilosa/pilosa/pull/580)) -- Improve test coverage for ctl package ([#586](https://github.com/pilosa/pilosa/pull/586)) -- Add support for bit flip (negate) in roaring ([#592](https://github.com/pilosa/pilosa/pull/592)) -- Add xor support to roaring ([#571](https://github.com/pilosa/pilosa/pull/571)) -- Improve WebUI autocomplete ([#560](https://github.com/pilosa/pilosa/pull/560)) -- Add syntax hints tooltip to WebUI ([#537](https://github.com/pilosa/pilosa/pull/537)) -- Implement 'config' CLI command ([#541](https://github.com/pilosa/pilosa/pull/541)) -- Move docs into repo ([#563](https://github.com/pilosa/pilosa/pull/563)) -- Add inverse TopN() support ([#551](https://github.com/pilosa/pilosa/pull/551)) -- Add various Makefile updates ([#540](https://github.com/pilosa/pilosa/pull/540)) -- Provide details on Glide checksum mismatch ([#546](https://github.com/pilosa/pilosa/pull/546)) -- Add Docker multi-stage build ([#535](https://github.com/pilosa/pilosa/pull/535)) -- Support inverse Range() queries ([#533](https://github.com/pilosa/pilosa/pull/533)) -- Support colon commands in WebUI ([#529](https://github.com/pilosa/pilosa/pull/529), [#510](https://github.com/pilosa/pilosa/pull/510)) - -### Changed -- Increase default partition count from 16 to 256 (BREAKING CHANGE) ([#520](https://github.com/pilosa/pilosa/pull/520)) -- Validate unknown query params ([#578](https://github.com/pilosa/pilosa/pull/578)) -- Validate configuration file ([#573](https://github.com/pilosa/pilosa/pull/573)) -- Change default cache type to ranked ([#524](https://github.com/pilosa/pilosa/pull/524)) -- Add max-writes-per-requests limit ([#525](https://github.com/pilosa/pilosa/pull/525)) - -### Fixed -- Add "make test" to PHONY section of Makefile ([#605](https://github.com/pilosa/pilosa/pull/605)) -- Fix failing tests when IPv6 is disabled ([#594](https://github.com/pilosa/pilosa/pull/594)) -- Add minor docs fix, indent in JSON ([#599](https://github.com/pilosa/pilosa/pull/599)) -- Fix BroadcastHandler handle missing index error ([#597](https://github.com/pilosa/pilosa/pull/597)) -- Add WebUI fixes ([#589](https://github.com/pilosa/pilosa/pull/589)) -- Fix support for 32-bit Linux ([#549](https://github.com/pilosa/pilosa/pull/549), [#565](https://github.com/pilosa/pilosa/pull/565)) -- Fix 3 separate bugs in bitmapCountRange ([#559](https://github.com/pilosa/pilosa/pull/559)) -- Add client support for MaxInverseSliceByIndex ([#555](https://github.com/pilosa/pilosa/pull/555)) -- Fix bug in `handleGetSliceMax` ([#554](https://github.com/pilosa/pilosa/pull/554)) -- Default to `standard` view in export command ([#548](https://github.com/pilosa/pilosa/pull/548)) -- Fix vet issues with the assembly code in Roaring ([#528](https://github.com/pilosa/pilosa/pull/528)) -- Prevent row labels that match the column label ([#503](https://github.com/pilosa/pilosa/pull/503)) -- Fix roaring test: TestBitmap_Quick_Array1 ([#507](https://github.com/pilosa/pilosa/pull/507)) -- Don't try to create inverse views on Import() when inverseEnabled is false ([#462](https://github.com/pilosa/pilosa/pull/462)) - -### Performance -- Set n based on array length instead of incrementing repeatedly ([#590](https://github.com/pilosa/pilosa/pull/590)) -- Rewrite intersectCountArrayBitmap for perf test ([#577](https://github.com/pilosa/pilosa/pull/577)) -- Check for duplicate attributes under read lock on insert ([#562](https://github.com/pilosa/pilosa/pull/562)) - -[Unreleased]: https://github.com/pilosa/pilosa/compare/v1.2...HEAD -[0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4 -[0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5 -[0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6 -[0.7.0]: https://github.com/pilosa/pilosa/compare/v0.6...v0.7 -[0.8.0]: https://github.com/pilosa/pilosa/compare/v0.7...v0.8 -[0.9.0]: https://github.com/pilosa/pilosa/compare/v0.8...v0.9 -[0.10.0]: https://github.com/pilosa/pilosa/compare/v0.9...v0.10 -[1.0.0]: https://github.com/pilosa/pilosa/compare/v0.10...v1.0 -[1.1.0]: https://github.com/pilosa/pilosa/compare/v1.0...v1.1 -[1.2.0]: https://github.com/pilosa/pilosa/compare/v1.1...v1.2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 8e4b45e4e..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,177 +0,0 @@ -# Contributing to Pilosa - -The workflow components of these instructions apply to all Pilosa repositories. - -## Reporting a bug - -If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1]. - -## Submitting a feature request - -Feature requests are managed in Github issues, organized with [Zenhub](https://www.zenhub.com/), which is publicly available as a browser extension. New features typically go through a [Proposal Process][4] -which starts by [opening a new issue][1] that describes the new feature proposal. - -## Making code contributions - -Before you start working on new features, you should [open a new issue][1] to let others know what -you're doing, otherwise you run the risk of duplicating effort. This also -gives others an opportunity to provide input for your feature. - -If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6]. - - -### Development Environment - -- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [CircleCI config file](../master/.circleci/config.yml) for the most up-to-date information. - -- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH). - -- Fork the [Pilosa repository][2] to your own account. - -- It will be easier to follow these instructions if you: - - ```sh - export GH_USERNAME= - ``` - -- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone Pilosa: - - ```sh - mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ - git clone https://github.com/pilosa/pilosa.git - ``` - -- `cd` to your pilosa directory: - - ```sh - cd ${GOPATH}/src/github.com/pilosa/pilosa - ``` - -- Install Pilosa command line tools: - - ```sh - make install - ``` - - Running `pilosa` should now run a Pilosa instance. - -- The official Pilosa repository is your "origin" remote in git. Add your fork as your github username - - ```sh - cd ${GOPATH}/src/github.com/pilosa/pilosa - git remote add ${GH_USERNAME} git@github.com:${GH_USERNAME}/pilosa.git - ``` - -### Makefile - -Pilosa includes a Makefile that automates several tasks: - -- Install Pilosa: - - ```sh - make install - ``` - -- Install build dependencies: - - ```sh - make install-build-deps - ``` - -- Create the vendor directory: - - ```sh - make vendor - ``` - -- Run the test suite: - - ```sh - make test - ``` - -- View the coverage report: - - ```sh - make cover-viz - ``` - -- Clear the `vendor/` and `build/` directories: - - ```sh - make clean - ``` - -- Create release tarballs: - - ```sh - make release - ``` - -- Regenerate protocol buffer files in `internal/`: - - ```sh - make generate-protoc - ``` - -- Create tagged Docker image: - - ```sh - make docker - ``` - -- Run tests inside Docker container: - - ```sh - make docker-test - ``` - -Additional commands are available in the `Makefile`. - -### Submitting code changes - -- Before starting to work on a task, sync your branch with the upstream: - - ```sh - git checkout master - git pull - ``` - -- Create a local feature branch: - - ```sh - git checkout -b something-amazing - ``` - -- Commit your changes locally using `git add` and `git commit`. Please use [appropriate commit messages](https://chris.beams.io/posts/git-commit/). - -- Make sure that you've written tests for your new feature, and then run the tests: - - ```sh - make test - ``` - -- Verify that your pull request is applied to the latest version of code on github: - - ```sh - git checkout master - git pull - git checkout something-amazing - git rebase master - ``` - -- Push to your fork: - - ```sh - git push -u $GH_USERNAME something-amazing:something-amazing - ``` - -- Submit a [pull request][3] - - -[1]: https://github.com/pilosa/pilosa/issues/new -[2]: https://github.com/pilosa/pilosa -[3]: https://github.com/pilosa/pilosa/compare/ -[4]: https://github.com/pilosa/general/blob/master/proposal.md -[5]: https://github.com/pilosa/pilosa/issues -[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer diff --git a/LICENSE b/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/NOTES b/NOTES deleted file mode 100644 index 8da55fe11..000000000 --- a/NOTES +++ /dev/null @@ -1,26 +0,0 @@ - - Index Column - ┌───────────▼────────────────────────────┐ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - Row──▶0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │────────────────────────────────────────┤ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │────────────────────────────────────────┤ - F ▶│0000000000000000000000000000000000000000│ - i ││0000000000000000000000000000000000000000│ - e ││0000000000000000000000000000000000000000│ - l ││0000000000000000000000000000000000000000│ - d ▶│0000000000000000000000000000000000000000│ - └────────────────────────────────────────┘ - ▲───────────▲ - Shard - - -Fragment=intersection of field & shard From c72406819ee3eb46f056cf12c70429c42e2b675a Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 09:40:39 -0600 Subject: [PATCH 05/12] added seed param --- cmd/random-query/main.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 22f5209ad..7b01575ee 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -56,6 +56,7 @@ type RandomQueryConfig struct { TimeRange int64 // hours between parsed times Index string QPM int + Seed int SrcFile string Duration time.Duration Target vegeta.Target @@ -109,9 +110,10 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.") fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate") fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") + fs.IntVar(&cfg.Seed, "seed", 0, "RNG seed") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") - fs.IntVar(&cfg.QPM, "qps", 10, "number of currernt requests per minute to simulate, default 10") + fs.IntVar(&cfg.QPM, "qpm", 10, "number of currernt requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") @@ -179,7 +181,7 @@ func (cfg *RandomQueryConfig) Run() (err error) { if err != nil { return err } - rate := vegeta.Rate{Freq: cfg.QPM, Per: time.Second} + rate := vegeta.Rate{Freq: cfg.QPM, Per: time.Minute} duration := cfg.Duration targeter := vegeta.NewStaticTargeter(cfg.Target) attacker := vegeta.NewAttacker() @@ -348,8 +350,7 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { if foundIntField { cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct") } - seed := int64(42) - cfg.Rnd = rand.New(rand.NewSource(seed)) + cfg.Rnd = rand.New(rand.NewSource(int64(cfg.Seed))) return cfg.buildPayload() } From 10894352181a27addae7c0c87d2adb714581cbcc Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 09:48:45 -0600 Subject: [PATCH 06/12] add check for index presence --- cmd/random-query/main.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 7b01575ee..5fbe72290 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -113,7 +113,7 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.Seed, "seed", 0, "RNG seed") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") - fs.IntVar(&cfg.QPM, "qpm", 10, "number of currernt requests per minute to simulate, default 10") + fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") @@ -317,9 +317,10 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { return err } foundIntField := false + any := false for i, ii := range cfg.Info { if ii.Name == cfg.Index { - + any = true _ = i for k, fld := range ii.Fields { _ = k @@ -346,6 +347,9 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { } } } + if !any { + return errors.New(fmt.Sprintf("index %v not found", cfg.Index)) + } cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"} if foundIntField { cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct") From f7b4f621a15f3df150b3665b99787bddc20587b0 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 19 Nov 2021 10:38:06 -0600 Subject: [PATCH 07/12] remove references to LICENSE and checks for it in source files --- .circleci/config.yml | 11 --- .cloudbuild/Dockerfile | 1 - .cloudbuild/LICENSE | 202 ---------------------------------------- .cloudbuild/NOTICE | 13 +-- .gitlab/Dockerfile | 1 - Dockerfile | 1 - Dockerfile-clustertests | 1 - Dockerfile.pilosa | 1 - Makefile | 10 -- roaring/btree_test.go | 2 +- 10 files changed, 2 insertions(+), 241 deletions(-) delete mode 100644 .cloudbuild/LICENSE diff --git a/.circleci/config.yml b/.circleci/config.yml index 65534faa6..498c56ea3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,13 +63,6 @@ jobs: - checkout-plus - run: go mod download - save-mod-cache - check-license-headers: - executor: - name: golang - steps: - - checkout-plus - - skip-if-root-unchanged - - run: make check-license-headers linter: executor: name: golang @@ -239,10 +232,6 @@ workflows: context: molecula requires: - setup - - check-license-headers: - context: molecula - requires: - - setup - go-mod-tidy: context: molecula requires: diff --git a/.cloudbuild/Dockerfile b/.cloudbuild/Dockerfile index 407e5b994..fac3bf2fd 100644 --- a/.cloudbuild/Dockerfile +++ b/.cloudbuild/Dockerfile @@ -5,7 +5,6 @@ LABEL org.opencontainers.image.authors="dev@molecula.com" RUN apk add --no-cache curl jq -COPY LICENSE /LICENSE COPY NOTICE /NOTICE ARG SHORT_SHA diff --git a/.cloudbuild/LICENSE b/.cloudbuild/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/.cloudbuild/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/.cloudbuild/NOTICE b/.cloudbuild/NOTICE index 594272a27..5d88ea9c0 100644 --- a/.cloudbuild/NOTICE +++ b/.cloudbuild/NOTICE @@ -1,18 +1,7 @@ Software license ================ -Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright (C) 2017-2021 Molecula Corp. All rights reserved. Third-party software licenses ============================= diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index b22392ce1..b041f58ce 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -7,7 +7,6 @@ WORKDIR /featurebase RUN apk add --no-cache curl jq -COPY LICENSE . COPY NOTICE . COPY featurebase_linux_amd64 . diff --git a/Dockerfile b/Dockerfile index 115855034..25ea9c3f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,6 @@ RUN apk add --no-cache curl jq COPY --from=pilosa-builder /pilosa/build/featurebase / -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests index 8c60f11bf..4a3a0c196 100644 --- a/Dockerfile-clustertests +++ b/Dockerfile-clustertests @@ -20,7 +20,6 @@ RUN apt install -y docker.io RUN cp /go/bin/featurebase /featurebase -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 diff --git a/Dockerfile.pilosa b/Dockerfile.pilosa index a24c70ec7..6ee2e0bbf 100644 --- a/Dockerfile.pilosa +++ b/Dockerfile.pilosa @@ -24,7 +24,6 @@ RUN apk add --no-cache curl jq COPY --from=pilosa-builder /pilosa/build/featurebase / -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 diff --git a/Makefile b/Makefile index 55d1332ba..f72517684 100644 --- a/Makefile +++ b/Makefile @@ -18,10 +18,6 @@ GO_VERSION=1.16.10 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release BUILD_TAGS += shardwidth$(SHARD_WIDTH) TEST_TAGS = roaringparanoia -define LICENSE_HASH_CODE - head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/' -e 's/Pilosa Corp\./Molecula Corp./' | shasum | cut -f 1 -d " " -endef -LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go)) UNAME := $(shell uname -s) ifeq ($(UNAME), Darwin) IS_MACOS:=1 @@ -313,12 +309,6 @@ gometalinter: require-gometalinter vendor --exclude "^pql/pql.peg.go" \ ./... -# Verify that all Go files have license header -check-license-headers: SHELL:=/bin/bash -check-license-headers: - @! find . -path ./vendor -prune -o -name '*.go' -print | grep -v -F -f license.exceptions | while read fn;\ - do [[ `$(call LICENSE_HASH_CODE, $$fn)` == $(LICENSE_HASH) ]] || echo $$fn; done | grep '.' - ###################### # Build dependencies # ###################### diff --git a/roaring/btree_test.go b/roaring/btree_test.go index 7a952a937..ee51aeb0e 100644 --- a/roaring/btree_test.go +++ b/roaring/btree_test.go @@ -1,6 +1,6 @@ // Copyright 2014 The b Authors. All rights reserved. // Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. +// license that can be found in the NOTICE file. package roaring From b12f00db590dfe05549483219a0d087016d7ff43 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 10:57:36 -0600 Subject: [PATCH 08/12] changed default seed to current time --- cmd/random-query/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 5fbe72290..475ba9e60 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -110,7 +110,7 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.") fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate") fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") - fs.IntVar(&cfg.Seed, "seed", 0, "RNG seed") + fs.IntVar(&cfg.Seed, "seed", int(time.Now().Unix()), "RNG seed, defaults to currentime") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") @@ -440,6 +440,7 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci f = &Features{} cfg.IndexMap[index] = f } + vprint.VV("Add: %s %v %v %v", field, min, max, scale) /* if min.Scale != scale || max.Scale != scale { PanicOn(fmt.Sprintf("scale error; %v:%v min scale %d, max scale %d, field scale %d, assumed they'd be equal", From 25e5c6f5fbc1b9d4729cf5dcd6c98812e2366c65 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 19 Nov 2021 11:22:15 -0600 Subject: [PATCH 09/12] remove references to CONTRIBUTING and CHANGELOG --- .github/PULL_REQUEST_TEMPLATE.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 2c1f4979d..f579dc42f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,13 +6,10 @@ Fixes # ## Pull request checklist -- [ ] I have read the [contributing guide](https://github.com/molecula/featurebase/blob/master/CONTRIBUTING.md). -- [ ] I have agreed to the [Contributor License Agreement](https://cla-assistant.io/pilosa/pilosa). - [ ] I have updated the [documentation](https://github.com/molecula/docs). - [ ] I have resolved any merge conflicts. - [ ] I have included tests that cover my changes. - [ ] All new and existing tests pass. -- [ ] Make sure PR title conforms to convention in CHANGELOG.md. - [ ] Add appropriate changelog label to PR (if applicable). ## Code review checklist @@ -24,5 +21,4 @@ This is the checklist that the reviewer will follow while reviewing your pull re - [ ] Check that tests have been written and that they cover the new functionality. - [ ] Run tests and ensure they pass. - [ ] Build and run the code, performing any applicable integration testing. -- [ ] Make sure PR title conforms to convention in CHANGELOG.md. - [ ] Make sure PR is tagged with appropriate changelog label. From 989f9c3f48c480f58be9251c884131c5bdcae8c9 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 11:29:17 -0600 Subject: [PATCH 10/12] applied suggestions --- cmd/random-query/main.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 475ba9e60..0fcd71ce9 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -42,18 +42,16 @@ import ( // RandomQueryConfig type RandomQueryConfig struct { - - // user facing flags - HostPort string // -hostport - TreeDepth int // -d - QueryCount int // -n - Verbose bool // -v + HostPort string + TreeDepth int + QueryCount int + Verbose bool NumRuns int - TimeFromArg string // --time.from - TimeToArg string // --time.to - TimeFrom time.Time // parsed time - TimeTo time.Time // parsed time - TimeRange int64 // hours between parsed times + TimeFromArg string + TimeToArg string + TimeFrom time.Time + TimeTo time.Time + TimeRange int64 Index string QPM int Seed int @@ -110,10 +108,10 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.") fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate") fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") - fs.IntVar(&cfg.Seed, "seed", int(time.Now().Unix()), "RNG seed, defaults to currentime") + fs.IntVar(&cfg.Seed, "seed", int(time.Now().Unix()), "RNG seed, defaults to current time") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") - fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") + fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") @@ -142,6 +140,9 @@ func (c *RandomQueryConfig) ValidateConfig() error { return fmt.Errorf("time.to (%s) should be at least one hour after time.from (%s)", c.TimeToArg, c.TimeFromArg) } + if c.QPM <= 0 { + return fmt.Errorf("-qpm must be positive") + } return nil } From f4a40e79b976512113993c1b3e371624e01ce041 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 12:08:24 -0600 Subject: [PATCH 11/12] final cleanup --- cmd/random-query/main.go | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 0fcd71ce9..bef9a92c2 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -42,22 +42,23 @@ import ( // RandomQueryConfig type RandomQueryConfig struct { - HostPort string - TreeDepth int - QueryCount int - Verbose bool - NumRuns int - TimeFromArg string - TimeToArg string - TimeFrom time.Time - TimeTo time.Time - TimeRange int64 - Index string - QPM int - Seed int - SrcFile string - Duration time.Duration - Target vegeta.Target + HostPort string + TreeDepth int + QueryCount int + Verbose bool + GenerateOnly bool + NumRuns int + TimeFromArg string + TimeToArg string + TimeFrom time.Time + TimeTo time.Time + TimeRange int64 + Index string + QPM int + Seed int + SrcFile string + Duration time.Duration + Target vegeta.Target IndexMap map[string]*Features @@ -441,13 +442,6 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci f = &Features{} cfg.IndexMap[index] = f } - vprint.VV("Add: %s %v %v %v", field, min, max, scale) - /* if min.Scale != scale || max.Scale != scale { - - PanicOn(fmt.Sprintf("scale error; %v:%v min scale %d, max scale %d, field scale %d, assumed they'd be equal", - index, field, min.Scale, max.Scale, scale)) - } - */ effectiveRange := uint64(max.Value) - uint64(min.Value) + 1 // if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which From 5c5e16af33a7aeb1fb3f889ecbe72334b9dcb8ad Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 12:26:23 -0600 Subject: [PATCH 12/12] added a generate only flag --- cmd/random-query/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index bef9a92c2..e9dce0354 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -114,6 +114,7 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") + fs.BoolVar(&cfg.GenerateOnly, "generate-only", false, "only generate do not run package") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.SrcFile, "query-file", "", "use pql contained in this file for query batch instead of generating") @@ -390,6 +391,10 @@ func (cfg *RandomQueryConfig) buildPayload() error { Query: request.String(), } vprint.VV("%v", request.String()) + if cfg.GenerateOnly { + // just output the PQL and exit + os.Exit(0) + } payload, err := proto.Marshal(req) if err != nil { return errors.Wrap(err, "marshaling request to protobuf")