diff --git a/executor.go b/executor.go index ac9f4455e..82f215fef 100644 --- a/executor.go +++ b/executor.go @@ -605,41 +605,6 @@ func (e *executor) preprocessQuery(ctx context.Context, qcx *Qcx, index string, } } -type shardSlice []uint64 - -// String creates a run-length encoded representation of a slice of shard IDs (integers). -// For example, []uint64{0, 1, 3, 4, 5, 7, 8, 9, 11, 13} is represented as -// [0-1,3-5,7-9,11,13]. -func (s shardSlice) String() string { - if len(s) == 0 { - // surely this is impossible - return "[]" - } - runs := make([]string, 0, len(s)/2) - start := s[0] - end := start - for n := 1; n < len(s); n++ { - if s[n] == end+1 { - end = s[n] - } else { - repr := fmt.Sprintf("%d", start) - if end > start { - repr += fmt.Sprintf("-%d", end) - } - runs = append(runs, repr) - start = s[n] - end = start - } - } - repr := fmt.Sprintf("%d", start) - if end > start { - repr += fmt.Sprintf("-%d", end) - } - runs = append(runs, repr) - - return "[" + strings.Join(runs, ",") + "]" -} - // executeCall executes a call. func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCall") @@ -689,105 +654,105 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p case "Sum": statFn() res, err := e.executeSum(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeSum %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeSum") case "Min": statFn() res, err := e.executeMin(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMin %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeMin") case "Max": statFn() res, err := e.executeMax(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMax %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeMax") case "MinRow": statFn() res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMinRow %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeMinRow") case "MaxRow": statFn() res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMaxRow %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeMaxRow") case "Clear": statFn() res, err := e.executeClearBit(ctx, qcx, index, c, opt) - return res, errors.Wrapf(err, "executeClearBit %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeClearBit") case "ClearRow": statFn() res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeClearRow %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeClearRow") case "Distinct": statFn() res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeDistinct %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeDistinct") case "Store": statFn() res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeSetRow %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeSetRow") case "Count": statFn() res, err := e.executeCount(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeCount %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeCount") case "Set": statFn() res, err := e.executeSet(ctx, qcx, index, c, opt) - return res, errors.Wrapf(err, "executeSet %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeSet") case "TopK": statFn() res, err := e.executeTopK(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeTopK %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeTopK") case "TopN": statFn() res, err := e.executeTopN(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeTopN %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeTopN") case "Rows": statFn() res, err := e.executeRows(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeRows %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeRows") case "ExternalLookup": statFn() res, err := e.executeExternalLookup(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeExternalLookup %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeExternalLookup") case "Extract": statFn() res, err := e.executeExtract(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeExtract %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeExtract") case "GroupBy": statFn() res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeGroupBy %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeGroupBy") case "Options": statFn() res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeOptionsCall %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeOptionsCall") case "IncludesColumn": res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeIncludesColumnCall") case "FieldValue": statFn() res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeFieldValueCall %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeFieldValueCall") case "Precomputed": res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executePrecomputedCall %v", shardSlice(shards)) + return res, errors.Wrap(err, "executePrecomputedCall") case "UnionRows": res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeUnionRows %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeUnionRows") case "ConstRow": res, err := e.executeConstRow(ctx, index, c) - return res, errors.Wrapf(err, "executeConstRow %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeConstRow") case "Limit": res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeLimitCall %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeLimitCall") case "Percentile": res, err := e.executePercentile(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executePercentile %v", shardSlice(shards)) + return res, errors.Wrap(err, "executePercentile") case "Delete": statFn() //TODO(twg) need this? res, err := e.executeDeleteRecords(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeDelete %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeDelete") default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap. statFn() res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeBitmapCall %v", shardSlice(shards)) + return res, errors.Wrap(err, "executeBitmapCall") } } @@ -2939,7 +2904,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, errors.Wrapf(err, "mapReduce shards: %v", shardSlice(shards)) + return nil, errors.Wrap(err, "mapReduce") } results, _ := other.([]GroupCount) diff --git a/go.mod b/go.mod index 67d9c30bd..25e8925e5 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/rs/cors v1.7.0 // indirect github.com/satori/go.uuid v1.2.0 - github.com/shirou/gopsutil/v3 v3.20.11 + github.com/shirou/gopsutil/v3 v3.21.9 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 @@ -51,7 +51,6 @@ require ( golang.org/x/mod v0.4.2 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect golang.org/x/text v0.3.5 // indirect google.golang.org/grpc v1.28.0 gopkg.in/yaml.v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 60db445b5..7f90a4524 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgR github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -92,8 +92,8 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= @@ -283,8 +283,8 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554 h1:88K0ffxhVphUHxlqW4ewOaXdnJByH4LcCuvYfv0QI/M= github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA= -github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4= +github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg= +github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= @@ -319,11 +319,14 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= +github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -435,17 +438,17 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71 h1:ikCpsnYR+Ew0vu99XlDp55lGgDJdIMx3f4a18jfse/s= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/http/client.go b/http/client.go index 1d41a114f..ca3c52c11 100644 --- a/http/client.go +++ b/http/client.go @@ -522,7 +522,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index str // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { - return nil, errors.Wrapf(err, "'%s', shards %v", queryRequest.Query, queryRequest.Shards) + return nil, errors.Wrapf(err, "'%s'", queryRequest.Query) } defer resp.Body.Close() diff --git a/rbf/db.go b/rbf/db.go index 4f88de259..68a41ac08 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -206,6 +206,13 @@ func (db *DB) checkpoint() error { return nil // skip if transactions open } + // Check if there are any WAL pages, if not do nothing as + // checkpointing and calling fsync can be very expensive even if + // there are no writes. + if db.walPageN == 0 { + return nil + } + for i := 0; i < db.walPageN; i++ { page, err := db.readWALPageAt(i) if err != nil { @@ -245,7 +252,7 @@ func (db *DB) checkpoint() error { db.walPageN = 0 db.pageMap = NewPageMap() - // Notify halted tranactions that the WAL has been checkpointed. + // Notify halted transactions that the WAL has been checkpointed. db.haltCond.Broadcast() return nil