Merge branch 'master' into 54mir/toggle-schema-details

This commit is contained in:
Samir Patel 2021-11-01 09:31:35 -05:00 • committed by GitHub
commit 8ee0528858
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 51 additions and 77 deletions

View file

@ -605,41 +605,6 @@ func (e *executor) preprocessQuery(ctx context.Context, qcx *Qcx, index string,
}
}
type shardSlice []uint64
// String creates a run-length encoded representation of a slice of shard IDs (integers).
// For example, []uint64{0, 1, 3, 4, 5, 7, 8, 9, 11, 13} is represented as
// [0-1,3-5,7-9,11,13].
func (s shardSlice) String() string {
if len(s) == 0 {
// surely this is impossible
return "[]"
}
runs := make([]string, 0, len(s)/2)
start := s[0]
end := start
for n := 1; n < len(s); n++ {
if s[n] == end+1 {
end = s[n]
} else {
repr := fmt.Sprintf("%d", start)
if end > start {
repr += fmt.Sprintf("-%d", end)
}
runs = append(runs, repr)
start = s[n]
end = start
}
}
repr := fmt.Sprintf("%d", start)
if end > start {
repr += fmt.Sprintf("-%d", end)
}
runs = append(runs, repr)
return "[" + strings.Join(runs, ",") + "]"
}
// executeCall executes a call.
func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCall")
@ -689,105 +654,105 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p
case "Sum":
statFn()
res, err := e.executeSum(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeSum %v", shardSlice(shards))
return res, errors.Wrap(err, "executeSum")
case "Min":
statFn()
res, err := e.executeMin(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeMin %v", shardSlice(shards))
return res, errors.Wrap(err, "executeMin")
case "Max":
statFn()
res, err := e.executeMax(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeMax %v", shardSlice(shards))
return res, errors.Wrap(err, "executeMax")
case "MinRow":
statFn()
res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeMinRow %v", shardSlice(shards))
return res, errors.Wrap(err, "executeMinRow")
case "MaxRow":
statFn()
res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeMaxRow %v", shardSlice(shards))
return res, errors.Wrap(err, "executeMaxRow")
case "Clear":
statFn()
res, err := e.executeClearBit(ctx, qcx, index, c, opt)
return res, errors.Wrapf(err, "executeClearBit %v", shardSlice(shards))
return res, errors.Wrap(err, "executeClearBit")
case "ClearRow":
statFn()
res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeClearRow %v", shardSlice(shards))
return res, errors.Wrap(err, "executeClearRow")
case "Distinct":
statFn()
res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeDistinct %v", shardSlice(shards))
return res, errors.Wrap(err, "executeDistinct")
case "Store":
statFn()
res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeSetRow %v", shardSlice(shards))
return res, errors.Wrap(err, "executeSetRow")
case "Count":
statFn()
res, err := e.executeCount(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeCount %v", shardSlice(shards))
return res, errors.Wrap(err, "executeCount")
case "Set":
statFn()
res, err := e.executeSet(ctx, qcx, index, c, opt)
return res, errors.Wrapf(err, "executeSet %v", shardSlice(shards))
return res, errors.Wrap(err, "executeSet")
case "TopK":
statFn()
res, err := e.executeTopK(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeTopK %v", shardSlice(shards))
return res, errors.Wrap(err, "executeTopK")
case "TopN":
statFn()
res, err := e.executeTopN(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeTopN %v", shardSlice(shards))
return res, errors.Wrap(err, "executeTopN")
case "Rows":
statFn()
res, err := e.executeRows(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeRows %v", shardSlice(shards))
return res, errors.Wrap(err, "executeRows")
case "ExternalLookup":
statFn()
res, err := e.executeExternalLookup(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeExternalLookup %v", shardSlice(shards))
return res, errors.Wrap(err, "executeExternalLookup")
case "Extract":
statFn()
res, err := e.executeExtract(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeExtract %v", shardSlice(shards))
return res, errors.Wrap(err, "executeExtract")
case "GroupBy":
statFn()
res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeGroupBy %v", shardSlice(shards))
return res, errors.Wrap(err, "executeGroupBy")
case "Options":
statFn()
res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeOptionsCall %v", shardSlice(shards))
return res, errors.Wrap(err, "executeOptionsCall")
case "IncludesColumn":
res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shardSlice(shards))
return res, errors.Wrap(err, "executeIncludesColumnCall")
case "FieldValue":
statFn()
res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeFieldValueCall %v", shardSlice(shards))
return res, errors.Wrap(err, "executeFieldValueCall")
case "Precomputed":
res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executePrecomputedCall %v", shardSlice(shards))
return res, errors.Wrap(err, "executePrecomputedCall")
case "UnionRows":
res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeUnionRows %v", shardSlice(shards))
return res, errors.Wrap(err, "executeUnionRows")
case "ConstRow":
res, err := e.executeConstRow(ctx, index, c)
return res, errors.Wrapf(err, "executeConstRow %v", shardSlice(shards))
return res, errors.Wrap(err, "executeConstRow")
case "Limit":
res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeLimitCall %v", shardSlice(shards))
return res, errors.Wrap(err, "executeLimitCall")
case "Percentile":
res, err := e.executePercentile(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executePercentile %v", shardSlice(shards))
return res, errors.Wrap(err, "executePercentile")
case "Delete":
statFn() //TODO(twg) need this?
res, err := e.executeDeleteRecords(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeDelete %v", shardSlice(shards))
return res, errors.Wrap(err, "executeDelete")
default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap.
statFn()
res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeBitmapCall %v", shardSlice(shards))
return res, errors.Wrap(err, "executeBitmapCall")
}
}
@ -2939,7 +2904,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c
// Get full result set.
other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return nil, errors.Wrapf(err, "mapReduce shards: %v", shardSlice(shards))
return nil, errors.Wrap(err, "mapReduce")
}
results, _ := other.([]GroupCount)

3
go.mod
View file

@ -37,7 +37,7 @@ require (
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
github.com/rs/cors v1.7.0 // indirect
github.com/satori/go.uuid v1.2.0
github.com/shirou/gopsutil/v3 v3.20.11
github.com/shirou/gopsutil/v3 v3.21.9
github.com/spf13/cobra v1.1.1
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.7.1
@ -51,7 +51,6 @@ require (
golang.org/x/mod v0.4.2
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
golang.org/x/text v0.3.5 // indirect
google.golang.org/grpc v1.28.0
gopkg.in/yaml.v2 v2.3.0 // indirect

23
go.sum
View file

@ -22,8 +22,8 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgR
github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@ -92,8 +92,8 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
@ -283,8 +283,8 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554 h1:88K0ffxhVphUHxlqW4ewOaXdnJByH4LcCuvYfv0QI/M=
github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA=
github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4=
github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg=
github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
@ -319,11 +319,14 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@ -435,17 +438,17 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71 h1:ikCpsnYR+Ew0vu99XlDp55lGgDJdIMx3f4a18jfse/s=
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

View file

@ -522,7 +522,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index str
// Execute request against the host.
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
return nil, errors.Wrapf(err, "'%s', shards %v", queryRequest.Query, queryRequest.Shards)
return nil, errors.Wrapf(err, "'%s'", queryRequest.Query)
}
defer resp.Body.Close()

View file

@ -206,6 +206,13 @@ func (db *DB) checkpoint() error {
return nil // skip if transactions open
}
// Check if there are any WAL pages, if not do nothing as
// checkpointing and calling fsync can be very expensive even if
// there are no writes.
if db.walPageN == 0 {
return nil
}
for i := 0; i < db.walPageN; i++ {
page, err := db.readWALPageAt(i)
if err != nil {
@ -245,7 +252,7 @@ func (db *DB) checkpoint() error {
db.walPageN = 0
db.pageMap = NewPageMap()
// Notify halted tranactions that the WAL has been checkpointed.
// Notify halted transactions that the WAL has been checkpointed.
db.haltCond.Broadcast()
return nil