From 63196e7981da6ec518e537352e8c711d956d393c Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Thu, 13 May 2021 11:39:23 -0500 Subject: [PATCH 1/4] add String() to shard slices to reduce error length --- executor.go | 54 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/executor.go b/executor.go index f37c23172..97e72c2e2 100644 --- a/executor.go +++ b/executor.go @@ -743,47 +743,47 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p case "Sum": statFn() res, err := e.executeSum(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeSum %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeSum %v", shardSlice(shards).String()) case "Min": statFn() res, err := e.executeMin(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMin %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeMin %v", shardSlice(shards).String()) case "Max": statFn() res, err := e.executeMax(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMax %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeMax %v", shardSlice(shards).String()) case "MinRow": statFn() res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMinRow %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeMinRow %v", shardSlice(shards).String()) case "MaxRow": statFn() res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMaxRow %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeMaxRow %v", shardSlice(shards).String()) case "Clear": statFn() res, err := e.executeClearBit(ctx, qcx, index, c, opt) - return res, errors.Wrapf(err, "executeClearBit %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeClearBit %v", shardSlice(shards).String()) case "ClearRow": statFn() res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeClearRow %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeClearRow %v", shardSlice(shards).String()) case "Distinct": statFn() res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeDistinct %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeDistinct %v", shardSlice(shards).String()) case "Store": statFn() res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeSetRow %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeSetRow %v", shardSlice(shards).String()) case "Count": statFn() res, err := e.executeCount(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeCount %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeCount %v", shardSlice(shards).String()) case "Set": statFn() res, err := e.executeSet(ctx, qcx, index, c, opt) - return res, errors.Wrapf(err, "executeSet %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeSet %v", shardSlice(shards).String()) case "SetRowAttrs": statFn() return nil, errors.Wrap(e.executeSetRowAttrs(ctx, qcx, index, c, opt), "executeSetRowAttrs") @@ -793,61 +793,61 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p case "TopK": statFn() res, err := e.executeTopK(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeTopK %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeTopK %v", shardSlice(shards).String()) case "TopN": statFn() res, err := e.executeTopN(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeTopN %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeTopN %v", shardSlice(shards).String()) case "Rows": statFn() res, err := e.executeRows(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeRows %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeRows %v", shardSlice(shards).String()) case "ExternalLookup": statFn() res, err := e.executeExternalLookup(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeExternalLookup %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeExternalLookup %v", shardSlice(shards).String()) case "Extract": statFn() res, err := e.executeExtract(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeExtract %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeExtract %v", shardSlice(shards).String()) case "GroupBy": statFn() res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeGroupBy %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeGroupBy %v", shardSlice(shards).String()) case "Options": statFn() res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeOptionsCall %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeOptionsCall %v", shardSlice(shards).String()) case "IncludesColumn": res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shardSlice(shards).String()) case "FieldValue": statFn() res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeFieldValueCall %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeFieldValueCall %v", shardSlice(shards).String()) case "Precomputed": res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executePrecomputedCall %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executePrecomputedCall %v", shardSlice(shards).String()) case "UnionRows": res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeUnionRows %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeUnionRows %v", shardSlice(shards).String()) case "ConstRow": res, err := e.executeConstRow(ctx, index, c) - return res, errors.Wrapf(err, "executeConstRow %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeConstRow %v", shardSlice(shards).String()) case "Limit": res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeLimitCall %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeLimitCall %v", shardSlice(shards).String()) case "Percentile": res, err := e.executePercentile(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executePercentile %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executePercentile %v", shardSlice(shards).String()) case "Delete": statFn() //TODO(twg) need this? res, err := e.executeDeleteRecords(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeDelete %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeDelete %v", shardSlice(shards).String()) default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap. statFn() res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeBitmapCall %v", shardSlice(shards)) + return res, errors.Wrapf(err, "executeBitmapCall %v", shardSlice(shards).String()) } } From 53da35eb00f228c6403a8ded1811743231863046 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Thu, 13 May 2021 12:01:12 -0500 Subject: [PATCH 2/4] add missing String() calls in error messages --- executor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 97e72c2e2..d93e92bbe 100644 --- a/executor.go +++ b/executor.go @@ -3052,7 +3052,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, errors.Wrapf(err, "mapReduce shards: %v", shardSlice(shards)) + return nil, errors.Wrapf(err, "mapReduce shards: %v", shardSlice(shards).String()) } results, _ := other.([]GroupCount) @@ -6066,7 +6066,7 @@ func (e *executor) mapper(ctx context.Context, eg *errgroup.Group, ch chan mapRe // Group shards together by nodes. m, err := e.shardsByNode(nodes, index, shards) if err != nil { - return errors.Wrapf(err, "shards by node %v", shardSlice(shards)) + return errors.Wrapf(err, "shards by node %v", shardSlice(shards).String()) } done := ctx.Done() From 04bf214f8111b649767e0eeed09e61ec8c9ac631 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Wed, 19 May 2021 12:20:39 -0500 Subject: [PATCH 3/4] remove shard list from mapper error message to avoid duplicate output --- executor.go | 58 ++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/executor.go b/executor.go index d93e92bbe..f24a5fdbd 100644 --- a/executor.go +++ b/executor.go @@ -743,47 +743,47 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p case "Sum": statFn() res, err := e.executeSum(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeSum %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeSum %v", shardSlice(shards)) case "Min": statFn() res, err := e.executeMin(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMin %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeMin %v", shardSlice(shards)) case "Max": statFn() res, err := e.executeMax(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMax %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeMax %v", shardSlice(shards)) case "MinRow": statFn() res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMinRow %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeMinRow %v", shardSlice(shards)) case "MaxRow": statFn() res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMaxRow %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeMaxRow %v", shardSlice(shards)) case "Clear": statFn() res, err := e.executeClearBit(ctx, qcx, index, c, opt) - return res, errors.Wrapf(err, "executeClearBit %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeClearBit %v", shardSlice(shards)) case "ClearRow": statFn() res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeClearRow %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeClearRow %v", shardSlice(shards)) case "Distinct": statFn() res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeDistinct %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeDistinct %v", shardSlice(shards)) case "Store": statFn() res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeSetRow %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeSetRow %v", shardSlice(shards)) case "Count": statFn() res, err := e.executeCount(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeCount %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeCount %v", shardSlice(shards)) case "Set": statFn() res, err := e.executeSet(ctx, qcx, index, c, opt) - return res, errors.Wrapf(err, "executeSet %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeSet %v", shardSlice(shards)) case "SetRowAttrs": statFn() return nil, errors.Wrap(e.executeSetRowAttrs(ctx, qcx, index, c, opt), "executeSetRowAttrs") @@ -793,61 +793,61 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p case "TopK": statFn() res, err := e.executeTopK(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeTopK %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeTopK %v", shardSlice(shards)) case "TopN": statFn() res, err := e.executeTopN(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeTopN %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeTopN %v", shardSlice(shards)) case "Rows": statFn() res, err := e.executeRows(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeRows %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeRows %v", shardSlice(shards)) case "ExternalLookup": statFn() res, err := e.executeExternalLookup(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeExternalLookup %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeExternalLookup %v", shardSlice(shards)) case "Extract": statFn() res, err := e.executeExtract(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeExtract %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeExtract %v", shardSlice(shards)) case "GroupBy": statFn() res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeGroupBy %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeGroupBy %v", shardSlice(shards)) case "Options": statFn() res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeOptionsCall %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeOptionsCall %v", shardSlice(shards)) case "IncludesColumn": res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shardSlice(shards)) case "FieldValue": statFn() res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeFieldValueCall %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeFieldValueCall %v", shardSlice(shards)) case "Precomputed": res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executePrecomputedCall %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executePrecomputedCall %v", shardSlice(shards)) case "UnionRows": res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeUnionRows %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeUnionRows %v", shardSlice(shards)) case "ConstRow": res, err := e.executeConstRow(ctx, index, c) - return res, errors.Wrapf(err, "executeConstRow %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeConstRow %v", shardSlice(shards)) case "Limit": res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeLimitCall %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeLimitCall %v", shardSlice(shards)) case "Percentile": res, err := e.executePercentile(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executePercentile %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executePercentile %v", shardSlice(shards)) case "Delete": statFn() //TODO(twg) need this? res, err := e.executeDeleteRecords(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeDelete %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeDelete %v", shardSlice(shards)) default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap. statFn() res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeBitmapCall %v", shardSlice(shards).String()) + return res, errors.Wrapf(err, "executeBitmapCall %v", shardSlice(shards)) } } @@ -3052,7 +3052,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, errors.Wrapf(err, "mapReduce shards: %v", shardSlice(shards).String()) + return nil, errors.Wrapf(err, "mapReduce shards: %v", shardSlice(shards)) } results, _ := other.([]GroupCount) @@ -6066,7 +6066,7 @@ func (e *executor) mapper(ctx context.Context, eg *errgroup.Group, ch chan mapRe // Group shards together by nodes. m, err := e.shardsByNode(nodes, index, shards) if err != nil { - return errors.Wrapf(err, "shards by node %v", shardSlice(shards).String()) + return errors.Wrapf(err, "") } done := ctx.Done() From e646d7ac79548655b71a7f3025e8ea4d9ecd0e01 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Thu, 20 May 2021 09:55:42 -0500 Subject: [PATCH 4/4] wrap mapper error with shards by node --- executor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor.go b/executor.go index f24a5fdbd..025b5baf5 100644 --- a/executor.go +++ b/executor.go @@ -6066,7 +6066,7 @@ func (e *executor) mapper(ctx context.Context, eg *errgroup.Group, ch chan mapRe // Group shards together by nodes. m, err := e.shardsByNode(nodes, index, shards) if err != nil { - return errors.Wrapf(err, "") + return errors.Wrapf(err, "shards by node") } done := ctx.Done()