From 8bfa719484494473920311634ff72b81dddfcb6d Mon Sep 17 00:00:00 2001 From: jaffee Date: Thu, 1 Dec 2016 16:58:56 -0700 Subject: [PATCH] copy errgroup code to bench and remove giant blob of dependencies --- bench/LICENSE | 30 +++++++ bench/bench.go | 3 +- bench/errgroup.go | 67 +++++++++++++++ bench/errgroup_test.go | 176 ++++++++++++++++++++++++++++++++++++++++ cmd/pilosa/main_test.go | 1 - glide.lock | 12 +-- glide.yaml | 4 +- 7 files changed, 277 insertions(+), 16 deletions(-) create mode 100644 bench/LICENSE create mode 100644 bench/errgroup.go create mode 100644 bench/errgroup_test.go diff --git a/bench/LICENSE b/bench/LICENSE new file mode 100644 index 000000000..e9de11fa5 --- /dev/null +++ b/bench/LICENSE @@ -0,0 +1,30 @@ +The following applies to the files errgroup.go and errgroup_test.go +in the same directory this file resides: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/bench/bench.go b/bench/bench.go index 167cb4adb..b8faeea00 100644 --- a/bench/bench.go +++ b/bench/bench.go @@ -2,7 +2,6 @@ package bench import ( "fmt" - "golang.org/x/sync/errgroup" "strconv" "sync" "time" @@ -51,7 +50,7 @@ type parallelBenchmark struct { // Init calls Init for each benchmark. If there are any errors, it will return a // non-nil error value. func (pb *parallelBenchmark) Init(hosts []string, agentNum int) error { - var g errgroup.Group + var g ErrGroup for i, _ := range pb.benchmarkers { b := pb.benchmarkers[i] g.Go(func() error { diff --git a/bench/errgroup.go b/bench/errgroup.go new file mode 100644 index 000000000..ac0bb4819 --- /dev/null +++ b/bench/errgroup.go @@ -0,0 +1,67 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package errgroup provides synchronization, error propagation, and Context +// cancelation for groups of goroutines working on subtasks of a common task. +package bench + +import ( + "sync" + + "context" +) + +// An ErrGroup is a collection of goroutines working on subtasks that are part of +// the same overall task. +// +// A zero ErrGroup is valid and does not cancel on error. +type ErrGroup struct { + cancel func() + + wg sync.WaitGroup + + errOnce sync.Once + err error +} + +// WithContext returns a new ErrGroup and an associated Context derived from ctx. +// +// The derived Context is canceled the first time a function passed to Go +// returns a non-nil error or the first time Wait returns, whichever occurs +// first. +func WithContext(ctx context.Context) (*ErrGroup, context.Context) { + ctx, cancel := context.WithCancel(ctx) + return &ErrGroup{cancel: cancel}, ctx +} + +// Wait blocks until all function calls from the Go method have returned, then +// returns the first non-nil error (if any) from them. +func (g *ErrGroup) Wait() error { + g.wg.Wait() + if g.cancel != nil { + g.cancel() + } + return g.err +} + +// Go calls the given function in a new goroutine. +// +// The first call to return a non-nil error cancels the group; its error will be +// returned by Wait. +func (g *ErrGroup) Go(f func() error) { + g.wg.Add(1) + + go func() { + defer g.wg.Done() + + if err := f(); err != nil { + g.errOnce.Do(func() { + g.err = err + if g.cancel != nil { + g.cancel() + } + }) + } + }() +} diff --git a/bench/errgroup_test.go b/bench/errgroup_test.go new file mode 100644 index 000000000..546cceb60 --- /dev/null +++ b/bench/errgroup_test.go @@ -0,0 +1,176 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bench_test + +import ( + "errors" + "fmt" + "net/http" + "os" + "testing" + + "context" + "github.com/pilosa/pilosa/bench" +) + +var ( + Web = fakeSearch("web") + Image = fakeSearch("image") + Video = fakeSearch("video") +) + +type Result string +type Search func(ctx context.Context, query string) (Result, error) + +func fakeSearch(kind string) Search { + return func(_ context.Context, query string) (Result, error) { + return Result(fmt.Sprintf("%s result for %q", kind, query)), nil + } +} + +// JustErrors illustrates the use of a ErrGroup in place of a sync.WaitGroup to +// simplify goroutine counting and error handling. This example is derived from +// the sync.WaitGroup example at https://golang.org/pkg/sync/#example_WaitGroup. +func ExampleGroup_justErrors() { + var g bench.ErrGroup + var urls = []string{ + "http://www.golang.org/", + "http://www.google.com/", + "http://www.somestupidname.com/", + } + for _, url := range urls { + // Launch a goroutine to fetch the URL. + url := url // https://golang.org/doc/faq#closures_and_goroutines + g.Go(func() error { + // Fetch the URL. + resp, err := http.Get(url) + if err == nil { + resp.Body.Close() + } + return err + }) + } + // Wait for all HTTP fetches to complete. + if err := g.Wait(); err == nil { + fmt.Println("Successfully fetched all URLs.") + } +} + +// Parallel illustrates the use of a ErrGroup for synchronizing a simple parallel +// task: the "Google Search 2.0" function from +// https://talks.golang.org/2012/concurrency.slide#46, augmented with a Context +// and error-handling. +func ExampleGroup_parallel() { + Google := func(ctx context.Context, query string) ([]Result, error) { + g, ctx := bench.WithContext(ctx) + + searches := []Search{Web, Image, Video} + results := make([]Result, len(searches)) + for i, search := range searches { + i, search := i, search // https://golang.org/doc/faq#closures_and_goroutines + g.Go(func() error { + result, err := search(ctx, query) + if err == nil { + results[i] = result + } + return err + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + return results, nil + } + + results, err := Google(context.Background(), "golang") + if err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + for _, result := range results { + fmt.Println(result) + } + + // Output: + // web result for "golang" + // image result for "golang" + // video result for "golang" +} + +func TestZeroGroup(t *testing.T) { + err1 := errors.New("errgroup_test: 1") + err2 := errors.New("errgroup_test: 2") + + cases := []struct { + errs []error + }{ + {errs: []error{}}, + {errs: []error{nil}}, + {errs: []error{err1}}, + {errs: []error{err1, nil}}, + {errs: []error{err1, nil, err2}}, + } + + for _, tc := range cases { + var g bench.ErrGroup + + var firstErr error + for i, err := range tc.errs { + err := err + g.Go(func() error { return err }) + + if firstErr == nil && err != nil { + firstErr = err + } + + if gErr := g.Wait(); gErr != firstErr { + t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+ + "g.Wait() = %v; want %v", + g, tc.errs[:i+1], err, firstErr) + } + } + } +} + +func TestWithContext(t *testing.T) { + errDoom := errors.New("group_test: doomed") + + cases := []struct { + errs []error + want error + }{ + {want: nil}, + {errs: []error{nil}, want: nil}, + {errs: []error{errDoom}, want: errDoom}, + {errs: []error{errDoom, nil}, want: errDoom}, + } + + for _, tc := range cases { + g, ctx := bench.WithContext(context.Background()) + + for _, err := range tc.errs { + err := err + g.Go(func() error { return err }) + } + + if err := g.Wait(); err != tc.want { + t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+ + "g.Wait() = %v; want %v", + g, tc.errs, err, tc.want) + } + + canceled := false + select { + case <-ctx.Done(): + canceled = true + default: + } + if !canceled { + t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+ + "ctx.Done() was not closed", + g, tc.errs) + } + } +} diff --git a/cmd/pilosa/main_test.go b/cmd/pilosa/main_test.go index 20d2e8e64..e4969c1e0 100644 --- a/cmd/pilosa/main_test.go +++ b/cmd/pilosa/main_test.go @@ -16,7 +16,6 @@ import ( "testing" "testing/quick" - "context" "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" main "github.com/pilosa/pilosa/cmd/pilosa" diff --git a/glide.lock b/glide.lock index d4a83ea8f..56d3ba799 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: d912921b0318c326771cd49debeec509d4ffb0558bd37e53570f6404fbec6604 -updated: 2016-11-29T16:10:39.53889541-06:00 +hash: 9afe91839785e89880061fd2919b0a5cdb10d6f4c06b344c2924eeed0c9bfd65 +updated: 2016-12-01T16:53:45.096741528-07:00 imports: - name: github.com/boltdb/bolt version: 4b1ebc1869ad66568b313d0dc410e2be72670dda @@ -21,14 +21,6 @@ imports: version: a6b377e3400b08991b80d6805d627f347f983866 subpackages: - lru -- name: golang.org/x/net - version: 4971afdc2f162e82d185353533d3cf16188a9f4e - subpackages: - - context -- name: golang.org/x/sync - version: 1ae7c7b29e06598039be46c5083819ba6fd7a97e - subpackages: - - errgroup - name: golang.org/x/sys version: c200b10b5d5e122be351b67af224adc6128af5bf subpackages: diff --git a/glide.yaml b/glide.yaml index be20b88e5..5b1808065 100644 --- a/glide.yaml +++ b/glide.yaml @@ -24,6 +24,4 @@ import: version: c200b10b5d5e122be351b67af224adc6128af5bf subpackages: - unix -- package: golang.org/x/sync - subpackages: - - errgroup +