diff --git a/NOTICE b/NOTICE index 2d655865e..63c95e3c3 100644 --- a/NOTICE +++ b/NOTICE @@ -115,3 +115,27 @@ The file /server/tlsconfig.go contains a modified redistribution of bridge CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows: + + The MIT License (MIT) + + Copyright (c) 2015 Nick Galbreath + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. diff --git a/go.mod b/go.mod index ccdca5ab4..9d9ecd366 100644 --- a/go.mod +++ b/go.mod @@ -2,16 +2,12 @@ module github.com/pilosa/pilosa/v2 replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 -// TODO: Remove the following line if this is merged: https://github.com/client9/reopen/pull/9 -replace github.com/client9/reopen => github.com/codysoyland/reopen v1.0.1-0.20200414204206-42cbe848be3b - require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/boltdb/bolt v1.3.1 github.com/cespare/xxhash v1.1.0 - github.com/client9/reopen v0.0.0-00010101000000-000000000000 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/davecgh/go-spew v1.1.1 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 109caa573..9da5feace 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/codysoyland/reopen v1.0.1-0.20200414204206-42cbe848be3b h1:CP/etmJf4LXC6I1MJQt+I7oh8geIkWllD/NT0T23Y0c= -github.com/codysoyland/reopen v1.0.1-0.20200414204206-42cbe848be3b/go.mod h1:8fFEqM7bujfAJw/3T2Z/K8FMh+5vEsZ/8eDbNAVGFH8= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= diff --git a/license.exceptions b/license.exceptions index b9087d473..9f453efc4 100644 --- a/license.exceptions +++ b/license.exceptions @@ -8,3 +8,5 @@ ./roaring/btree.go ./roaring/btree_test.go ./proto/pilosa.pb.go +./logger/filewriter.go +./logger/filewriter_test.go diff --git a/logger/filewriter.go b/logger/filewriter.go new file mode 100644 index 000000000..9bf870145 --- /dev/null +++ b/logger/filewriter.go @@ -0,0 +1,108 @@ +// This file is a modified redistribution of reopen (github.com/client9/reopen), +// which is governed by the following license notice: +// +// The MIT License (MIT) +// +// Copyright (c) 2015 Nick Galbreath +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package logger + +import ( + "os" + "sync" +) + +// FileWriter that can also be reopened +type FileWriter struct { + mu sync.Mutex // ensures close / reopen / write are not called at the same time, protects f + f *os.File + mode os.FileMode + name string +} + +// Close calls the underlyding File.Close() +func (f *FileWriter) Close() error { + f.mu.Lock() + err := f.f.Close() + f.mu.Unlock() + return err +} + +// mutex free version +func (f *FileWriter) reopen() error { + if f.f != nil { + f.f.Close() + f.f = nil + } + newf, err := os.OpenFile(f.name, os.O_WRONLY|os.O_APPEND|os.O_CREATE, f.mode) + if err != nil { + f.f = nil + return err + } + f.f = newf + + return nil +} + +// Reopen the file +func (f *FileWriter) Reopen() error { + f.mu.Lock() + err := f.reopen() + f.mu.Unlock() + return err +} + +// Write implements the stander io.Writer interface +func (f *FileWriter) Write(p []byte) (int, error) { + f.mu.Lock() + n, err := f.f.Write(p) + f.mu.Unlock() + return n, err +} + +// Fd returns the file descriptor of the underlying file. +func (f *FileWriter) Fd() uintptr { + f.mu.Lock() + n := f.f.Fd() + f.mu.Unlock() + return n +} + +// NewFileWriter opens a file for appending and writing and can be reopened. +// it is a ReopenWriteCloser... +func NewFileWriter(name string) (*FileWriter, error) { + // Standard default mode + return NewFileWriterMode(name, 0666) +} + +// NewFileWriterMode opens a Reopener file with a specific permission +func NewFileWriterMode(name string, mode os.FileMode) (*FileWriter, error) { + writer := FileWriter{ + f: nil, + name: name, + mode: mode, + } + err := writer.reopen() + if err != nil { + return nil, err + } + return &writer, nil +} diff --git a/logger/filewriter_test.go b/logger/filewriter_test.go new file mode 100644 index 000000000..1004daaad --- /dev/null +++ b/logger/filewriter_test.go @@ -0,0 +1,162 @@ +// This file is a modified redistribution of reopen (github.com/client9/reopen), +// which is governed by the following license notice: +// +// The MIT License (MIT) +// +// Copyright (c) 2015 Nick Galbreath +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package logger + +import ( + "io/ioutil" + "os" + "testing" +) + +// TestReopenAppend -- make sure we always append to an existing file +// +// 1. Create a sample file using normal means +// 2. Open a ioreopen.File +// write line 1 +// 3. call Reopen +// write line 2 +// 4. close file +// 5. read file, make sure it contains line0,line1,line2 +// +func TestReopenAppend(t *testing.T) { + // TODO fix + var fname = "/tmp/foo" + + // Step 1 -- Create a sample file using normal means + forig, err := os.Create(fname) + if err != nil { + t.Fatalf("Unable to create initial file %s: %s", fname, err) + } + _, err = forig.Write([]byte("line0\n")) + if err != nil { + t.Fatalf("Unable to write initial line %s: %s", fname, err) + } + err = forig.Close() + if err != nil { + t.Fatalf("Unable to close initial file: %s", err) + } + + // Test that making a new File appends + f, err := NewFileWriter(fname) + if err != nil { + t.Fatalf("Unable to create %s", fname) + } + _, err = f.Write([]byte("line1\n")) + if err != nil { + t.Errorf("Got write error1: %s", err) + } + + // Test that reopen always appends + err = f.Reopen() + if err != nil { + t.Errorf("Got reopen error %s: %s", fname, err) + } + _, err = f.Write([]byte("line2\n")) + if err != nil { + t.Errorf("Got write error2 on %s: %s", fname, err) + } + err = f.Close() + if err != nil { + t.Errorf("Got closing error for %s: %s", fname, err) + } + + out, err := ioutil.ReadFile(fname) + if err != nil { + t.Fatalf("Unable read in final file %s: %s", fname, err) + } + + outstr := string(out) + if outstr != "line0\nline1\nline2\n" { + t.Errorf("Result was %s", outstr) + } +} + +// Test that reopen works when Inode is swapped out +// 1. Create a sample file using normal means +// 2. Open a ioreopen.File +// write line 1 +// 3. call Reopen +// write line 2 +// 4. close file +// 5. read file, make sure it contains line0,line1,line2 +// +func TestChangeInode(t *testing.T) { + // TODO fix + var fname = "/tmp/foo" + + // Step 1 -- Create a empty sample file + forig, err := os.Create(fname) + if err != nil { + t.Fatalf("Unable to create initial file %s: %s", fname, err) + } + err = forig.Close() + if err != nil { + t.Fatalf("Unable to close initial file: %s", err) + } + + // Test that making a new File appends + f, err := NewFileWriter(fname) + if err != nil { + t.Fatalf("Unable to create %s", fname) + } + _, err = f.Write([]byte("line1\n")) + if err != nil { + t.Errorf("Got write error1: %s", err) + } + + // Now move file + err = os.Rename(fname, fname+".orig") + if err != nil { + t.Errorf("Renaming error: %s", err) + } + _, err = f.Write([]byte("after1\n")) + if err != nil { + t.Errorf("Write error: %s", err) + } + + // Test that reopen always appends + err = f.Reopen() + if err != nil { + t.Errorf("Got reopen error %s: %s", fname, err) + } + _, err = f.Write([]byte("line2\n")) + if err != nil { + t.Errorf("Got write error2 on %s: %s", fname, err) + } + err = f.Close() + if err != nil { + t.Errorf("Got closing error for %s: %s", fname, err) + } + + out, err := ioutil.ReadFile(fname) + if err != nil { + t.Fatalf("Unable read in final file %s: %s", fname, err) + } + outstr := string(out) + if outstr != "line2\n" { + t.Errorf("Result was %s", outstr) + } +} diff --git a/server/server.go b/server/server.go index f20a52ebb..87fc1c361 100644 --- a/server/server.go +++ b/server/server.go @@ -36,7 +36,6 @@ import ( "golang.org/x/sync/errgroup" - "github.com/client9/reopen" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/boltdb" "github.com/pilosa/pilosa/v2/encoding/proto" @@ -430,12 +429,12 @@ func (m *Command) setupNetworking() error { // setupLogger sets up the logger based on the configuration. func (m *Command) setupLogger() error { - var f *reopen.FileWriter + var f *logger.FileWriter var err error if m.Config.LogPath == "" { m.logOutput = m.Stderr } else { - f, err = reopen.NewFileWriter(m.Config.LogPath) + f, err = logger.NewFileWriter(m.Config.LogPath) if err != nil { return errors.Wrap(err, "opening file") }