scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #307 from codysoyland/copy-reopen

Browse source 
Copy reopen.FileWriter into pilosa
This commit is contained in:
Cody Soyland 2020-04-20 12:41:09 -05:00 • committed by GitHub
commit a353527705
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 298 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
NOTICE
View file

@ -115,3 +115,27 @@ The file /server/tlsconfig.go contains a modified redistribution of bridge
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows:
The MIT License (MIT)
Copyright (c) 2015 Nick Galbreath
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

4
go.mod
View file

@ -2,16 +2,12 @@ module github.com/pilosa/pilosa/v2
replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021
// TODO: Remove the following line if this is merged: https://github.com/client9/reopen/pull/9
replace github.com/client9/reopen => github.com/codysoyland/reopen v1.0.1-0.20200414204206-42cbe848be3b
require (
github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/boltdb/bolt v1.3.1
github.com/cespare/xxhash v1.1.0
github.com/client9/reopen v0.0.0-00010101000000-000000000000
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
github.com/davecgh/go-spew v1.1.1
github.com/go-ole/go-ole v1.2.4 // indirect

2
go.sum
View file

@ -26,8 +26,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/codysoyland/reopen v1.0.1-0.20200414204206-42cbe848be3b h1:CP/etmJf4LXC6I1MJQt+I7oh8geIkWllD/NT0T23Y0c=
github.com/codysoyland/reopen v1.0.1-0.20200414204206-42cbe848be3b/go.mod h1:8fFEqM7bujfAJw/3T2Z/K8FMh+5vEsZ/8eDbNAVGFH8=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=

2
license.exceptions
View file

@ -8,3 +8,5 @@
./roaring/btree.go
./roaring/btree_test.go
./proto/pilosa.pb.go
./logger/filewriter.go
./logger/filewriter_test.go

108
logger/filewriter.go Normal file
View file

@ -0,0 +1,108 @@
// This file is a modified redistribution of reopen (github.com/client9/reopen),
// which is governed by the following license notice:
//
// The MIT License (MIT)
//
// Copyright (c) 2015 Nick Galbreath
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package logger
import (
"os"
"sync"
)
// FileWriter that can also be reopened
type FileWriter struct {
mu sync.Mutex // ensures close / reopen / write are not called at the same time, protects f
f *os.File
mode os.FileMode
name string
}
// Close calls the underlyding File.Close()
func (f *FileWriter) Close() error {
f.mu.Lock()
err := f.f.Close()
f.mu.Unlock()
return err
}
// mutex free version
func (f *FileWriter) reopen() error {
if f.f != nil {
f.f.Close()
f.f = nil
}
newf, err := os.OpenFile(f.name, os.O_WRONLY|os.O_APPEND|os.O_CREATE, f.mode)
if err != nil {
f.f = nil
return err
}
f.f = newf
return nil
}
// Reopen the file
func (f *FileWriter) Reopen() error {
f.mu.Lock()
err := f.reopen()
f.mu.Unlock()
return err
}
// Write implements the stander io.Writer interface
func (f *FileWriter) Write(p []byte) (int, error) {
f.mu.Lock()
n, err := f.f.Write(p)
f.mu.Unlock()
return n, err
}
// Fd returns the file descriptor of the underlying file.
func (f *FileWriter) Fd() uintptr {
f.mu.Lock()
n := f.f.Fd()
f.mu.Unlock()
return n
}
// NewFileWriter opens a file for appending and writing and can be reopened.
// it is a ReopenWriteCloser...
func NewFileWriter(name string) (*FileWriter, error) {
// Standard default mode
return NewFileWriterMode(name, 0666)
}
// NewFileWriterMode opens a Reopener file with a specific permission
func NewFileWriterMode(name string, mode os.FileMode) (*FileWriter, error) {
writer := FileWriter{
f: nil,
name: name,
mode: mode,
}
err := writer.reopen()
if err != nil {
return nil, err
}
return &writer, nil
}

162
logger/filewriter_test.go Normal file
View file

@ -0,0 +1,162 @@
// This file is a modified redistribution of reopen (github.com/client9/reopen),
// which is governed by the following license notice:
//
// The MIT License (MIT)
//
// Copyright (c) 2015 Nick Galbreath
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package logger
import (
"io/ioutil"
"os"
"testing"
)
// TestReopenAppend -- make sure we always append to an existing file
//
// 1. Create a sample file using normal means
// 2. Open a ioreopen.File
// write line 1
// 3. call Reopen
// write line 2
// 4. close file
// 5. read file, make sure it contains line0,line1,line2
//
func TestReopenAppend(t *testing.T) {
// TODO fix
var fname = "/tmp/foo"
// Step 1 -- Create a sample file using normal means
forig, err := os.Create(fname)
if err != nil {
t.Fatalf("Unable to create initial file %s: %s", fname, err)
}
_, err = forig.Write([]byte("line0\n"))
if err != nil {
t.Fatalf("Unable to write initial line %s: %s", fname, err)
}
err = forig.Close()
if err != nil {
t.Fatalf("Unable to close initial file: %s", err)
}
// Test that making a new File appends
f, err := NewFileWriter(fname)
if err != nil {
t.Fatalf("Unable to create %s", fname)
}
_, err = f.Write([]byte("line1\n"))
if err != nil {
t.Errorf("Got write error1: %s", err)
}
// Test that reopen always appends
err = f.Reopen()
if err != nil {
t.Errorf("Got reopen error %s: %s", fname, err)
}
_, err = f.Write([]byte("line2\n"))
if err != nil {
t.Errorf("Got write error2 on %s: %s", fname, err)
}
err = f.Close()
if err != nil {
t.Errorf("Got closing error for %s: %s", fname, err)
}
out, err := ioutil.ReadFile(fname)
if err != nil {
t.Fatalf("Unable read in final file %s: %s", fname, err)
}
outstr := string(out)
if outstr != "line0\nline1\nline2\n" {
t.Errorf("Result was %s", outstr)
}
}
// Test that reopen works when Inode is swapped out
// 1. Create a sample file using normal means
// 2. Open a ioreopen.File
// write line 1
// 3. call Reopen
// write line 2
// 4. close file
// 5. read file, make sure it contains line0,line1,line2
//
func TestChangeInode(t *testing.T) {
// TODO fix
var fname = "/tmp/foo"
// Step 1 -- Create a empty sample file
forig, err := os.Create(fname)
if err != nil {
t.Fatalf("Unable to create initial file %s: %s", fname, err)
}
err = forig.Close()
if err != nil {
t.Fatalf("Unable to close initial file: %s", err)
}
// Test that making a new File appends
f, err := NewFileWriter(fname)
if err != nil {
t.Fatalf("Unable to create %s", fname)
}
_, err = f.Write([]byte("line1\n"))
if err != nil {
t.Errorf("Got write error1: %s", err)
}
// Now move file
err = os.Rename(fname, fname+".orig")
if err != nil {
t.Errorf("Renaming error: %s", err)
}
_, err = f.Write([]byte("after1\n"))
if err != nil {
t.Errorf("Write error: %s", err)
}
// Test that reopen always appends
err = f.Reopen()
if err != nil {
t.Errorf("Got reopen error %s: %s", fname, err)
}
_, err = f.Write([]byte("line2\n"))
if err != nil {
t.Errorf("Got write error2 on %s: %s", fname, err)
}
err = f.Close()
if err != nil {
t.Errorf("Got closing error for %s: %s", fname, err)
}
out, err := ioutil.ReadFile(fname)
if err != nil {
t.Fatalf("Unable read in final file %s: %s", fname, err)
}
outstr := string(out)
if outstr != "line2\n" {
t.Errorf("Result was %s", outstr)
}
}

5
server/server.go
View file

@ -36,7 +36,6 @@ import (
"golang.org/x/sync/errgroup"
"github.com/client9/reopen"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/boltdb"
"github.com/pilosa/pilosa/v2/encoding/proto"
@ -430,12 +429,12 @@ func (m *Command) setupNetworking() error {
// setupLogger sets up the logger based on the configuration.
func (m *Command) setupLogger() error {
var f *reopen.FileWriter
var f *logger.FileWriter
var err error
if m.Config.LogPath == "" {
m.logOutput = m.Stderr
} else {
f, err = reopen.NewFileWriter(m.Config.LogPath)
f, err = logger.NewFileWriter(m.Config.LogPath)
if err != nil {
return errors.Wrap(err, "opening file")
}