From bc94c7bcdf1885529bf0294a0528d3083e01959a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 9 Dec 2020 08:43:23 -0600 Subject: [PATCH] Remove lmdb dependency and references, vendor Barrier --- barrier.go | 219 ++++++++++++++++++++++++++++ barrier_test.go | 90 ++++++++++++ cmd/demo-lmdb/lmdb.go | 267 ---------------------------------- cmd/demo-lmdb/vprint.go | 179 ----------------------- cmd/pilosa-keydump/keydump.go | 180 ----------------------- cmd/pilosa-keydump/vprint.go | 179 ----------------------- ctl/server.go | 2 +- go.mod | 4 +- go.sum | 2 - license.exceptions | 5 +- tournament.sh | 2 +- txfactory_internal_test.go | 8 +- 12 files changed, 319 insertions(+), 818 deletions(-) create mode 100644 barrier.go create mode 100644 barrier_test.go delete mode 100644 cmd/demo-lmdb/lmdb.go delete mode 100644 cmd/demo-lmdb/vprint.go delete mode 100644 cmd/pilosa-keydump/keydump.go delete mode 100644 cmd/pilosa-keydump/vprint.go diff --git a/barrier.go b/barrier.go new file mode 100644 index 000000000..b150dc486 --- /dev/null +++ b/barrier.go @@ -0,0 +1,219 @@ +// home https://github.com/glycerine/lmdb-go +// Copyright (c) 2020, the lmdb-go authors +// Copyright (c) 2015, Bryan Matsuo +// All rights reserved. + +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: + +// Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. + +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. + +// Neither the name of the author nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. + +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package pilosa + +import ( + "github.com/glycerine/idem" +) + +// Barrier allows us to temporarily halt all readers, so that +// a writer can commit alone and thus compact the db. +// The Barrier starts unblocked, alllowing passage to any +// caller of WaitAtGate(). +type Barrier struct { + wait chan *appointment // send upon entering the waiting room. + halt *idem.Halter + blockReqCh chan *blockReq + unblockCh chan *unblock +} + +type blockReq struct { + count int + done chan struct{} +} + +func newBlockReq(count int) *blockReq { + return &blockReq{ + count: count, + done: make(chan struct{}), + } +} + +type appointment struct { + id int + done chan struct{} +} + +func newAppointment(id int) *appointment { + return &appointment{ + id: id, + done: make(chan struct{}), + } +} + +// NewBarrier is either open, allowing immediate passage, +// or blocked, halting all callers at WaitAtGate() +// until the barrier is opened. By default it is open. +// +// Barrier.Close() must be called when the barrier +// is no longer needed to avoid a goroutine leak. +func NewBarrier() (b *Barrier) { + b = &Barrier{ + wait: make(chan *appointment), // waiters indicate they are waiting for the gate by sending here. + halt: idem.NewHalter(), + blockReqCh: make(chan *blockReq), + unblockCh: make(chan *unblock), + } + go func() { + defer b.halt.Done.Close() + + var waitlist []*appointment + var curBlockReq *blockReq + + for { + select { + case br := <-b.blockReqCh: + if br.count == 0 { + close(br.done) + continue + } + if curBlockReq == nil { + // good, changing state from open to closed barrier. + } else { + panic("got 2nd block request atop of first") + } + curBlockReq = br + //vv("barrier: request to block for %v waiters", br.count) + if len(waitlist) != 0 { + panic("had waiters when we were open, internal/client bug") + } + case appt := <-b.wait: + //vv("barrier.wait sees appt = '%#v' and curBlockReq = '%#v'", appt, curBlockReq) + if curBlockReq == nil { + close(appt.done) + continue + } + waitlist = append(waitlist, appt) + n := len(waitlist) + th := curBlockReq.count + if th < 0 { + // infinite waiters. we block everybody until we + // see an unblock request. + continue + } + if n >= th { + close(curBlockReq.done) + curBlockReq = nil + } + case ub := <-b.unblockCh: + for _, appt := range waitlist { + close(appt.done) + } + waitlist = nil + curBlockReq = nil + close(ub.done) + case <-b.halt.ReqStop.Chan: + return + } + } + }() + return +} + +// WaitAtGate will return immediately +// if the barrier is unblocked. Otherwise +// it will not return until another +// goroutine unblocks the barrier. +func (b *Barrier) WaitAtGate(id int) { + appt := newAppointment(id) + select { + case b.wait <- appt: + select { + case <-appt.done: + case <-b.halt.ReqStop.Chan: + } + case <-b.halt.ReqStop.Chan: + } +} + +// Close should be called to stop the +// barrier's background goroutine when +// you are done using the barrier. +func (b *Barrier) Close() { + b.halt.ReqStop.Close() + <-b.halt.Done.Chan +} + +type unblock struct { + done chan struct{} +} + +func newUnblock() *unblock { + return &unblock{ + done: make(chan struct{}), + } +} + +// Unblock lets all waiting goroutines resume execution. +func (b *Barrier) UnblockReaders() { + ub := newUnblock() + select { + case b.unblockCh <- ub: + select { + case <-ub.done: + case <-b.halt.ReqStop.Chan: + } + case <-b.halt.ReqStop.Chan: + } +} + +// BlockUntil is called with a count, the +// number of waiters required to be present and waiting +// at the gate before call returns. +// A count of < 0 will return immediately and raise +// the barrier to any number of arriving readers. +// A count of 0 is a no-op. +// +// Otherwise we raise the barrier +// and wait until we have seen count other goroutines waiting +// on it. +// +// We return without releasing the waiters. Call +// Open when you want them to resume. +func (b *Barrier) BlockUntil(count int) { + if count == 0 { + return + } + req := newBlockReq(count) + b.blockReqCh <- req + if count > 0 { + <-req.done + } +} + +// BlockAllReadersNoWait raises the barrier to +// an infinite number of waiters and returns immediately +// to the caller. +func (b *Barrier) BlockAllReadersNoWait() { + req := newBlockReq(-1) // -1 means block any number of readers. + b.blockReqCh <- req + // don't wait. <-req.done +} diff --git a/barrier_test.go b/barrier_test.go new file mode 100644 index 000000000..70fd4d311 --- /dev/null +++ b/barrier_test.go @@ -0,0 +1,90 @@ +// home https://github.com/glycerine/lmdb-go +// Copyright (c) 2020, the lmdb-go authors +// Copyright (c) 2015, Bryan Matsuo +// All rights reserved. + +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: + +// Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. + +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. + +// Neither the name of the author nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. + +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package pilosa + +import ( + "fmt" + "sync/atomic" + "testing" + "time" +) + +func TestBarrierHolds(t *testing.T) { + b := NewBarrier() + defer b.Close() + + released := int64(0) + + waiter := func(i int) { + b.WaitAtGate(i) + //vv("goro %v is released", i) + atomic.AddInt64(&released, 1) + } + + for i := 0; i < 3; i++ { + go waiter(i) + } + time.Sleep(time.Second) + r := atomic.SwapInt64(&released, 0) + if r != 3 { + panic("open barrier held back goro") + } + //vv("good: barrier started open") + + //seenAll := make(chan bool) + b.BlockAllReadersNoWait() + for i := 0; i < 3; i++ { + go waiter(i) + } + + time.Sleep(time.Second) + r = atomic.SwapInt64(&released, 0) + if r != 0 { + panic("bad: barrier did not hold back goro") + } + //vv("good: barrier of 4 did not release on 3") + go waiter(4) + + time.Sleep(time.Second) + r = atomic.SwapInt64(&released, 0) + if r != 0 { + panic(fmt.Sprintf("bad: barrier did not hold back goro, should wait for unblock. r = %v", r)) + } + + b.UnblockReaders() + + time.Sleep(time.Second) + r = atomic.SwapInt64(&released, 0) + if r != 4 { + panic("bad: unblock should have released 4 goro") + } + +} diff --git a/cmd/demo-lmdb/lmdb.go b/cmd/demo-lmdb/lmdb.go deleted file mode 100644 index bf91af113..000000000 --- a/cmd/demo-lmdb/lmdb.go +++ /dev/null @@ -1,267 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !386 - -package main - -import ( - "bytes" - "fmt" - "log" - "os" - - "github.com/glycerine/lmdb-go/lmdb" -) - -// note: use runtime.LockOSThread on any write goroutine; must create and write the txn from -// the same goroutine. - -// This example demonstrates a complete workflow for a simple application -// working with LMDB. First, an Env is configured and mapped to memory. Once -// mapped, database handles are opened and normal database operations may -// begin. -func main() { - // Create an environment and make sure it is eventually closed. - env, err := lmdb.NewEnv() - panicOn(err) - defer env.Close() - - // Configure and open the environment. Most configuration must be done - // before opening the environment. The go documentation for each method - // should indicate if it must be called before calling env.Open() - err = env.SetMaxDBs(1) - panicOn(err) - err = env.SetMapSize(1 << 30) - panicOn(err) - path := "./db-lmdb" - panicOn(os.MkdirAll(path, 0755)) - err = env.Open(path, 0, 0644) // lmdb.Create ? - panicOn(err) - - // In any real application it is important to check for readers that were - // never closed by their owning process, and for which the owning process - // has exited. See the documentation on transactions for more information. - staleReaders, err := env.ReaderCheck() - panicOn(err) - if staleReaders > 0 { - log.Printf("cleared %d reader slots from dead processes", staleReaders) - } - - // Open a database handle that will be used for the entire lifetime of this - // application. Because the database may not have existed before, and the - // database may need to be created, we need to get the database handle in - // an update transacation. - var dbi lmdb.DBI - _ = dbi - err = env.Update(func(txn *lmdb.Txn) (err error) { - dbi, err = txn.CreateDBI("example") - return err - }) - panicOn(err) - - // The database referenced by our DBI handle is now ready for the - // application to use. Here the application just opens a readonly - // transaction and reads the data stored in the "hello" key and prints its - // value to the application's standard output. - err = env.View(func(txn *lmdb.Txn) (err error) { - v, err := txn.Get(dbi, []byte("hello")) - if err != nil { - return err - } - fmt.Println(string(v)) - return nil - }) - _ = err - //panicOn(err) // mdb_get: MDB_NOTFOUND: No matching key/data pair found - - err = env.Update(func(txn *lmdb.Txn) (err error) { - panicOn(txn.Put(dbi, []byte("099"), []byte("A"), 0)) - panicOn(txn.Put(dbi, []byte("101"), []byte("B"), 0)) - panicOn(txn.Put(dbi, []byte("199"), []byte("C"), 0)) - panicOn(txn.Put(dbi, []byte("200"), []byte("D"), 0)) - panicOn(txn.Put(dbi, []byte("300"), []byte("E"), 0)) - panicOn(txn.Put(dbi, []byte("399"), []byte("F"), 0)) - return nil - }) - _ = err - - // find max in [000,100) and get 099 - // find max in [100,200) and get 199 - // find max in [300,400) and get 399 - // find max in [400,500) and get nothing back - err = env.View(func(txn *lmdb.Txn) (err error) { - v, err := txn.Get(dbi, []byte("hello")) - if err != nil { - return err - } - fmt.Printf("key 'hello' retreived value: '%v'\n", string(v)) - return nil - }) - _ = err - - err = env.View(func(txn *lmdb.Txn) (err error) { - cur, err := txn.OpenCursor(dbi) - panicOn(err) - defer cur.Close() - - var cur2 *lmdb.Cursor - var err2 error - var k, k2, v, v2 []byte - i := 0 - for { - if i == 0 { - // lmdb.SetRange : The first key no less than the specified key. - k, v, err = cur.Get([]byte("200"), nil, lmdb.SetRange) - - // cur2 should start at 'a' - cur2, err2 = txn.OpenCursor(dbi) - panicOn(err2) - defer cur2.Close() - k2, v2, err2 = cur2.Get(nil, nil, lmdb.Next) - _ = err2 - } else { - k, v, err = cur.Get(nil, nil, lmdb.Next) - k2, v2, err2 = cur2.Get(nil, nil, lmdb.Next) - _ = err2 - } - if lmdb.IsNotFound(err) { - return nil - } - if err != nil { - return err - } - - fmt.Printf("i=%v, %s %s\n", i, k, v) - fmt.Printf("i=%v, k2:%s v2:%s\n", i, k2, v2) - i++ - } - // return nil // unreachable - }) - _ = err - - // panicOn(txn.Put(dbi, []byte("099"), []byte("A"), 0)) - // panicOn(txn.Put(dbi, []byte("101"), []byte("B"), 0)) - // panicOn(txn.Put(dbi, []byte("199"), []byte("C"), 0)) - // panicOn(txn.Put(dbi, []byte("200"), []byte("D"), 0)) - // panicOn(txn.Put(dbi, []byte("300"), []byte("E"), 0)) - // panicOn(txn.Put(dbi, []byte("399"), []byte("F"), 0)) - // - // find max in [300,400) and get 399 - // find max in [000,100) and get 099 - // find max in [100,200) and get 199 - // find max in [400,500) and get nothing back - // find max in [201,300) and get nothing back - - err = env.View(func(txn *lmdb.Txn) (err error) { - cur, err := txn.OpenCursor(dbi) - panicOn(err) - defer cur.Close() - - var k, v []byte - - // find max in [300,400) and get 399 - - // lmdb.SetRange : The first key no less than the specified key. - k, v, err = cur.Get([]byte("400"), nil, lmdb.SetRange) - if lmdb.IsNotFound(err) { - fmt.Printf("400 not found, as expected\n") // happens on starting empty db - } else { - fmt.Printf("Get 400 => %v: %v\n", string(k), string(v)) - } - - k, v, err = cur.Get(nil, nil, lmdb.Prev) - if lmdb.IsNotFound(err) { - fmt.Printf("Get 400 then Get Prev => not found\n") - } else { - fmt.Printf("Get 400 then Get Prev => %v: %v\n", string(k), string(v)) // 399: F, so wraps backwards from beginning. - } - panicOn(err) - - // now try for 199 in [100,200) - - k, v, err = cur.Get([]byte("200"), nil, lmdb.SetRange) - if lmdb.IsNotFound(err) { - panic("200 not found, not expected") - } else { - fmt.Printf("Get 200 => %v: %v\n", string(k), string(v)) // Get 200 => 200: D - } - - k, v, err = cur.Get(nil, nil, lmdb.Prev) - if lmdb.IsNotFound(err) { - fmt.Printf("Get 200 then Get Prev => not found\n") - } else { - fmt.Printf("Get 200 then Get Prev => %v: %v\n", string(k), string(v)) // Get 200 then Get Prev => 199: C - } - panicOn(err) - - k, v, err = cur.Get([]byte("500"), nil, lmdb.SetRange) - if lmdb.IsNotFound(err) { - fmt.Printf("500 not found, as expected\n") // 500 not found, as expected - } else { - panic(fmt.Sprintf("Get 500 => %v: %v\n", string(k), string(v))) - } - - k, v, err = cur.Get(nil, nil, lmdb.Prev) - if lmdb.IsNotFound(err) { - fmt.Printf("Get 500 then Get Prev => not found\n") - } else { - fmt.Printf("Get 500 then Get Prev => %v: %v\n", string(k), string(v)) // Get 500 then Get Prev => 399: F - } - panicOn(err) - - k, v, err = cur.Get([]byte("100"), nil, lmdb.SetRange) - if lmdb.IsNotFound(err) { - panic("100 not found, not expected") - } else { - fmt.Printf("Get 100 => %v: %v\n", string(k), string(v)) // Get 100 => 101: B - } - - k, v, err = cur.Get(nil, nil, lmdb.Prev) - if lmdb.IsNotFound(err) { - fmt.Printf("Get 100 then Get Prev => not found\n") - } else { - fmt.Printf("Get 100 then Get Prev => %v: %v\n", string(k), string(v)) // Get 100 then Get Prev => 099: A - } - panicOn(err) - - // find max in [201,300) and get nothing back - - k, v, err = cur.Get([]byte("300"), nil, lmdb.SetRange) - if lmdb.IsNotFound(err) { - panic("300 not found, not expected") - } else { - fmt.Printf("Get 300 => %v: %v\n", string(k), string(v)) // Get 300 => 300: E - } - - k, v, err = cur.Get(nil, nil, lmdb.Prev) - if lmdb.IsNotFound(err) { - fmt.Printf("Get 300 then Get Prev => not found\n") - } else { - fmt.Printf("Get 300 then Get Prev => %v: %v\n", string(k), string(v)) // Get 300 then Get Prev => 200: D - } - cmp := bytes.Compare(k, []byte("201")) - if cmp >= 0 { - fmt.Printf("key k = '%v' was >= 201", string(k)) - } else { - fmt.Printf("key k = '%v' was < 201", string(k)) // key k = '200' was < 201 - } - panicOn(err) - - return nil - }) - panicOn(err) - - vv("done") -} diff --git a/cmd/demo-lmdb/vprint.go b/cmd/demo-lmdb/vprint.go deleted file mode 100644 index 2cfa2a283..000000000 --- a/cmd/demo-lmdb/vprint.go +++ /dev/null @@ -1,179 +0,0 @@ -// home: https://github.com/glycerine/vprint -// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved. -// License: MIT -// -// MIT License -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -// +build !386 - -package main - -import ( - "fmt" - "io" - "os" - "path" - "runtime" - "runtime/debug" - "sync" - "time" -) - -const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00" -const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00" - -// for tons of debug output -var VerboseVerbose bool = false - -// convience functions for . import -var pp = PP -var vv = VV - -var panicOn = PanicOn - -func init() { - // keeper linter happy - _ = pp - _ = vv -} - -func PanicOn(err error) { - if err != nil { - panic(err) - } -} - -func PP(format string, a ...interface{}) { - if VerboseVerbose { - TSPrintf(format, a...) - } -} - -func VV(format string, a ...interface{}) { - TSPrintf(format, a...) -} - -func AlwaysPrintf(format string, a ...interface{}) { - TSPrintf(format, a...) -} - -var tsPrintfMut sync.Mutex - -// time-stamped printf -func TSPrintf(format string, a ...interface{}) { - tsPrintfMut.Lock() - Printf("\n%s %s ", FileLine(3), ts()) - Printf(format+"\n", a...) - tsPrintfMut.Unlock() -} - -// get timestamp for logging purposes -func ts() string { - return time.Now().Format(RFC3339UsecTz0) -} - -// so we can multi write easily, use our own printf -var OurStdout io.Writer = os.Stdout - -// Printf formats according to a format specifier and writes to standard output. -// It returns the number of bytes written and any write error encountered. -func Printf(format string, a ...interface{}) (n int, err error) { - return fmt.Fprintf(OurStdout, format, a...) -} - -func FileLine(depth int) string { - _, fileName, fileLine, ok := runtime.Caller(depth) - var s string - if ok { - s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine) - } else { - s = "" - } - return s -} - -func stack() string { - return string(debug.Stack()) -} - -func FileExists(name string) bool { - fi, err := os.Stat(name) - if err != nil { - return false - } - if fi.IsDir() { - return false - } - return true -} - -func DirExists(name string) bool { - fi, err := os.Stat(name) - if err != nil { - return false - } - if fi.IsDir() { - return true - } - return false -} - -func FileSize(name string) (int64, error) { - fi, err := os.Stat(name) - if err != nil { - return -1, err - } - return fi.Size(), nil -} - -// Caller returns the name of the calling function. -func Caller(upStack int) string { - // elide ourself and runtime.Callers - target := upStack + 2 - - pc := make([]uintptr, target+2) - n := runtime.Callers(0, pc) - - f := runtime.Frame{Function: "unknown"} - if n > 0 { - frames := runtime.CallersFrames(pc[:n]) - for i := 0; i <= target; i++ { - contender, more := frames.Next() - if i == target { - f = contender - } - if !more { - break - } - } - } - return f.Function -} - -// happy linter: -var _ = DirExists -var _ = FileExists -var _ = Caller -var _ = stack -var _ = RFC3339MsecTz0 -var _ = RFC3339UsecTz0 -var _ = AlwaysPrintf -var _ = FileSize diff --git a/cmd/pilosa-keydump/keydump.go b/cmd/pilosa-keydump/keydump.go deleted file mode 100644 index 3cd759a33..000000000 --- a/cmd/pilosa-keydump/keydump.go +++ /dev/null @@ -1,180 +0,0 @@ -// home https://github.com/glycerine/lmdb-go -// Copyright (c) 2020, the lmdb-go authors -// Copyright (c) 2015, Bryan Matsuo -// All rights reserved. - -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: - -// Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. - -// Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. - -// Neither the name of the author nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. - -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// +build amd64 - -package main - -import ( - "fmt" - "os" - "runtime" - - "github.com/glycerine/lmdb-go/lmdb" - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/hash" - "github.com/pilosa/pilosa/v2/roaring" - "github.com/pilosa/pilosa/v2/txkey" -) - -// pilosa-keydump is a diagnostic tool that simply prints all the -// database keys in the database directory path specified -// as the first argument on the command line. -func main() { - runtime.LockOSThread() - defer runtime.UnlockOSThread() - - if len(os.Args) < 2 { - fmt.Fprintf(os.Stderr, "must supply path to database directory as only arg\n") - os.Exit(1) - } - - path := os.Args[1] - if !DirExists(path) { - fmt.Fprintf(os.Stderr, "directory path '%v' does not exist.\n", path) - os.Exit(1) - } - - maxr := 1 - env, err := lmdb.NewEnvMaxReaders(maxr) - panicOn(err) - defer env.Close() - - panicOn(env.SetMapSize(256 << 30)) - - err = env.SetMaxDBs(10) - panicOn(err) - - //var myflags uint = NoReadahead | NoSubdir - var myflags uint = 0 //lmdb.NoSubdir - err = env.Open(path, myflags, 0664) - panicOn(err) - - // In any real application it is important to check for readers that were - // never closed by their owning process, and for which the owning process - // has exited. See the documentation on transactions for more information. - staleReaders, err := env.ReaderCheck() - panicOn(err) - if staleReaders > 0 { - vv("cleared %d reader slots from dead processes", staleReaders) - } - - dbnames := []string{} - - var dbiRoot lmdb.DBI - var dbi lmdb.DBI - env.UseSphynxReader() - err = env.SphynxReader(func(txn *lmdb.Txn, readslot int) (err error) { - //txn.RawRead = true - - dbiRoot, err = txn.OpenRoot(0) - panicOn(err) - - cur, err := txn.OpenCursor(dbiRoot) - panicOn(err) - defer cur.Close() - - for i := 0; true; i++ { - var k, v []byte - var err error - if i == 0 { - // must give it at least a zero byte here to start. - k, v, err = cur.Get([]byte{0}, nil, lmdb.SetRange) - panicOn(err) - } else { - k, v, err = cur.Get([]byte(nil), nil, lmdb.Next) - if lmdb.IsNotFound(err) { - break - } else { - panicOn(err) - } - } - dbnames = append(dbnames, string(k)) - _ = v - } - cur.Close() - return - }) - panicOn(err) - - for _, dbn := range dbnames { - fmt.Printf(` -========================= -database '%v': -========================= - -`, dbn) - err = env.SphynxReader(func(txn *lmdb.Txn, readslot int) (err error) { - //txn.RawRead = true - - dbi, err = txn.OpenDBI(dbn, 0) - panicOn(err) - - cur, err := txn.OpenCursor(dbi) - panicOn(err) - defer cur.Close() - - for i := 0; true; i++ { - var k, v []byte - var err error - if i == 0 { - // must give it at least a zero byte here to start. - k, v, err = cur.Get([]byte{0}, nil, lmdb.SetRange) - panicOn(err) - } else { - k, v, err = cur.Get([]byte(nil), nil, lmdb.Next) - if lmdb.IsNotFound(err) { - break - } else { - panicOn(err) - } - } - - ckey := txkey.KeyExtractContainerKey(k) - - n := len(v) - - hash := hash.Blake3sum16(v[0:(n - 1)]) - ct := pilosa.ToContainer(v[n-1], v[0:(n-1)]) - cts := roaring.NewSliceContainers() - cts.Put(ckey, ct) - rbm := &roaring.Bitmap{Containers: cts} - srbm := pilosa.BitmapAsString(rbm) - - fmt.Printf("%04v %v -> %v (%v hot)\n", i, txkey.ToString(k), hash, ct.N()) - fmt.Printf(" .......%v\n", srbm) - } - return - }) - panicOn(err) - } // for dbnames - - fmt.Printf("=================== done.\n") -} diff --git a/cmd/pilosa-keydump/vprint.go b/cmd/pilosa-keydump/vprint.go deleted file mode 100644 index e7adc54cc..000000000 --- a/cmd/pilosa-keydump/vprint.go +++ /dev/null @@ -1,179 +0,0 @@ -// home: https://github.com/glycerine/vprint -// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved. -// License: MIT -// -// MIT License -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -// +build amd64 - -package main - -import ( - "fmt" - "io" - "os" - "path" - "runtime" - "runtime/debug" - "sync" - "time" -) - -const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00" -const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00" - -// for tons of debug output -var VerboseVerbose bool = false - -// convience functions for . import -var pp = PP -var vv = VV - -var panicOn = PanicOn - -func init() { - // keeper linter happy - _ = pp - _ = vv -} - -func PanicOn(err error) { - if err != nil { - panic(err) - } -} - -func PP(format string, a ...interface{}) { - if VerboseVerbose { - TSPrintf(format, a...) - } -} - -func VV(format string, a ...interface{}) { - TSPrintf(format, a...) -} - -func AlwaysPrintf(format string, a ...interface{}) { - TSPrintf(format, a...) -} - -var tsPrintfMut sync.Mutex - -// time-stamped printf -func TSPrintf(format string, a ...interface{}) { - tsPrintfMut.Lock() - Printf("\n%s %s ", FileLine(3), ts()) - Printf(format+"\n", a...) - tsPrintfMut.Unlock() -} - -// get timestamp for logging purposes -func ts() string { - return time.Now().Format(RFC3339UsecTz0) -} - -// so we can multi write easily, use our own printf -var OurStdout io.Writer = os.Stdout - -// Printf formats according to a format specifier and writes to standard output. -// It returns the number of bytes written and any write error encountered. -func Printf(format string, a ...interface{}) (n int, err error) { - return fmt.Fprintf(OurStdout, format, a...) -} - -func FileLine(depth int) string { - _, fileName, fileLine, ok := runtime.Caller(depth) - var s string - if ok { - s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine) - } else { - s = "" - } - return s -} - -func stack() string { - return string(debug.Stack()) -} - -func FileExists(name string) bool { - fi, err := os.Stat(name) - if err != nil { - return false - } - if fi.IsDir() { - return false - } - return true -} - -func DirExists(name string) bool { - fi, err := os.Stat(name) - if err != nil { - return false - } - if fi.IsDir() { - return true - } - return false -} - -func FileSize(name string) (int64, error) { - fi, err := os.Stat(name) - if err != nil { - return -1, err - } - return fi.Size(), nil -} - -// Caller returns the name of the calling function. -func Caller(upStack int) string { - // elide ourself and runtime.Callers - target := upStack + 2 - - pc := make([]uintptr, target+2) - n := runtime.Callers(0, pc) - - f := runtime.Frame{Function: "unknown"} - if n > 0 { - frames := runtime.CallersFrames(pc[:n]) - for i := 0; i <= target; i++ { - contender, more := frames.Next() - if i == target { - f = contender - } - if !more { - break - } - } - } - return f.Function -} - -// happy linter: -var _ = DirExists -var _ = FileExists -var _ = Caller -var _ = stack -var _ = RFC3339MsecTz0 -var _ = RFC3339UsecTz0 -var _ = AlwaysPrintf -var _ = FileSize diff --git a/ctl/server.go b/ctl/server.go index 8fd828d2d..a55bca6ca 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -93,7 +93,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // Transactional storage engine // Note: the default for --tx must be kept "" empty string. Otherwise we // cannot detect and honor the PILOSA_TXSRC env var over-ride. - flags.StringVarP(&srv.Config.Txsrc, "txsrc", "", "", fmt.Sprintf("transaction/storage to use: one of roaring, rbf, bolt, lmdb, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc. The default is: %v. The env var PILOSA_TXSRC is over-ridden by --tx option on the command line.", pilosa.DefaultTxsrc)) + flags.StringVarP(&srv.Config.Txsrc, "txsrc", "", "", fmt.Sprintf("transaction/storage to use: one of roaring, rbf, bolt, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc. The default is: %v. The env var PILOSA_TXSRC is over-ridden by --txsrc option on the command line.", pilosa.DefaultTxsrc)) // RowcacheOn flags.BoolVarP((&srv.Config.RowcacheOn), "rowcache-on", "", srv.Config.RowcacheOn, "turn on the rowcache for all backends (may speed some queries)") diff --git a/go.mod b/go.mod index 9f4728a40..16b3f2ead 100644 --- a/go.mod +++ b/go.mod @@ -12,16 +12,18 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dustin/go-humanize v1.0.0 + github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 - github.com/glycerine/lmdb-go v1.9.34 github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.1 github.com/golang/protobuf v1.3.3 github.com/google/go-cmp v0.4.0 + github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.0 github.com/hashicorp/memberlist v0.1.3 github.com/improbable-eng/grpc-web v0.13.0 + github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/lib/pq v1.8.0 github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b github.com/opentracing/opentracing-go v1.1.0 diff --git a/go.sum b/go.sum index b4de84573..39abeebda 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,6 @@ github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E= github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y= -github.com/glycerine/lmdb-go v1.9.34 h1:0lymJjpdelYnIMcNzsKROfIaApt99zhaHtjDJTHjGkE= -github.com/glycerine/lmdb-go v1.9.34/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= diff --git a/license.exceptions b/license.exceptions index dc5b5bebe..4d0dc19dc 100644 --- a/license.exceptions +++ b/license.exceptions @@ -16,12 +16,11 @@ ./rbf/page_map.go ./cmd/slurp/vprint.go ./cmd/badloader/vprint.go -./cmd/demo-lmdb/vprint.go ./gid.go -./cmd/pilosa-keydump/vprint.go -./cmd/pilosa-keydump/keydump.go ./synthload/vprint.go ./proto/vdsm/vdsm.proto ./proto/vdsm/vdsm.pb.go ./cmd/pilosa-fsck/vprint.go ./cmd/random-query/vprint.go +./barrier.go +./barrier_test.go diff --git a/tournament.sh b/tournament.sh index af78ec798..793ef7231 100755 --- a/tournament.sh +++ b/tournament.sh @@ -5,7 +5,7 @@ ## one or two backends through the rigors of make testv-race. ## logs are saved to the tourna.log.${i} files. -for i in rbf lmdb roaring bolt rbf_lmdb rbf_roaring lmdb_rbf lmdb_roaring roaring_rbf roaring_lmdb roaring_bolt lmdb_bolt; do +for i in rbf roaring bolt rbf_roaring roaring_rbf roaring_bolt; do echo "$(date) starting ${i}, output to tourna.log.${i}" echo "***=== ${i} ====================*** $(date)" &> tourna.log.${i} PILOSA_TXSRC=${i} make testv-race 2>&1 > tourna.log.${i} diff --git a/txfactory_internal_test.go b/txfactory_internal_test.go index 2f4b62ee3..7a71243d4 100644 --- a/txfactory_internal_test.go +++ b/txfactory_internal_test.go @@ -20,16 +20,14 @@ import ( "os" "testing" "time" - - "github.com/glycerine/lmdb-go/lmdb" ) func Test_TxFactory_Qcx_query_context(t *testing.T) { src := os.Getenv("PILOSA_TXSRC") - if src == "rbf" || src == "lmdb" || src == "bolt" { + if src == "rbf" || src == "bolt" { // ok } else { - t.Skip("this test only for lmdb and rbf and bolt") + t.Skip("this test only for rbf and bolt") } shard := uint64(0) @@ -37,7 +35,7 @@ func Test_TxFactory_Qcx_query_context(t *testing.T) { defer f.Clean(t) tx.Rollback() - barrier := lmdb.NewBarrier() + barrier := NewBarrier() defer barrier.Close() done := make(chan bool)