diff --git a/cmd/pilosa-chk/chk.go b/cmd/pilosa-chk/chk.go index 006d3c563..f0d5d9258 100644 --- a/cmd/pilosa-chk/chk.go +++ b/cmd/pilosa-chk/chk.go @@ -22,6 +22,7 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/boltdb" + "github.com/pilosa/pilosa/v2/hash" "github.com/zeebo/blake3" ) @@ -55,7 +56,7 @@ func main() { fmt.Printf("opening dir '%v'... this may take a few seconds...\n", dir) if dirChecksum { - fmt.Printf("path '%v' has dirhash %v\n", dir, pilosa.HashOfDir(dir)) + fmt.Printf("path '%v' has dirhash %v\n", dir, hash.HashOfDir(dir)) return } diff --git a/cmd/pilosa-keydump/keydump.go b/cmd/pilosa-keydump/keydump.go index f82782aa7..3cd759a33 100644 --- a/cmd/pilosa-keydump/keydump.go +++ b/cmd/pilosa-keydump/keydump.go @@ -39,6 +39,7 @@ import ( "github.com/glycerine/lmdb-go/lmdb" "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/hash" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/txkey" ) @@ -160,7 +161,7 @@ database '%v': n := len(v) - hash := pilosa.Blake3sum16(v[0:(n - 1)]) + hash := hash.Blake3sum16(v[0:(n - 1)]) ct := pilosa.ToContainer(v[n-1], v[0:(n-1)]) cts := roaring.NewSliceContainers() cts.Put(ckey, ct) diff --git a/blake3.go b/hash/blake3.go similarity index 91% rename from blake3.go rename to hash/blake3.go index 78cbdfe8d..9cc4ddd3a 100644 --- a/blake3.go +++ b/hash/blake3.go @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa +package hash import ( + cryptorand "crypto/rand" "encoding/binary" "fmt" "sync" - cryptorand "crypto/rand" "github.com/zeebo/blake3" "golang.org/x/mod/sumdb/dirhash" ) @@ -70,7 +70,7 @@ func (w *Blake3Hasher) CryptoHash(input []byte, buffer []byte) (outputCryptohash return buffer } -// blake3sum16 might be slower because we allocate a new hasher every time, but +// Blake3sum16 might be slower because we allocate a new hasher every time, but // it is more conenient for writing debug code. It returns // a 16 byte hash as a hexidecimal string. func Blake3sum16(input []byte) string { @@ -83,8 +83,8 @@ func Blake3sum16(input []byte) string { return fmt.Sprintf("%x", buf) } -// cryptoRandInt64 uses crypto/rand to get an random int64 -func cryptoRandInt64() int64 { +// CryptoRandInt64 uses crypto/rand to get an random int64 +func CryptoRandInt64() int64 { c := 8 b := make([]byte, c) _, err := cryptorand.Read(b) @@ -95,9 +95,13 @@ func cryptoRandInt64() int64 { return r } +// HashOfDir returns the hash of the local file system directory dir func HashOfDir(path string) string { prefix := "" h, err := dirhash.HashDir(path, prefix, dirhash.Hash1) - panicOn(err) + if err != nil { + panic(err) + } + return h } diff --git a/blake3_test.go b/hash/blake3_test.go similarity index 71% rename from blake3_test.go rename to hash/blake3_test.go index 76cc0156f..af2342ea3 100644 --- a/blake3_test.go +++ b/hash/blake3_test.go @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa +package hash import ( + "encoding/hex" "fmt" "io/ioutil" "os" + "path" "testing" - "encoding/hex" - "github.com/pilosa/pilosa/v2/testhook" ) @@ -44,7 +44,7 @@ func TestBlake3Hasher(t *testing.T) { } func TestCryptoRandInt64(t *testing.T) { - rnd := cryptoRandInt64() + rnd := CryptoRandInt64() if rnd == 0 { panic("cryptoRandInt64() gave 0, very high odds it has broken") } @@ -52,21 +52,39 @@ func TestCryptoRandInt64(t *testing.T) { func TestHashOfDir(t *testing.T) { dir, err := testhook.TempDir(t, "TestHashOfDir-dir") - panicOn(err) - b := dir + sep + "A" + sep + "B" - c := dir + sep + "A" + sep + "C" - panicOn(os.MkdirAll(b, 0755)) - panicOn(os.MkdirAll(c, 0755)) + if err != nil { + t.Fatal(err) + } + + b := path.Join(dir, "A", "B") + if err := os.MkdirAll(b, 0755); err != nil { + t.Fatal(err) + } + + c := path.Join(dir, "A", "C") + if err := os.MkdirAll(c, 0755); err != nil { + t.Fatal(err) + } + bmessage := []byte("hello B\n") - panicOn(ioutil.WriteFile(b+sep+"b_content", bmessage, 0644)) + if err := ioutil.WriteFile(path.Join(b, "b_content"), bmessage, 0644); err != nil { + t.Fatal(err) + } + cmessage := []byte("hello C\n") - panicOn(ioutil.WriteFile(c+sep+"c_content", cmessage, 0644)) + if err := ioutil.WriteFile(path.Join(c, "c_content"), cmessage, 0644); err != nil { + t.Fatal(err) + } + hsh := HashOfDir(dir) c2message := []byte("hello C2\n") - panicOn(ioutil.WriteFile(c+sep+"c_content", c2message, 0644)) + if err := ioutil.WriteFile(path.Join(c, "c_content"), c2message, 0644); err != nil { + t.Fatal(err) + } + hsh2 := HashOfDir(dir) if hsh2 == hsh { - panic("HashOfDir did not detect 1 byte change") + t.Fatal("HashOfDir did not detect 1 byte change") } } diff --git a/index.go b/index.go index 13be604c0..726dca7c7 100644 --- a/index.go +++ b/index.go @@ -27,6 +27,7 @@ import ( "time" "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/v2/hash" "github.com/pilosa/pilosa/v2/internal" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/stats" @@ -791,7 +792,7 @@ func (i *Index) ComputeTranslatorSummary(verbose bool) (ats *AllTranslatorSummar } sum.Field = fld.name sum.Index = i.Name() - sum.Checksum = Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, i.Name()))) + sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, i.Name()))) if verbose { fmt.Printf("row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name) @@ -813,7 +814,7 @@ func (i *Index) ComputeTranslatorSummary(verbose bool) (ats *AllTranslatorSummar sum.PartitionID = partitionID sum.Index = i.Name() - sum.Checksum = Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, i.Name()))) + sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, i.Name()))) if verbose { fmt.Printf("col blake3-%v keyN: %10v idN: %10v paritionID: %03v \n", sum.Checksum, sum.KeyCount, sum.IDCount, partitionID) } diff --git a/lmdb.go b/lmdb.go index d740d971b..f652b6d7e 100644 --- a/lmdb.go +++ b/lmdb.go @@ -32,6 +32,7 @@ import ( "sync/atomic" "github.com/glycerine/lmdb-go/lmdb" + "github.com/pilosa/pilosa/v2/hash" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/txkey" "github.com/pkg/errors" @@ -1531,24 +1532,24 @@ func stringifiedLMDBKeysTx(tx *LMDBTx) (r string) { bkey := it.lastKey key := txkey.ToString(bkey) ckey := txkey.KeyExtractContainerKey(bkey) - hash := "" + h := "" srbm := "" v := it.lastVal n := len(v) if n == 0 { panic("should not have empty v here") } - hash = Blake3sum16(v[0:(n - 1)]) + h = hash.Blake3sum16(v[0:(n - 1)]) ct := tx.toContainer(v[n-1], v[0:(n-1)]) cts := roaring.NewSliceContainers() cts.Put(ckey, ct) rbm := &roaring.Bitmap{Containers: cts} srbm = BitmapAsString(rbm) - r += fmt.Sprintf("%v -> %v (%v hot)\n", key, hash, tx.countBitsSet(bkey)) + r += fmt.Sprintf("%v -> %v (%v hot)\n", key, h, tx.countBitsSet(bkey)) r += " ......." + srbm + "\n" } - r += "]\n all-in-blake3:" + Blake3sum16([]byte(r)) + r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) if !any { return "" diff --git a/rbf/blake3.go b/rbf/blake3.go deleted file mode 100644 index 3935d2576..000000000 --- a/rbf/blake3.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rbf - -import ( - "encoding/binary" - "fmt" - "sync" - - cryptorand "crypto/rand" - "github.com/zeebo/blake3" -) - -// Blake3Hasher is a thread/goroutine safe way to -// obtain a blake3 cryptographic hash of input []byte. -// Reference https://github.com/BLAKE3-team/BLAKE3 -// suggests it is 6x faster than BLAKE2B. -// The Go github.com/zeebo/blake3 version is -// AVX2 and SSE4.1 accelerated. -type Blake3Hasher struct { - hasher *blake3.Hasher - hasherMu sync.Mutex -} - -// NewBlake3Hasher returns a new Blake3Hasher. -func NewBlake3Hasher() *Blake3Hasher { - return &Blake3Hasher{ - hasher: blake3.New(), - } -} - -// CryptoHash writes the blake3 cryptographic hash of -// input into buffer and returns it. -// Like the standard libary's hash.Hash interface's Sum() method, -// the buffer is re-used and overwritten -// to avoid allocation. The caller determines the byte length of -// the outputCryptohash by the size of the supplied buffer -// slice, and this will be exactly equal to the supplies bytes. -// In this way, shorter or longer hashes can be provided as -// needed. -func (w *Blake3Hasher) CryptoHash(input []byte, buffer []byte) (outputCryptohash []byte) { - w.hasherMu.Lock() - w.hasher.Reset() - - // "Write implements part of the hash.Hash interface. It never returns an error." - // -- https://godoc.org/github.com/zeebo/blake3#Hasher.Write - _, _ = w.hasher.Write(input) - - // Digest.Read reads data from the hasher into buffer. - // "It always fills the entire buffer and never errors." - // -- https://godoc.org/github.com/zeebo/blake3#Digest - _, _ = w.hasher.Digest().Read(buffer) - - // no chance of panic, so avoid any defer cost. - w.hasherMu.Unlock() - - return buffer -} - -// blake3sum16 might be slower because we allocate a new hasher every time, but -// it is more conenient for writing debug code. It returns -// a 16 byte hash as a hexidecimal string. -func blake3sum16(input []byte) string { - hasher := blake3.New() - - _, _ = hasher.Write(input) - var buf [16]byte - _, _ = hasher.Digest().Read(buf[0:]) - - return fmt.Sprintf("%x", buf) -} - -// cryptoRandInt64 uses crypto/rand to get an random int64 -func cryptoRandInt64() int64 { - c := 8 - b := make([]byte, c) - _, err := cryptorand.Read(b) - if err != nil { - panic(err) - } - r := int64(binary.LittleEndian.Uint64(b)) - return r -} - -var _ = cryptoRandInt64 // happy linter diff --git a/rbf/tx.go b/rbf/tx.go index 9b7e69111..d1d4f1ad4 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -18,11 +18,13 @@ import ( "io" "math" "sort" + //"strconv" "strings" "sync" "github.com/benbjohnson/immutable" + "github.com/pilosa/pilosa/v2/hash" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/txkey" ) @@ -1372,7 +1374,7 @@ func (tx *Tx) DumpString() (r string) { return "" } // note that we can have a bitmap present, but it can be empty - r += "]\n all-in-blake3:" + blake3sum16([]byte(r)) + "\n" + r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + "\n" return "rbf-" + r } @@ -1420,7 +1422,7 @@ func bitmapAsString(rbm *roaring.Bitmap) (r string) { func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string) (s string) { by := containerToBytes(ct) - hash := blake3sum16(by) + hash := hash.Blake3sum16(by) cts := roaring.NewSliceContainers() cts.Put(ckey, ct) diff --git a/txfactory.go b/txfactory.go index 53d063a5e..6eddbe19e 100644 --- a/txfactory.go +++ b/txfactory.go @@ -25,6 +25,7 @@ import ( "syscall" "text/tabwriter" + "github.com/pilosa/pilosa/v2/hash" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/txkey" "github.com/pkg/errors" @@ -793,7 +794,7 @@ func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool) (r string) { return "" // new convention that empty database => empty string returned. } // note that we can have a bitmap present, but it can be empty - r += "]\n all-in-blake3:" + Blake3sum16([]byte(r)) + "\n" + r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + "\n" return "roaring-" + r } @@ -871,7 +872,7 @@ func stringifiedRawRoaringFragment(path string, index, field, view string, shard for citer.Next() { ckey, ct := citer.Value() by := containerToBytes(ct) - hash := Blake3sum16(by) + hash := hash.Blake3sum16(by) cts := roaring.NewSliceContainers() cts.Put(ckey, ct)