scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-10 23:11:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #782 from ajnavarro/remove-blake3-duplicated-code

Browse source
This commit is contained in:
Antonio Navarro Perez 2020-09-11 11:23:25 +02:00 • committed by GitHub
commit a68ee73f00
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 60 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cmd/pilosa-chk/chk.go
View file

@ -22,6 +22,7 @@ import (
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/boltdb"
"github.com/pilosa/pilosa/v2/hash"
"github.com/zeebo/blake3"
)
@ -55,7 +56,7 @@ func main() {
fmt.Printf("opening dir '%v'... this may take a few seconds...\n", dir)
if dirChecksum {
fmt.Printf("path '%v' has dirhash %v\n", dir, pilosa.HashOfDir(dir))
fmt.Printf("path '%v' has dirhash %v\n", dir, hash.HashOfDir(dir))
return
}

3
cmd/pilosa-keydump/keydump.go
View file

@ -39,6 +39,7 @@ import (
"github.com/glycerine/lmdb-go/lmdb"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/hash"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txkey"
)
@ -160,7 +161,7 @@ database '%v':
n := len(v)
hash := pilosa.Blake3sum16(v[0:(n - 1)])
hash := hash.Blake3sum16(v[0:(n - 1)])
ct := pilosa.ToContainer(v[n-1], v[0:(n-1)])
cts := roaring.NewSliceContainers()
cts.Put(ckey, ct)

16
blake3.go → hash/blake3.go
View file

@ -12,14 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
package hash
import (
cryptorand "crypto/rand"
"encoding/binary"
"fmt"
"sync"
cryptorand "crypto/rand"
"github.com/zeebo/blake3"
"golang.org/x/mod/sumdb/dirhash"
)
@ -70,7 +70,7 @@ func (w *Blake3Hasher) CryptoHash(input []byte, buffer []byte) (outputCryptohash
return buffer
}
// blake3sum16 might be slower because we allocate a new hasher every time, but
// Blake3sum16 might be slower because we allocate a new hasher every time, but
// it is more conenient for writing debug code. It returns
// a 16 byte hash as a hexidecimal string.
func Blake3sum16(input []byte) string {
@ -83,8 +83,8 @@ func Blake3sum16(input []byte) string {
return fmt.Sprintf("%x", buf)
}
// cryptoRandInt64 uses crypto/rand to get an random int64
func cryptoRandInt64() int64 {
// CryptoRandInt64 uses crypto/rand to get an random int64
func CryptoRandInt64() int64 {
c := 8
b := make([]byte, c)
_, err := cryptorand.Read(b)
@ -95,9 +95,13 @@ func cryptoRandInt64() int64 {
return r
}
// HashOfDir returns the hash of the local file system directory dir
func HashOfDir(path string) string {
prefix := ""
h, err := dirhash.HashDir(path, prefix, dirhash.Hash1)
panicOn(err)
if err != nil {
panic(err)
}
return h
}

44
blake3_test.go → hash/blake3_test.go
View file

@ -12,16 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
package hash
import (
"encoding/hex"
"fmt"
"io/ioutil"
"os"
"path"
"testing"
"encoding/hex"
"github.com/pilosa/pilosa/v2/testhook"
)
@ -44,7 +44,7 @@ func TestBlake3Hasher(t *testing.T) {
}
func TestCryptoRandInt64(t *testing.T) {
rnd := cryptoRandInt64()
rnd := CryptoRandInt64()
if rnd == 0 {
panic("cryptoRandInt64() gave 0, very high odds it has broken")
}
@ -52,21 +52,39 @@ func TestCryptoRandInt64(t *testing.T) {
func TestHashOfDir(t *testing.T) {
dir, err := testhook.TempDir(t, "TestHashOfDir-dir")
panicOn(err)
b := dir + sep + "A" + sep + "B"
c := dir + sep + "A" + sep + "C"
panicOn(os.MkdirAll(b, 0755))
panicOn(os.MkdirAll(c, 0755))
if err != nil {
t.Fatal(err)
}
b := path.Join(dir, "A", "B")
if err := os.MkdirAll(b, 0755); err != nil {
t.Fatal(err)
}
c := path.Join(dir, "A", "C")
if err := os.MkdirAll(c, 0755); err != nil {
t.Fatal(err)
}
bmessage := []byte("hello B\n")
panicOn(ioutil.WriteFile(b+sep+"b_content", bmessage, 0644))
if err := ioutil.WriteFile(path.Join(b, "b_content"), bmessage, 0644); err != nil {
t.Fatal(err)
}
cmessage := []byte("hello C\n")
panicOn(ioutil.WriteFile(c+sep+"c_content", cmessage, 0644))
if err := ioutil.WriteFile(path.Join(c, "c_content"), cmessage, 0644); err != nil {
t.Fatal(err)
}
hsh := HashOfDir(dir)
c2message := []byte("hello C2\n")
panicOn(ioutil.WriteFile(c+sep+"c_content", c2message, 0644))
if err := ioutil.WriteFile(path.Join(c, "c_content"), c2message, 0644); err != nil {
t.Fatal(err)
}
hsh2 := HashOfDir(dir)
if hsh2 == hsh {
panic("HashOfDir did not detect 1 byte change")
t.Fatal("HashOfDir did not detect 1 byte change")
}
}

5
index.go
View file

@ -27,6 +27,7 @@ import (
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/v2/hash"
"github.com/pilosa/pilosa/v2/internal"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
@ -791,7 +792,7 @@ func (i *Index) ComputeTranslatorSummary(verbose bool) (ats *AllTranslatorSummar
}
sum.Field = fld.name
sum.Index = i.Name()
sum.Checksum = Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, i.Name())))
sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, i.Name())))
if verbose {
fmt.Printf("row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name)
@ -813,7 +814,7 @@ func (i *Index) ComputeTranslatorSummary(verbose bool) (ats *AllTranslatorSummar
sum.PartitionID = partitionID
sum.Index = i.Name()
sum.Checksum = Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, i.Name())))
sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, i.Name())))
if verbose {
fmt.Printf("col blake3-%v keyN: %10v idN: %10v paritionID: %03v \n", sum.Checksum, sum.KeyCount, sum.IDCount, partitionID)
}

9
lmdb.go
View file

@ -32,6 +32,7 @@ import (
"sync/atomic"
"github.com/glycerine/lmdb-go/lmdb"
"github.com/pilosa/pilosa/v2/hash"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txkey"
"github.com/pkg/errors"
@ -1531,24 +1532,24 @@ func stringifiedLMDBKeysTx(tx *LMDBTx) (r string) {
bkey := it.lastKey
key := txkey.ToString(bkey)
ckey := txkey.KeyExtractContainerKey(bkey)
hash := ""
h := ""
srbm := ""
v := it.lastVal
n := len(v)
if n == 0 {
panic("should not have empty v here")
}
hash = Blake3sum16(v[0:(n - 1)])
h = hash.Blake3sum16(v[0:(n - 1)])
ct := tx.toContainer(v[n-1], v[0:(n-1)])
cts := roaring.NewSliceContainers()
cts.Put(ckey, ct)
rbm := &roaring.Bitmap{Containers: cts}
srbm = BitmapAsString(rbm)
r += fmt.Sprintf("%v -> %v (%v hot)\n", key, hash, tx.countBitsSet(bkey))
r += fmt.Sprintf("%v -> %v (%v hot)\n", key, h, tx.countBitsSet(bkey))
r += " ......." + srbm + "\n"
}
r += "]\n all-in-blake3:" + Blake3sum16([]byte(r))
r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r))
if !any {
return "<empty lmdb database>"

97
rbf/blake3.go
View file

@ -1,97 +0,0 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rbf
import (
"encoding/binary"
"fmt"
"sync"
cryptorand "crypto/rand"
"github.com/zeebo/blake3"
)
// Blake3Hasher is a thread/goroutine safe way to
// obtain a blake3 cryptographic hash of input []byte.
// Reference https://github.com/BLAKE3-team/BLAKE3
// suggests it is 6x faster than BLAKE2B.
// The Go github.com/zeebo/blake3 version is
// AVX2 and SSE4.1 accelerated.
type Blake3Hasher struct {
hasher *blake3.Hasher
hasherMu sync.Mutex
}
// NewBlake3Hasher returns a new Blake3Hasher.
func NewBlake3Hasher() *Blake3Hasher {
return &Blake3Hasher{
hasher: blake3.New(),
}
}
// CryptoHash writes the blake3 cryptographic hash of
// input into buffer and returns it.
// Like the standard libary's hash.Hash interface's Sum() method,
// the buffer is re-used and overwritten
// to avoid allocation. The caller determines the byte length of
// the outputCryptohash by the size of the supplied buffer
// slice, and this will be exactly equal to the supplies bytes.
// In this way, shorter or longer hashes can be provided as
// needed.
func (w *Blake3Hasher) CryptoHash(input []byte, buffer []byte) (outputCryptohash []byte) {
w.hasherMu.Lock()
w.hasher.Reset()
// "Write implements part of the hash.Hash interface. It never returns an error."
// -- https://godoc.org/github.com/zeebo/blake3#Hasher.Write
_, _ = w.hasher.Write(input)
// Digest.Read reads data from the hasher into buffer.
// "It always fills the entire buffer and never errors."
// -- https://godoc.org/github.com/zeebo/blake3#Digest
_, _ = w.hasher.Digest().Read(buffer)
// no chance of panic, so avoid any defer cost.
w.hasherMu.Unlock()
return buffer
}
// blake3sum16 might be slower because we allocate a new hasher every time, but
// it is more conenient for writing debug code. It returns
// a 16 byte hash as a hexidecimal string.
func blake3sum16(input []byte) string {
hasher := blake3.New()
_, _ = hasher.Write(input)
var buf [16]byte
_, _ = hasher.Digest().Read(buf[0:])
return fmt.Sprintf("%x", buf)
}
// cryptoRandInt64 uses crypto/rand to get an random int64
func cryptoRandInt64() int64 {
c := 8
b := make([]byte, c)
_, err := cryptorand.Read(b)
if err != nil {
panic(err)
}
r := int64(binary.LittleEndian.Uint64(b))
return r
}
var _ = cryptoRandInt64 // happy linter

6
rbf/tx.go
View file

@ -18,11 +18,13 @@ import (
"io"
"math"
"sort"
//"strconv"
"strings"
"sync"
"github.com/benbjohnson/immutable"
"github.com/pilosa/pilosa/v2/hash"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txkey"
)
@ -1372,7 +1374,7 @@ func (tx *Tx) DumpString() (r string) {
return ""
}
// note that we can have a bitmap present, but it can be empty
r += "]\n all-in-blake3:" + blake3sum16([]byte(r)) + "\n"
r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + "\n"
return "rbf-" + r
}
@ -1420,7 +1422,7 @@ func bitmapAsString(rbm *roaring.Bitmap) (r string) {
func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string) (s string) {
by := containerToBytes(ct)
hash := blake3sum16(by)
hash := hash.Blake3sum16(by)
cts := roaring.NewSliceContainers()
cts.Put(ckey, ct)

5
txfactory.go
View file

@ -25,6 +25,7 @@ import (
"syscall"
"text/tabwriter"
"github.com/pilosa/pilosa/v2/hash"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txkey"
"github.com/pkg/errors"
@ -793,7 +794,7 @@ func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool) (r string) {
return "" // new convention that empty database => empty string returned.
}
// note that we can have a bitmap present, but it can be empty
r += "]\n all-in-blake3:" + Blake3sum16([]byte(r)) + "\n"
r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + "\n"
return "roaring-" + r
}
@ -871,7 +872,7 @@ func stringifiedRawRoaringFragment(path string, index, field, view string, shard
for citer.Next() {
ckey, ct := citer.Value()
by := containerToBytes(ct)
hash := Blake3sum16(by)
hash := hash.Blake3sum16(by)
cts := roaring.NewSliceContainers()
cts.Put(ckey, ct)