From eca01e4b4fa8e8418dc58a75a175e5a4cb92f91d Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 6 Feb 2018 08:51:38 -0600 Subject: [PATCH] replace sha1 hash with faster xxhash --- Gopkg.lock | 16 ++++++++++++++-- attr.go | 6 ++++-- fragment.go | 7 ++++--- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 7837cadff..76fcccbd4 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -37,6 +37,12 @@ revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" version = "v1.3.1" +[[projects]] + name = "github.com/cespare/xxhash" + packages = ["."] + revision = "5c37fe3735342a2e0d01c87a907579987c8936cc" + version = "v1.0.0" + [[projects]] name = "github.com/davecgh/go-spew" packages = ["spew"] @@ -189,10 +195,16 @@ [[projects]] name = "github.com/shirou/gopsutil" - packages = ["host","internal/common","mem","process"] + packages = ["cpu","host","internal/common","mem","net","process"] revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" +[[projects]] + branch = "master" + name = "github.com/shirou/w32" + packages = ["."] + revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" + [[projects]] name = "github.com/sony/gobreaker" packages = ["."] @@ -268,6 +280,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "d91110a10c830f7a9cc439b9578840d97d9921e84d08242316da8d4a18c68c56" + inputs-digest = "53ae0cdcbfa8419b233a0544d7ebb80614af3dd04bebdc0c8f2d3620a87efa91" solver-name = "gps-cdcl" solver-version = 1 diff --git a/attr.go b/attr.go index ab29fc9c0..437cab433 100644 --- a/attr.go +++ b/attr.go @@ -16,13 +16,15 @@ package pilosa import ( "bytes" - "crypto/sha1" + "encoding/binary" "fmt" "sort" "sync" "time" + "github.com/cespare/xxhash" + "github.com/boltdb/bolt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" @@ -242,7 +244,7 @@ func (s *AttrStore) Blocks() ([]AttrBlock, error) { block := AttrBlock{ID: cur.blockID()} // Compute checksum of every key/value in block. - h := sha1.New() + h := xxhash.New() for k, v := cur.next(); k != nil; k, v = cur.next() { h.Write(k) h.Write(v) diff --git a/fragment.go b/fragment.go index 930c8e02b..95bcb7f2d 100644 --- a/fragment.go +++ b/fragment.go @@ -20,7 +20,6 @@ import ( "bytes" "container/heap" "context" - "crypto/sha1" "encoding/binary" "errors" "fmt" @@ -36,6 +35,8 @@ import ( "time" "unsafe" + "github.com/cespare/xxhash" + "math" "github.com/gogo/protobuf/proto" @@ -1020,7 +1021,7 @@ type TopOptions struct { // Checksum returns a checksum for the entire fragment. // If two fragments have the same checksum then they have the same data. func (f *Fragment) Checksum() []byte { - h := sha1.New() + h := xxhash.New() for _, block := range f.Blocks() { h.Write(block.Checksum) } @@ -1660,7 +1661,7 @@ type blockHasher struct { func newBlockHasher() blockHasher { return blockHasher{ blockID: -1, - hash: sha1.New(), + hash: xxhash.New(), } } func (h *blockHasher) Reset() {