From d0abcfc9102fa665334c2a9fc7a1c792bf58714a Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 13 May 2016 19:59:14 -0500 Subject: [PATCH 1/5] changed slicewidth --- client.go | 29 +++++++++++++++++++---------- cmd/pilosa/main.go | 10 +++++----- fragment.go | 2 +- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/client.go b/client.go index 15f4b401b..77fcdf771 100644 --- a/client.go +++ b/client.go @@ -175,12 +175,29 @@ func (c *Client) Import(db, frame string, slice uint64, bits []Bit) error { return ErrFrameRequired } + buf, err := ImportPayload(db, frame, slice, bits) + if err != nil { + return fmt.Errorf("Error Creating Payload: %s", err) + } + // Retrieve a list of nodes that own the slice. nodes, err := c.SliceNodes(slice) + if err != nil { return fmt.Errorf("slice nodes: %s", err) } + // Import to each node. + for _, node := range nodes { + if err := c.importNode(node, buf); err != nil { + return fmt.Errorf("import node: host=%s, err=%s", node.Host, err) + } + } + + return nil +} + +func ImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, error) { // Separate bitmap and profile IDs to reduce allocations. bitmapIDs := Bits(bits).BitmapIDs() profileIDs := Bits(bits).ProfileIDs() @@ -194,17 +211,9 @@ func (c *Client) Import(db, frame string, slice uint64, bits []Bit) error { ProfileIDs: profileIDs, }) if err != nil { - return fmt.Errorf("marshal import request: %s", err) + return nil, fmt.Errorf("marshal import request: %s", err) } - - // Import to each node. - for _, node := range nodes { - if err := c.importNode(node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.Host, err) - } - } - - return nil + return buf, nil } // importNode sends a pre-marshaled import request to a node. diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go index 5c7971e6a..32b0f2e18 100644 --- a/cmd/pilosa/main.go +++ b/cmd/pilosa/main.go @@ -13,7 +13,6 @@ import ( "net/url" "os" "os/signal" - "os/user" "path/filepath" "runtime/pprof" "strconv" @@ -371,13 +370,14 @@ func (m *Main) ParseFlags(args []string) error { // Expand home directory. prefix := "~" + string(filepath.Separator) if strings.HasPrefix(m.Config.DataDir, prefix) { - u, err := user.Current() - if err != nil { + // u, err := user.Current() + HomeDir := os.Getenv("HOME") + /*if err != nil { return err - } else if u.HomeDir == "" { + } else*/if HomeDir == "" { return errors.New("data directory not specified and no home dir available") } - m.Config.DataDir = filepath.Join(u.HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix)) + m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix)) } return nil diff --git a/fragment.go b/fragment.go index 545f7aa8d..52ffd9f9a 100644 --- a/fragment.go +++ b/fragment.go @@ -25,7 +25,7 @@ import ( const ( // SliceWidth is the number of profile IDs in a slice. - SliceWidth = 65536 + SliceWidth = 2097152 // SnapshotExt is the file extension used for an in-process snapshot. SnapshotExt = ".snapshotting" From 954d8b5346dd85461273494231accbcae9c7f12a Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 25 May 2016 09:12:56 -0500 Subject: [PATCH 2/5] no alloc on search --- bitmap.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bitmap.go b/bitmap.go index 41bd98e6c..f7c5217a1 100644 --- a/bitmap.go +++ b/bitmap.go @@ -16,9 +16,14 @@ import ( "github.com/yasushi-saito/rbtree" ) -const CounterMask = uint64(0xffffffffffffffff) +const ( + CounterMask = uint64(0xffffffffffffffff) +) -var CounterKey = int64(-1) +var ( + CounterKey = int64(-1) + EMPTY_BLOCK = make(Blocks, 32) +) // Bitmap represents a bitmap broken up into Chunks. // Internally it is represented as a red-black tree of chunks. @@ -367,7 +372,7 @@ func (b *Bitmap) Bits() []uint64 { func (b *Bitmap) SetBit(i uint64) (changed bool) { address := deref(i) - chunk := b.Chunk(&Chunk{address.ChunkKey, make(Blocks, 32)}) + chunk := b.Chunk(&Chunk{address.ChunkKey, EMPTY_BLOCK}) if chunk == nil { chunk = &Chunk{address.ChunkKey, make(Blocks, 32)} b.AddChunk(chunk) @@ -385,7 +390,7 @@ func (b *Bitmap) SetBit(i uint64) (changed bool) { func (b *Bitmap) ClearBit(i uint64) (changed bool) { address := deref(i) - chunk := b.Chunk(&Chunk{address.ChunkKey, make(Blocks, 32)}) + chunk := b.Chunk(&Chunk{address.ChunkKey, EMPTY_BLOCK}) if chunk == nil { return false } From 89b582f4989fdf12208ec3d657ed4332244af087 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 27 May 2016 11:15:14 -0500 Subject: [PATCH 3/5] patched in assembly routines for reference --- roaring/assembly.go | 7 ++++++ roaring/assembly_amd64.s | 21 ++++++++++++++++ roaring/assembly_test.go | 53 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 roaring/assembly.go create mode 100644 roaring/assembly_amd64.s create mode 100644 roaring/assembly_test.go diff --git a/roaring/assembly.go b/roaring/assembly.go new file mode 100644 index 000000000..16012be82 --- /dev/null +++ b/roaring/assembly.go @@ -0,0 +1,7 @@ +package roaring + +func hasAsm() bool + +func BSFQ(memory uint64) int + +func POPCNTQ(memory uint64) int diff --git a/roaring/assembly_amd64.s b/roaring/assembly_amd64.s new file mode 100644 index 000000000..a33ade6ab --- /dev/null +++ b/roaring/assembly_amd64.s @@ -0,0 +1,21 @@ +#include "textflag.h" + +TEXT ·hasAsm(SB),4,$0 + MOVQ $1, AX + CPUID + SHRQ $23, CX + ANDQ $1, CX + MOVB CX, ret+0(FP) + RET + +TEXT ·POPCNTQ(SB),NOSPLIT,$0-8 + MOVQ x+0(FP), BP + POPCNTQ BP, BX + MOVQ BX, ret+8(FP) + RET + +TEXT ·BSFQ(SB),NOSPLIT,$0-8 + MOVQ x+0(FP), BP + BSFQ BP, BX + MOVQ BX, ret+8(FP) + RET diff --git a/roaring/assembly_test.go b/roaring/assembly_test.go new file mode 100644 index 000000000..3a3aa2854 --- /dev/null +++ b/roaring/assembly_test.go @@ -0,0 +1,53 @@ +package roaring + +import "testing" + +func TestBSFQ(t *testing.T) { + result := BSFQ(2) + if result != 1 { + t.Fatalf("BSF INCORRECT: %d", result) + } +} + +func TestBSFQ_CompareGo(t *testing.T) { + v := uint64(1) + for i := 0; i < 64; i++ { + if BSFQ(v) != trailingZeroN(v) { + t.Fatalf("BSF INCORRECT: %d %d", BSFQ(v), trailingZeroN(v)) + } + if v == 0 { + v = 1 + } else { + v *= 2 + } + } + /* + if bsfq(0) != trailingZeroN(0) { + fmt.Println(bsfq(0)) + t.Fatalf("BSF INCORRECT") + } + */ +} +func BenchmarkBSF(b *testing.B) { + for i := 0; i < b.N; i++ { + BSFQ(uint64(i)) + } +} + +func BenchmarkTrailingZeroN(b *testing.B) { + for i := 0; i < b.N; i++ { + trailingZeroN(uint64(i)) + } +} + +func BenchmarkPOPCNTQ(b *testing.B) { + for i := 0; i < b.N; i++ { + POPCNTQ(uint64(i)) + } +} + +func BenchmarkPopcount(b *testing.B) { + for i := 0; i < b.N; i++ { + popcount(uint64(i)) + } +} From a935f601a1651c0ba0a3a618a7e1d89abff624f5 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 1 Jun 2016 14:10:34 -0500 Subject: [PATCH 4/5] Import optimization --- fragment.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/fragment.go b/fragment.go index 0643c4601..98cb30171 100644 --- a/fragment.go +++ b/fragment.go @@ -821,9 +821,11 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { // Process every bit. // If an error occurs then reopen the storage. if err := func() error { + last_id := uint64(0) + bm_counter := uint64(0) + var bitmap *Bitmap for i := range bitmapIDs { bitmapID, profileID := bitmapIDs[i], profileIDs[i] - // Determine the position of the bit in the storage. pos, err := f.pos(bitmapID, profileID) if err != nil { @@ -835,11 +837,24 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { return err } + // import optimization to avoid linear foreach calls + // slight risk of concurrent cache counter being off but + // no real danger + if bitmapID != last_id { + bitmap = f.bitmap(bitmapID) + if last_id != 0 { + f.cache.Add(last_id, bm_counter) + } + bm_counter = bitmap.Count() + last_id = bitmapID + } + // Invalidate block checksum. delete(f.checksums, int(bitmapID/HashBlockSize)) - + if bitmap.SetBit(profileID) { + bm_counter += 1 + } // Update the cache. - f.bitmap(bitmapID).SetBit(profileID) } return nil }(); err != nil { From 13f00b08ae498daaaa80783dc2cef98ae334bb08 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 1 Jun 2016 14:55:48 -0500 Subject: [PATCH 5/5] code cleanup --- client.go | 4 ++-- fragment.go | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/client.go b/client.go index 08088e442..42302bbaa 100644 --- a/client.go +++ b/client.go @@ -176,7 +176,7 @@ func (c *Client) Import(db, frame string, slice uint64, bits []Bit) error { return ErrFrameRequired } - buf, err := ImportPayload(db, frame, slice, bits) + buf, err := MarshalImportPayload(db, frame, slice, bits) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -198,7 +198,7 @@ func (c *Client) Import(db, frame string, slice uint64, bits []Bit) error { return nil } -func ImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, error) { +func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, error) { // Separate bitmap and profile IDs to reduce allocations. bitmapIDs := Bits(bits).BitmapIDs() profileIDs := Bits(bits).ProfileIDs() diff --git a/fragment.go b/fragment.go index 98cb30171..9142ad573 100644 --- a/fragment.go +++ b/fragment.go @@ -821,8 +821,8 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { // Process every bit. // If an error occurs then reopen the storage. if err := func() error { - last_id := uint64(0) - bm_counter := uint64(0) + lastID := uint64(0) + bmCounter := uint64(0) var bitmap *Bitmap for i := range bitmapIDs { bitmapID, profileID := bitmapIDs[i], profileIDs[i] @@ -840,19 +840,19 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { // import optimization to avoid linear foreach calls // slight risk of concurrent cache counter being off but // no real danger - if bitmapID != last_id { + if i == 0 || bitmapID != lastID { bitmap = f.bitmap(bitmapID) - if last_id != 0 { - f.cache.Add(last_id, bm_counter) + if i != 0 { + f.cache.Add(lastID, bmCounter) } - bm_counter = bitmap.Count() - last_id = bitmapID + bmCounter = bitmap.Count() + lastID = bitmapID } // Invalidate block checksum. delete(f.checksums, int(bitmapID/HashBlockSize)) if bitmap.SetBit(profileID) { - bm_counter += 1 + bmCounter += 1 } // Update the cache. }