From f62c817a75d0a511ccff0a01434078e316b41c47 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 29 Aug 2017 14:43:25 -0500 Subject: [PATCH 1/5] removed go1.7 support from travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fe4a2b8f1..457b991b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: go go: - - 1.7 - 1.8 - master addons: From eb98bdd8c68f7a147b824e79814e3e0d932c5e97 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 1 Sep 2017 13:39:51 -0500 Subject: [PATCH 2/5] no allocating WriteTo --- roaring/roaring.go | 57 ++++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 77dfd8803..ccb0b2368 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -518,6 +518,21 @@ func (b *Bitmap) Optimize() { } } +//hoping this inines +func WriteUint16(w io.Writer, b []byte, v uint16) (int, error) { + binary.LittleEndian.PutUint16(b, v) + return w.Write(b) +} +func WriteUint32(w io.Writer, b []byte, v uint32) (int, error) { + binary.LittleEndian.PutUint32(b, v) + return w.Write(b) +} + +func WriteUint64(w io.Writer, b []byte, v uint64) (int, error) { + binary.LittleEndian.PutUint64(b, v) + return w.Write(b) +} + // WriteTo writes b to w. func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { b.Optimize() @@ -526,15 +541,18 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { containerCount := len(b.keys) - b.countEmptyContainers() headerSize := headerBaseSize + byte2 := make([]byte, 2) + byte4 := make([]byte, 4) + byte8 := make([]byte, 8) // Build header before writing individual container blocks. // Metadata for each container is 8+2+2+4 = sizeof(key) + sizeof(container_type)+sizeof(cardinality) + sizeof(file offset) - buf := make([]byte, headerSize+(containerCount*(8+2+2+4))) + // buf := make([]byte, headerSize+(containerCount*(8+2+2+4))) // Cookie header section. - binary.LittleEndian.PutUint32(buf[0:], cookie) - binary.LittleEndian.PutUint32(buf[4:], uint32(containerCount)) - empty := 0 + WriteUint32(w, byte4, cookie) + WriteUint32(w, byte4, uint32(containerCount)) + // Descriptive header section: encode keys and cardinality. // Key and cardinality are stored interleaved here, 12 bytes per container. for i, key := range b.keys { @@ -545,31 +563,24 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { //count := c.count() //assert(c.count() == c.n, "cannot write container count, mismatch: count=%d, n=%d", count, c.n) if c.n > 0 { - binary.LittleEndian.PutUint64(buf[headerSize+(i-empty)*12:], uint64(key)) - binary.LittleEndian.PutUint16(buf[headerSize+(i-empty)*12+8:], uint16(c.container_type)) - binary.LittleEndian.PutUint16(buf[headerSize+(i-empty)*12+8+2:], uint16(c.n-1)) - } else { - empty++ + WriteUint64(w, byte8, uint64(key)) + WriteUint16(w, byte2, uint16(c.container_type)) + WriteUint16(w, byte2, uint16(c.n-1)) } } // Offset header section: write the offset for each container block. // 4 bytes per container. - offset := uint32(len(buf)) - empty = 0 - for i, c := range b.containers { + offset := uint32(headerSize + (containerCount * (8 + 2 + 2 + 4))) + for _, c := range b.containers { if c.n > 0 { - binary.LittleEndian.PutUint32(buf[headerSize+(containerCount*12)+((i-empty)*4):], uint32(offset)) + WriteUint32(w, byte4, uint32(offset)) offset += uint32(c.size()) - } else { - empty++ } } - // Write header. - i, err := w.Write(buf) - n += int64(i) + n = int64(headerSize + (containerCount * (8 + 2 + 2 + 4))) if err != nil { return n, err } @@ -1613,9 +1624,9 @@ func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) { // Verify all elements are valid. // TODO: instead of commenting this out, we need to make it a configuration option - for _, v := range c.array { - assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v) - } + // for _, v := range c.array { + // assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v) + //} // Write sizeof(uint32) * cardinality bytes. nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:2*c.n]) @@ -1632,8 +1643,10 @@ func (c *container) runWriteTo(w io.Writer) (n int64, err error) { if len(c.runs) == 0 { return 0, nil } + var byte2 [2]byte // Write sizeof(interval16) * runCount bytes. - err = binary.Write(w, binary.LittleEndian, uint16(len(c.runs))) + // err = binary.Write(w, binary.LittleEndian, uint16(len(c.runs))) + _, err = WriteUint16(w, byte2[:], uint16(len(c.runs))) if err != nil { return 0, err } From c418d73fa7250c78b1723ee5ecb0fa86fd5a2da9 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 1 Sep 2017 14:56:14 -0500 Subject: [PATCH 3/5] zero alloc WriteTo --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index ccb0b2368..7cc7cd3af 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -518,7 +518,7 @@ func (b *Bitmap) Optimize() { } } -//hoping this inines +//hoping this in-lines func WriteUint16(w io.Writer, b []byte, v uint16) (int, error) { binary.LittleEndian.PutUint16(b, v) return w.Write(b) From 6f3c5ac51bb145be788d26c67d5619f3fcc7e8a1 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 5 Sep 2017 13:23:51 -0500 Subject: [PATCH 4/5] wrapped fragment with buffered writer --- fragment.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fragment.go b/fragment.go index 7e56c5597..282c28523 100644 --- a/fragment.go +++ b/fragment.go @@ -206,10 +206,11 @@ func (f *Fragment) openStorage() error { if err != nil { return err } else if fi.Size() == 0 { - if _, err := f.storage.WriteTo(f.file); err != nil { + bi := bufio.NewWriter(f.file) + if _, err := f.storage.WriteTo(bi); err != nil { return fmt.Errorf("init storage file: %s", err) } - + bi.Flush() fi, err = f.file.Stat() if err != nil { return err @@ -1262,6 +1263,7 @@ func (f *Fragment) snapshot() error { } else if err := bw.Flush(); err != nil { return fmt.Errorf("flush: %s", err) } + bw.Flush() // Close current storage. if err := f.closeStorage(); err != nil { From 1d55d1c2c5e81210595c51a19677443b8bb474c0 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 5 Sep 2017 15:41:52 -0500 Subject: [PATCH 5/5] cleanup commented code --- fragment.go | 5 +++-- roaring/roaring.go | 4 +--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/fragment.go b/fragment.go index 282c28523..829b4e03b 100644 --- a/fragment.go +++ b/fragment.go @@ -1260,10 +1260,11 @@ func (f *Fragment) snapshot() error { bw := bufio.NewWriter(file) if _, err := f.storage.WriteTo(bw); err != nil { return fmt.Errorf("snapshot write to: %s", err) - } else if err := bw.Flush(); err != nil { + } + + if err := bw.Flush(); err != nil { return fmt.Errorf("flush: %s", err) } - bw.Flush() // Close current storage. if err := f.closeStorage(); err != nil { diff --git a/roaring/roaring.go b/roaring/roaring.go index 7cc7cd3af..456c9fccc 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -547,7 +547,6 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { // Build header before writing individual container blocks. // Metadata for each container is 8+2+2+4 = sizeof(key) + sizeof(container_type)+sizeof(cardinality) + sizeof(file offset) - // buf := make([]byte, headerSize+(containerCount*(8+2+2+4))) // Cookie header section. WriteUint32(w, byte4, cookie) @@ -1644,8 +1643,7 @@ func (c *container) runWriteTo(w io.Writer) (n int64, err error) { return 0, nil } var byte2 [2]byte - // Write sizeof(interval16) * runCount bytes. - // err = binary.Write(w, binary.LittleEndian, uint16(len(c.runs))) + _, err = WriteUint16(w, byte2[:], uint16(len(c.runs))) if err != nil { return 0, err