Adds Todd's performance improvements:

- Ingore asserts in fragment container.
- Only log queries that take longer than 90 seconds.

TODO:
- address the TODOs that make the asserts configurable.

fragment.go

@@ -350,6 +350,11 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error)
 		return false, err
 	}
 
+	// Don't update the cache if nothing changed.
+	if !changed {
+		return changed, nil
+	}
+
 	// Invalidate block checksum.
 	delete(f.checksums, int(bitmapID/HashBlockSize))
 
@@ -389,6 +394,11 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (bool, error) {
 		return false, err
 	}
 
+	// Don't update the cache if nothing changed.
+	if !changed {
+		return changed, nil
+	}
+
 	// Invalidate block checksum.
 	delete(f.checksums, int(bitmapID/HashBlockSize))
 
@@ -838,7 +848,6 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error {
 	// Process every bit.
 	// If an error occurs then reopen the storage.
 	lastID := uint64(0)
-	bmCounter := 0
 	if err := func() error {
 		set := make(map[uint64]struct{})
 		for i := range bitmapIDs {
@@ -851,7 +860,7 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error {
 			}
 
 			// Write to storage.
-			changed, err := f.storage.Add(pos)
+			_, err = f.storage.Add(pos)
 			if err != nil {
 				return err
 			}
@@ -863,9 +872,6 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error {
 				lastID = bitmapID
 				set[bitmapID] = struct{}{}
 			}
-			if changed {
-				bmCounter += 1
-			}
 
 			// Invalidate block checksum.
 			delete(f.checksums, int(bitmapID/HashBlockSize))

handler.go

@@ -202,7 +202,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.NotFound(w, r)
 	}
 
-	h.logger().Printf("%s %s %.03fs", r.Method, r.URL.String(), time.Since(t).Seconds())
+	dif := time.Since(t).Seconds()
+	if dif > 90 {
+		h.logger().Printf("%s %s %.03fs", r.Method, r.URL.String(), dif)
+	}
 }
 
 // handleGetSchema handles GET /schema requests.

roaring/roaring.go

@@ -460,8 +460,9 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 		c := b.containers[i]
 
 		// Verify container count before writing.
-		count := c.count()
-		assert(c.count() == c.n, "cannot write container count, mismatch: count=%d, n=%d", count, c.n)
+		// TODO: instead of commenting this out, we need to make it a configuration option
+		//count := c.count()
+		//assert(c.count() == c.n, "cannot write container count, mismatch: count=%d, n=%d", count, c.n)
 
 		binary.LittleEndian.PutUint64(buf[headerSize+i*12:], uint64(key))
 		binary.LittleEndian.PutUint32(buf[headerSize+i*12+8:], uint32(c.n-1))
@@ -532,9 +533,10 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		c := b.containers[i]
 		if c.n <= ArrayMaxSize {
 			c.array = (*[0xFFFFFFF]uint32)(unsafe.Pointer(&data[offset]))[:c.n]
-			for _, v := range c.array {
-				assert(lowbits(uint64(v)) == v, "array value out of range: %d", v)
-			}
+			// TODO: instead of commenting this out, we need to make it a configuration option
+			//for _, v := range c.array {
+			//    assert(lowbits(uint64(v)) == v, "array value out of range: %d", v)
+			//}
 			opsOffset = int(offset) + len(c.array)*4
 		} else {
 			c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN]
@@ -542,8 +544,9 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		}
 
 		// Verify container count on load.
-		count := c.count()
-		assert(c.count() == c.n, "container count mismatch: count=%d, n=%d", count, c.n)
+		// TODO: instead of commenting this out, we need to make it a configuration option
+		//count := c.count()
+		//assert(c.count() == c.n, "container count mismatch: count=%d, n=%d", count, c.n)
 	}
 
 	// Read ops log until the end of the file.
@@ -1074,9 +1077,10 @@ func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) {
 	}
 
 	// Verify all elements are valid.
-	for _, v := range c.array {
-		assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v)
-	}
+	// TODO: instead of commenting this out, we need to make it a configuration option
+	//for _, v := range c.array {
+	//  	assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v)
+	//}
 
 	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:4*c.n])
 	return int64(nn), err