WIP: allow translate log entry buffer to grow

In the case where a translate log entry contained
many key/id pairs, it was possible for the read
buffer (which was allocated at 65536 bytes) to
fail to handle it. This happened when the serialized
LogEntry was larger than 65536 bytes.

This PR adds logic which returns a custom error called
ErrTranslateReadTargetUndersized notifying the reader
to reallocate a larger read buffer and try the read
again.

TODO:
- [ ] Add a max buffer size check to prevent this from doubling the
buffer size with no limit.
- [ ] Add tests.

http/handler.go

@@ -1449,16 +1449,25 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request)
 	}
 
 	// Copy from reader to client until store or client disconnect.
-	buf := make([]byte, translateStoreBufferSize)
+	useBufferSize := translateStoreBufferSize
+	buf := make([]byte, useBufferSize)
 	for {
 		// Read from store.
 		n, err := rdr.Read(buf)
 		if err == io.EOF {
 			return
+		} else if err == pilosa.ErrTranslateReadTargetUndersized {
+			// Increase the buffer size and try to read again.
+			useBufferSize *= 2
+			buf = make([]byte, useBufferSize)
+			continue
 		} else if err != nil {
 			h.logger.Printf("http: translate store read error: %s", err)
 			return
 		} else if n == 0 {
+			// Reset the default buffer size.
+			useBufferSize = translateStoreBufferSize
+			buf = make([]byte, useBufferSize)
 			continue
 		}
 

translate.go

@@ -29,10 +29,11 @@ const (
 )
 
 var (
-	ErrTranslateStoreClosed       = errors.New("pilosa: translate store closed")
-	ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed")
-	ErrReplicationNotSupported    = errors.New("pilosa: replication not supported")
-	ErrTranslateStoreReadOnly     = errors.New("pilosa: translate store could not find or create key, translate store read only")
+	ErrTranslateStoreClosed          = errors.New("pilosa: translate store closed")
+	ErrTranslateStoreReaderClosed    = errors.New("pilosa: translate store reader closed")
+	ErrReplicationNotSupported       = errors.New("pilosa: replication not supported")
+	ErrTranslateStoreReadOnly        = errors.New("pilosa: translate store could not find or create key, translate store read only")
+	ErrTranslateReadTargetUndersized = errors.New("pilosa: translate read target is undersized")
 )
 
 // TranslateStore is the storage for translation string-to-uint64 values.
@@ -1089,8 +1090,13 @@ func (r *translateFileReader) read(p []byte) (n int, err error) {
 		return 0, nil
 	}
 
-	// Shorten buffer to maximum read size.
-	if max := sz - r.offset; int64(len(p)) > max {
+	if max := sz - r.offset; max > int64(len(p)) {
+		// If p is not large enough to hold a single entry,
+		// return an error so the client can increase the
+		// size of p and try again.
+		return 0, ErrTranslateReadTargetUndersized
+	} else if int64(len(p)) > max {
+		// Shorten buffer to maximum read size.
 		p = p[:max]
 	}