From 95f05ca4d00431e981d9d6bfb52335021b5948a7 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Fri, 14 Dec 2018 16:47:28 -0600
Subject: [PATCH] WIP: allow translate log entry buffer to grow

In the case where a translate log entry contained
many key/id pairs, it was possible for the read
buffer (which was allocated at 65536 bytes) to
fail to handle it. This happened when the serialized
LogEntry was larger than 65536 bytes.

This PR adds logic which returns a custom error called
ErrTranslateReadTargetUndersized notifying the reader
to reallocate a larger read buffer and try the read
again.

TODO:
- [ ] Add a max buffer size check to prevent this from doubling the
buffer size with no limit.
- [ ] Add tests.
---
 http/handler.go | 11 ++++++++++-
 translate.go    | 18 ++++++++++++------
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/http/handler.go b/http/handler.go
index c370656fa..f8188e8fd 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -1449,16 +1449,25 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request)
 	}
 
 	// Copy from reader to client until store or client disconnect.
-	buf := make([]byte, translateStoreBufferSize)
+	useBufferSize := translateStoreBufferSize
+	buf := make([]byte, useBufferSize)
 	for {
 		// Read from store.
 		n, err := rdr.Read(buf)
 		if err == io.EOF {
 			return
+		} else if err == pilosa.ErrTranslateReadTargetUndersized {
+			// Increase the buffer size and try to read again.
+			useBufferSize *= 2
+			buf = make([]byte, useBufferSize)
+			continue
 		} else if err != nil {
 			h.logger.Printf("http: translate store read error: %s", err)
 			return
 		} else if n == 0 {
+			// Reset the default buffer size.
+			useBufferSize = translateStoreBufferSize
+			buf = make([]byte, useBufferSize)
 			continue
 		}
 
diff --git a/translate.go b/translate.go
index 2c3125ad9..b86041e3d 100644
--- a/translate.go
+++ b/translate.go
@@ -29,10 +29,11 @@ const (
 )
 
 var (
-	ErrTranslateStoreClosed       = errors.New("pilosa: translate store closed")
-	ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed")
-	ErrReplicationNotSupported    = errors.New("pilosa: replication not supported")
-	ErrTranslateStoreReadOnly     = errors.New("pilosa: translate store could not find or create key, translate store read only")
+	ErrTranslateStoreClosed          = errors.New("pilosa: translate store closed")
+	ErrTranslateStoreReaderClosed    = errors.New("pilosa: translate store reader closed")
+	ErrReplicationNotSupported       = errors.New("pilosa: replication not supported")
+	ErrTranslateStoreReadOnly        = errors.New("pilosa: translate store could not find or create key, translate store read only")
+	ErrTranslateReadTargetUndersized = errors.New("pilosa: translate read target is undersized")
 )
 
 // TranslateStore is the storage for translation string-to-uint64 values.
@@ -1089,8 +1090,13 @@ func (r *translateFileReader) read(p []byte) (n int, err error) {
 		return 0, nil
 	}
 
-	// Shorten buffer to maximum read size.
-	if max := sz - r.offset; int64(len(p)) > max {
+	if max := sz - r.offset; max > int64(len(p)) {
+		// If p is not large enough to hold a single entry,
+		// return an error so the client can increase the
+		// size of p and try again.
+		return 0, ErrTranslateReadTargetUndersized
+	} else if int64(len(p)) > max {
+		// Shorten buffer to maximum read size.
 		p = p[:max]
 	}