diff --git a/http/handler.go b/http/handler.go index c370656fa..f8188e8fd 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1449,16 +1449,25 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) } // Copy from reader to client until store or client disconnect. - buf := make([]byte, translateStoreBufferSize) + useBufferSize := translateStoreBufferSize + buf := make([]byte, useBufferSize) for { // Read from store. n, err := rdr.Read(buf) if err == io.EOF { return + } else if err == pilosa.ErrTranslateReadTargetUndersized { + // Increase the buffer size and try to read again. + useBufferSize *= 2 + buf = make([]byte, useBufferSize) + continue } else if err != nil { h.logger.Printf("http: translate store read error: %s", err) return } else if n == 0 { + // Reset the default buffer size. + useBufferSize = translateStoreBufferSize + buf = make([]byte, useBufferSize) continue } diff --git a/translate.go b/translate.go index 2c3125ad9..b86041e3d 100644 --- a/translate.go +++ b/translate.go @@ -29,10 +29,11 @@ const ( ) var ( - ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") - ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") - ErrReplicationNotSupported = errors.New("pilosa: replication not supported") - ErrTranslateStoreReadOnly = errors.New("pilosa: translate store could not find or create key, translate store read only") + ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") + ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") + ErrReplicationNotSupported = errors.New("pilosa: replication not supported") + ErrTranslateStoreReadOnly = errors.New("pilosa: translate store could not find or create key, translate store read only") + ErrTranslateReadTargetUndersized = errors.New("pilosa: translate read target is undersized") ) // TranslateStore is the storage for translation string-to-uint64 values. @@ -1089,8 +1090,13 @@ func (r *translateFileReader) read(p []byte) (n int, err error) { return 0, nil } - // Shorten buffer to maximum read size. - if max := sz - r.offset; int64(len(p)) > max { + if max := sz - r.offset; max > int64(len(p)) { + // If p is not large enough to hold a single entry, + // return an error so the client can increase the + // size of p and try again. + return 0, ErrTranslateReadTargetUndersized + } else if int64(len(p)) > max { + // Shorten buffer to maximum read size. p = p[:max] }