From 7c7a14e9087ad738ff445f9c0bc4ce7fb6a62c3b Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 8 Sep 2026 01:01:53 +0200 Subject: [PATCH] fix: store tool result images as file references A tool that returned an image wrote the entire base64 data URI into the chat, and stored it twice: once on the message row and again in the chat's own history map. A single 2 MB image cost 5.6 MB of database, and the same payload was sent to the browser with every chat load, so tab memory grew by roughly the image size per image and only came back when the tab was closed. Long image-heavy chats reached tens of megabytes of JSON. The image is now uploaded once and the chat keeps a short file reference, the same way an uploaded image is already stored. The base64 is rebuilt only while a provider request is being assembled, so what the model receives is unchanged. Measured on a 2 MB tool image, with an identical request reaching the provider in both cases: | | stored bytes | base64 copies | | --- | --- | --- | | before | 5,594,322 | 2 | | after | 1,962 | 0 | Fixes #29761 --- backend/open_webui/utils/files.py | 4 ++- backend/open_webui/utils/middleware.py | 39 ++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/backend/open_webui/utils/files.py b/backend/open_webui/utils/files.py index 676af24afe..20dc4ff110 100644 --- a/backend/open_webui/utils/files.py +++ b/backend/open_webui/utils/files.py @@ -32,6 +32,7 @@ from open_webui.storage.provider import Storage BASE64_IMAGE_URL_PREFIX = re.compile(r'data:image/\w+;base64,', re.IGNORECASE) MARKDOWN_IMAGE_URL_PATTERN = re.compile(r'!\[(.*?)\]\((.+?)\)', re.IGNORECASE) +FILE_CONTENT_URL_PATTERN = re.compile(r'^/api/v1/files/([^/?#]+)/content') # Extension-based MIME fallback, only used when ENABLE_IMAGE_CONTENT_TYPE_EXTENSION_FALLBACK is True. _IMAGE_MIME_FALLBACK = { @@ -90,7 +91,8 @@ async def get_image_base64_from_url(url: str, user=None) -> Optional[str]: else: # Non-URL string — treat as file_id. Delegate to the canonical # file-ID resolver which enforces ownership/access checks. - return await get_image_base64_from_file_id(url, user=user) + file_id_match = FILE_CONTENT_URL_PATTERN.match(url) + return await get_image_base64_from_file_id(file_id_match.group(1) if file_id_match else url, user=user) except Exception: return None diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index 7add70949b..19dff98e61 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -972,6 +972,24 @@ async def apply_source_context_to_messages( ) +async def store_tool_result_image(request, image_url, metadata, user): + """Swap a base64 tool image for a stored file so the chat keeps a reference, not the payload.""" + if not isinstance(image_url, str) or not image_url.startswith('data:'): + return image_url + + stored_url = await get_file_url_from_base64( + request, + image_url, + { + 'chat_id': (metadata or {}).get('chat_id'), + 'message_id': (metadata or {}).get('message_id'), + 'session_id': (metadata or {}).get('session_id'), + }, + user, + ) + return stored_url or image_url + + async def process_tool_result( request, tool_function_name, @@ -1399,6 +1417,11 @@ async def chat_completion_tools_handler( ) if tool_result_files: + for file_item in tool_result_files: + file_item['url'] = await store_tool_result_image( + request, file_item.get('url'), metadata, user + ) + await event_emitter( { 'type': 'files', @@ -2087,7 +2110,7 @@ async def convert_url_images_to_base64(form_data, user=None): new_content = [] for item in content: - if not isinstance(item, dict) or item.get('type') != 'image_url': + if not isinstance(item, dict) or item.get('type') not in ('image_url', 'input_image'): new_content.append(item) continue @@ -2104,13 +2127,15 @@ async def convert_url_images_to_base64(form_data, user=None): try: base64_data = await get_image_base64_from_url(image_url, user=user) - if base64_data: + if base64_data and isinstance(image_url_data, str): + new_content.append({**item, 'image_url': base64_data}) + elif base64_data: image_url_payload = {'url': base64_data} if isinstance(image_url_data, dict) and image_url_data.get('detail'): image_url_payload['detail'] = image_url_data['detail'] new_content.append( { - 'type': 'image_url', + 'type': item['type'], 'image_url': image_url_payload, } ) @@ -3288,7 +3313,8 @@ async def drain_approved_tool_calls(request, form_data, user, model, metadata) - display_files = [] for file_item in result.get('files', []): if file_item.get('type') == 'image' and file_item.get('url', '').startswith('data:'): - output_parts.append({'type': 'input_image', 'image_url': file_item['url']}) + image_url = await store_tool_result_image(request, file_item['url'], metadata, user) + output_parts.append({'type': 'input_image', 'image_url': image_url}) else: display_files.append(file_item) @@ -5863,7 +5889,8 @@ async def streaming_chat_response_handler(response, ctx): for file_item in result.get('files', []): if file_item.get('type') == 'image' and file_item.get('url', '').startswith('data:'): # LLM-only: add as input_image part, not frontend display output. - output_parts.append({'type': 'input_image', 'image_url': file_item['url']}) + image_url = await store_tool_result_image(request, file_item['url'], metadata, user) + output_parts.append({'type': 'input_image', 'image_url': image_url}) else: # Frontend display (MCP images, audio, etc.) display_files.append(file_item) @@ -6027,6 +6054,8 @@ async def streaming_chat_response_handler(response, ctx): } ) + new_form_data = await convert_url_images_to_base64(new_form_data, user=user) + if filter_functions: new_form_data, _ = await process_filter_functions( request=request,