diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index a07bd9f6d0..be5ad7ae1c 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -976,6 +976,25 @@ async def apply_source_context_to_messages( ) +BASE64_IMAGE_DATA_URI_RE = re.compile(r'data:image/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/]+={0,2}', re.IGNORECASE) + + +def extract_base64_images(value: Any, files: list) -> Any: + """Move base64 image data URIs out of a tool result so they do not reach the model as text.""" + if isinstance(value, str): + if BASE64_IMAGE_DATA_URI_RE.fullmatch(value): + files.append({'type': 'image', 'url': value}) + return '[image]' + return value + if isinstance(value, dict): + return {key: extract_base64_images(item, files) for key, item in value.items()} + if isinstance(value, list): + return [extract_base64_images(item, files) for item in value] + if isinstance(value, tuple): + return tuple(extract_base64_images(item, files) for item in value) + return value + + async def process_tool_result( request, tool_function_name, @@ -1161,8 +1180,9 @@ async def process_tool_result( tool_response.append(resource.get('uri')) tool_result = tool_response[0] if len(tool_response) == 1 else tool_response else: # OpenAPI - for item in tool_result: - if isinstance(item, str) and item.startswith('data:'): + # Images are left to extract_base64_images below, which attaches them so the model can see them. + for item in list(tool_result): + if isinstance(item, str) and item.startswith('data:') and not BASE64_IMAGE_DATA_URI_RE.fullmatch(item): tool_result_files.append( { 'type': 'data', @@ -1171,6 +1191,8 @@ async def process_tool_result( ) tool_result.remove(item) + tool_result = extract_base64_images(tool_result, tool_result_files) + if isinstance(tool_result, list): tool_result = {'results': tool_result}