scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-11-15 09:31:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

feat: keep tool images away from models with Vision turned off (#33345)

Browse source 
When a tool returned an image, for example Open Terminal reading a PNG, a native tool or an MCP screenshot tool, the image was sent to the model even when the model has Vision turned off. A provider serving a text-only model rejects that request, so the reply failed with an error like "At most 0 image(s) may be provided in one prompt", and every later message in the chat failed the same way. With Vision off the model now gets a short "[image]" note in place of each tool image, including images already saved in the chat, so stuck chats work again. The images stay saved, so they reach the model again once Vision is turned on.

Fixes #32720
This commit is contained in:
Classic298 2026-10-29 09:24:47 +02:00 • committed by GitHub
parent d14d8788af
commit 63ca4237eb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 23 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
backend/open_webui/utils/middleware.py
View file

@ -1939,6 +1939,22 @@ def strip_reasoning_details(output: list) -> list:
]
def strip_tool_images(output: list, model: dict) -> list:
"""Replace tool result images with a text note when the model has Vision turned off."""
if (model.get('info', {}).get('meta', {}).get('capabilities') or {}).get('vision', True):
return output
stripped_output = []
for item in output:
if isinstance(item, dict) and item.get('type') == 'function_call_output':
parts = [
{'type': 'input_text', 'text': '\n[image]'} if part.get('type') == 'input_image' else part
for part in item.get('output', [])
]
item = {**item, 'output': parts}
stripped_output.append(item)
return stripped_output
def process_messages_with_output(
messages: list[dict],
reasoning_format: str | None = None,
@ -2192,6 +2208,8 @@ async def process_chat_payload(request, form_data, user, metadata, model):
# for output produced by the same model.
if message.get('role') == 'assistant' and message.get('model') != model['id'] and isinstance(output, list):
message['output'] = strip_reasoning_details(output)
if message.get('role') == 'assistant' and isinstance(output, list):
message['output'] = strip_tool_images(message['output'], model)
form_data['messages'] = process_messages_with_output(
form_data.get('messages', []),
@ -2990,6 +3008,8 @@ async def process_chat_payload(request, form_data, user, metadata, model):
assistant_message = dict(pending_assistant_message)
if assistant_message.get('model') != model['id'] and isinstance(assistant_message.get('output'), list):
assistant_message['output'] = strip_reasoning_details(assistant_message['output'])
if isinstance(assistant_message.get('output'), list):
assistant_message['output'] = strip_tool_images(assistant_message['output'], model)
form_data['messages'].extend(
process_messages_with_output([assistant_message], reasoning_format=get_reasoning_format(model))
)
@ -5911,12 +5931,12 @@ async def streaming_chat_response_handler(response, ctx):
new_form_data['messages'] = (
[system_message] if system_message else []
) + convert_output_to_messages(
output, raw=True, reasoning_format=get_reasoning_format(model)
strip_tool_images(output, model), raw=True, reasoning_format=get_reasoning_format(model)
)
new_form_data['previous_response_id'] = last_response_id
else:
tool_messages = convert_output_to_messages(
output,
strip_tool_images(output, model),
raw=True,
reasoning_format=get_reasoning_format(model),
flatten_tool_images=True,
@ -6241,7 +6261,7 @@ async def streaming_chat_response_handler(response, ctx):
'messages': [
*messages,
*convert_output_to_messages(
output,
strip_tool_images(output, model),
raw=True,
reasoning_format=get_reasoning_format(model),
flatten_tool_images=True,