From eca51269bb144d3509c48454df050cfc422fe106 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Mon, 23 Mar 2026 16:21:21 -0500 Subject: [PATCH 001/125] refac --- src/lib/components/chat/MessageInput.svelte | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index a7acf99920..3b0e4ac612 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -95,7 +95,7 @@ import CommandSuggestionList from './MessageInput/CommandSuggestionList.svelte'; import Knobs from '../icons/Knobs.svelte'; import ValvesModal from '../workspace/common/ValvesModal.svelte'; - import PageEdit from '../icons/PageEdit.svelte'; + import Note from '../icons/Note.svelte'; import { goto } from '$app/navigation'; import InputModal from '../common/InputModal.svelte'; import Expand from '../icons/Expand.svelte'; @@ -1834,19 +1834,19 @@ {:else} - {#if prompt !== '' && !history?.currentId && ($config?.features?.enable_notes ?? false) && ($_user?.role === 'admin' || ($_user?.permissions?.features?.notes ?? true))} + {#if prompt !== '' && !history?.currentId && !$selectedTerminalId && ($config?.features?.enable_notes ?? false) && ($_user?.role === 'admin' || ($_user?.permissions?.features?.notes ?? true))} {/if} From 5d7766e1b6f7ca7749c5a5a780d7b1bb2da28a2f Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Mon, 23 Mar 2026 16:46:54 -0500 Subject: [PATCH 002/125] refac --- backend/open_webui/tools/builtin.py | 318 +++++++++++++++++++++++-- backend/open_webui/utils/middleware.py | 5 +- backend/open_webui/utils/tools.py | 7 +- 3 files changed, 312 insertions(+), 18 deletions(-) diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py index 70ce20a98f..d0e5ba2ab5 100644 --- a/backend/open_webui/tools/builtin.py +++ b/backend/open_webui/tools/builtin.py @@ -1587,17 +1587,26 @@ async def search_knowledge_files( return json.dumps({'error': str(e)}) +# Hard cap for view_file / view_knowledge_file output +MAX_VIEW_FILE_CHARS = 100_000 +DEFAULT_VIEW_FILE_MAX_CHARS = 10_000 + + async def view_file( file_id: str, + offset: int = 0, + max_chars: int = DEFAULT_VIEW_FILE_MAX_CHARS, __request__: Request = None, __user__: dict = None, __model_knowledge__: Optional[list[dict]] = None, ) -> str: """ - Get the full content of a file by its ID. + Get the content of a file by its ID. Supports pagination for large files. :param file_id: The ID of the file to retrieve - :return: JSON with the file's id, filename, and full text content + :param offset: Character offset to start reading from (default: 0) + :param max_chars: Maximum characters to return (default: 10000, hard cap: 100000) + :return: JSON with the file's id, filename, content, and pagination metadata if truncated """ if __request__ is None: return json.dumps({'error': 'Request context not available'}) @@ -1605,6 +1614,22 @@ async def view_file( if not __user__: return json.dumps({'error': 'User context not available'}) + # Coerce parameters from LLM tool calls (may come as strings) + if isinstance(offset, str): + try: + offset = int(offset) + except ValueError: + offset = 0 + if isinstance(max_chars, str): + try: + max_chars = int(max_chars) + except ValueError: + max_chars = DEFAULT_VIEW_FILE_MAX_CHARS + + # Enforce hard cap + max_chars = min(max(max_chars, 1), MAX_VIEW_FILE_CHARS) + offset = max(offset, 0) + try: from open_webui.models.files import Files from open_webui.utils.access_control.files import has_access_to_file @@ -1634,16 +1659,27 @@ async def view_file( if file.data: content = file.data.get('content', '') - return json.dumps( - { - 'id': file.id, - 'filename': file.filename, - 'content': content, - 'updated_at': file.updated_at, - 'created_at': file.created_at, - }, - ensure_ascii=False, - ) + total_chars = len(content) + sliced = content[offset:offset + max_chars] + is_truncated = (offset + len(sliced)) < total_chars + + result = { + 'id': file.id, + 'filename': file.filename, + 'content': sliced, + 'updated_at': file.updated_at, + 'created_at': file.created_at, + } + + if is_truncated or offset > 0: + result['truncated'] = is_truncated + result['total_chars'] = total_chars + result['returned_chars'] = len(sliced) + result['offset'] = offset + if is_truncated: + result['next_offset'] = offset + len(sliced) + + return json.dumps(result, ensure_ascii=False) except Exception as e: log.exception(f'view_file error: {e}') return json.dumps({'error': str(e)}) @@ -1651,14 +1687,18 @@ async def view_file( async def view_knowledge_file( file_id: str, + offset: int = 0, + max_chars: int = DEFAULT_VIEW_FILE_MAX_CHARS, __request__: Request = None, __user__: dict = None, ) -> str: """ - Get the full content of a file from a knowledge base. + Get the content of a file from a knowledge base. Supports pagination for large files. :param file_id: The ID of the file to retrieve - :return: JSON with the file's id, filename, and full text content + :param offset: Character offset to start reading from (default: 0) + :param max_chars: Maximum characters to return (default: 10000, hard cap: 100000) + :return: JSON with the file's id, filename, content, and pagination metadata if truncated """ if __request__ is None: return json.dumps({'error': 'Request context not available'}) @@ -1666,6 +1706,22 @@ async def view_knowledge_file( if not __user__: return json.dumps({'error': 'User context not available'}) + # Coerce parameters from LLM tool calls (may come as strings) + if isinstance(offset, str): + try: + offset = int(offset) + except ValueError: + offset = 0 + if isinstance(max_chars, str): + try: + max_chars = int(max_chars) + except ValueError: + max_chars = DEFAULT_VIEW_FILE_MAX_CHARS + + # Enforce hard cap + max_chars = min(max(max_chars, 1), MAX_VIEW_FILE_CHARS) + offset = max(offset, 0) + try: from open_webui.models.files import Files from open_webui.models.knowledge import Knowledges @@ -1708,10 +1764,14 @@ async def view_knowledge_file( if file.data: content = file.data.get('content', '') + total_chars = len(content) + sliced = content[offset:offset + max_chars] + is_truncated = (offset + len(sliced)) < total_chars + result = { 'id': file.id, 'filename': file.filename, - 'content': content, + 'content': sliced, 'updated_at': file.updated_at, 'created_at': file.created_at, } @@ -1719,12 +1779,240 @@ async def view_knowledge_file( result['knowledge_id'] = knowledge_info['id'] result['knowledge_name'] = knowledge_info['name'] + if is_truncated or offset > 0: + result['truncated'] = is_truncated + result['total_chars'] = total_chars + result['returned_chars'] = len(sliced) + result['offset'] = offset + if is_truncated: + result['next_offset'] = offset + len(sliced) + return json.dumps(result, ensure_ascii=False) except Exception as e: log.exception(f'view_knowledge_file error: {e}') return json.dumps({'error': str(e)}) +async def list_attached_knowledge( + __request__: Request = None, + __user__: dict = None, + __model_knowledge__: Optional[list[dict]] = None, +) -> str: + """ + List all knowledge bases, files, and notes attached to the current model. + Use this first to discover what knowledge is available before querying or reading files. + + :return: JSON with knowledge_bases, files, and notes attached to this model + """ + if __request__ is None: + return json.dumps({'error': 'Request context not available'}) + + if not __user__: + return json.dumps({'error': 'User context not available'}) + + if not __model_knowledge__: + return json.dumps({'knowledge_bases': [], 'files': [], 'notes': []}) + + try: + from open_webui.models.knowledge import Knowledges + from open_webui.models.files import Files + from open_webui.models.notes import Notes + from open_webui.models.access_grants import AccessGrants + + user_id = __user__.get('id') + user_role = __user__.get('role', 'user') + user_group_ids = [group.id for group in Groups.get_groups_by_member_id(user_id)] + + knowledge_bases = [] + files = [] + notes = [] + + for item in __model_knowledge__: + item_type = item.get('type') + item_id = item.get('id') + + if item_type == 'collection': + knowledge = Knowledges.get_knowledge_by_id(item_id) + if knowledge and ( + user_role == 'admin' + or knowledge.user_id == user_id + or AccessGrants.has_access( + user_id=user_id, + resource_type='knowledge', + resource_id=knowledge.id, + permission='read', + user_group_ids=set(user_group_ids), + ) + ): + kb_files = Knowledges.get_files_by_id(knowledge.id) + file_count = len(kb_files) if kb_files else 0 + + kb_entry = { + 'id': knowledge.id, + 'name': knowledge.name, + 'description': knowledge.description or '', + 'file_count': file_count, + } + + # Include file listing for each KB + if kb_files: + kb_entry['files'] = [ + {'id': f.id, 'filename': f.filename} + for f in kb_files + ] + + knowledge_bases.append(kb_entry) + + elif item_type == 'file': + file = Files.get_file_by_id(item_id) + if file: + files.append({ + 'id': file.id, + 'filename': file.filename, + 'updated_at': file.updated_at, + }) + + elif item_type == 'note': + note = Notes.get_note_by_id(item_id) + if note and ( + user_role == 'admin' + or note.user_id == user_id + or AccessGrants.has_access( + user_id=user_id, + resource_type='note', + resource_id=note.id, + permission='read', + ) + ): + notes.append({ + 'id': note.id, + 'title': note.title, + }) + + return json.dumps({ + 'knowledge_bases': knowledge_bases, + 'files': files, + 'notes': notes, + }, ensure_ascii=False) + except Exception as e: + log.exception(f'list_attached_knowledge error: {e}') + return json.dumps({'error': str(e)}) + + +async def search_attached_files( + query: str, + knowledge_id: Optional[str] = None, + count: int = 10, + skip: int = 0, + __request__: Request = None, + __user__: dict = None, + __model_knowledge__: Optional[list[dict]] = None, +) -> str: + """ + Search files by filename within the attached knowledge scope. + Only searches knowledge bases and files that are attached to the current model. + + :param query: The filename search query + :param knowledge_id: Optional KB id to limit search to a specific attached knowledge base + :param count: Maximum number of results to return (default: 10) + :param skip: Number of results to skip for pagination (default: 0) + :return: JSON with matching files containing id, filename, knowledge_id, and knowledge_name + """ + if __request__ is None: + return json.dumps({'error': 'Request context not available'}) + + if not __user__: + return json.dumps({'error': 'User context not available'}) + + if not __model_knowledge__: + return json.dumps([]) + + try: + from open_webui.models.knowledge import Knowledges + from open_webui.models.files import Files + from open_webui.models.access_grants import AccessGrants + + user_id = __user__.get('id') + user_role = __user__.get('role', 'user') + user_group_ids = [group.id for group in Groups.get_groups_by_member_id(user_id)] + + # Collect attached KB IDs and direct file IDs + attached_kb_ids = set() + attached_file_ids = set() + + for item in __model_knowledge__: + item_type = item.get('type') + item_id = item.get('id') + if item_type == 'collection': + attached_kb_ids.add(item_id) + elif item_type == 'file': + attached_file_ids.add(item_id) + + # If knowledge_id is specified, verify it's in the attached set + if knowledge_id: + if knowledge_id not in attached_kb_ids: + return json.dumps({'error': f'Knowledge base {knowledge_id} is not attached to this model'}) + attached_kb_ids = {knowledge_id} + + all_files = [] + + # Search within attached KBs + for kb_id in attached_kb_ids: + knowledge = Knowledges.get_knowledge_by_id(kb_id) + if not knowledge: + continue + + if not ( + user_role == 'admin' + or knowledge.user_id == user_id + or AccessGrants.has_access( + user_id=user_id, + resource_type='knowledge', + resource_id=knowledge.id, + permission='read', + user_group_ids=set(user_group_ids), + ) + ): + continue + + result = Knowledges.search_files_by_id( + knowledge_id=kb_id, + user_id=user_id, + filter={'query': query}, + skip=0, + limit=count + skip, # Fetch enough for pagination across KBs + ) + + for file in result.items: + all_files.append({ + 'id': file.id, + 'filename': file.filename, + 'knowledge_id': knowledge.id, + 'knowledge_name': knowledge.name, + 'updated_at': file.updated_at, + }) + + # Search within directly attached files (filename match) + if not knowledge_id and attached_file_ids: + query_lower = query.lower() if query else '' + for file_id in attached_file_ids: + file = Files.get_file_by_id(file_id) + if file and (not query_lower or query_lower in file.filename.lower()): + all_files.append({ + 'id': file.id, + 'filename': file.filename, + 'updated_at': file.updated_at, + }) + + # Apply pagination across combined results + all_files = all_files[skip:skip + count] + + return json.dumps(all_files, ensure_ascii=False) + except Exception as e: + log.exception(f'search_attached_files error: {e}') + return json.dumps({'error': str(e)}) + + async def query_knowledge_files( query: str, knowledge_ids: Optional[list[str]] = None, diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index 044c2974b3..cbf644b53c 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -227,7 +227,7 @@ def get_citation_source_from_tool_result( Returns a list of sources (usually one, but query_knowledge_files may return multiple). """ _EXPECTS_LIST = {'search_web', 'query_knowledge_files'} - _EXPECTS_DICT = {'view_knowledge_file'} + _EXPECTS_DICT = {'view_knowledge_file', 'view_file'} try: try: @@ -271,7 +271,7 @@ def get_citation_source_from_tool_result( } ] - elif tool_name == 'view_knowledge_file': + elif tool_name in ('view_knowledge_file', 'view_file'): file_data = tool_result filename = file_data.get('filename', 'Unknown File') file_id = file_data.get('id', '') @@ -4143,6 +4143,7 @@ async def streaming_chat_response_handler(response, ctx): in [ 'search_web', 'fetch_url', + 'view_file', 'view_knowledge_file', 'query_knowledge_files', ] diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index a098a83979..6f24739b0d 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -79,6 +79,8 @@ from open_webui.tools.builtin import ( query_knowledge_bases, search_knowledge_files, query_knowledge_files, + list_attached_knowledge, + search_attached_files, view_file, view_knowledge_file, view_skill, @@ -405,12 +407,15 @@ def get_builtin_tools( model_knowledge = list(model_knowledge or []) + list(folder_knowledge) if is_builtin_tool_enabled('knowledge'): if model_knowledge: - # Model has attached knowledge - only allow semantic search within it + # Model has attached knowledge - provide discovery, search and semantic tools + builtin_functions.append(list_attached_knowledge) + builtin_functions.append(search_attached_files) builtin_functions.append(query_knowledge_files) knowledge_types = {item.get('type') for item in model_knowledge} if 'file' in knowledge_types or 'collection' in knowledge_types: builtin_functions.append(view_file) + builtin_functions.append(view_knowledge_file) if 'note' in knowledge_types: builtin_functions.append(view_note) else: From 0f0ba7dadd043460d205477fd3b57556aa970847 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Mon, 23 Mar 2026 16:56:50 -0500 Subject: [PATCH 003/125] refac --- backend/open_webui/tools/builtin.py | 199 ++++++++++++---------------- backend/open_webui/utils/tools.py | 7 +- 2 files changed, 85 insertions(+), 121 deletions(-) diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py index d0e5ba2ab5..9fbe5f75f1 100644 --- a/backend/open_webui/tools/builtin.py +++ b/backend/open_webui/tools/builtin.py @@ -1528,9 +1528,11 @@ async def search_knowledge_files( skip: int = 0, __request__: Request = None, __user__: dict = None, + __model_knowledge__: Optional[list[dict]] = None, ) -> str: """ - Search files across knowledge bases the user has access to. + Search files by filename across knowledge bases the user has access to. + When the model has attached knowledge, searches only within attached KBs and files. :param query: The search query to find matching files by filename :param knowledge_id: Optional KB id to limit search to a specific knowledge base @@ -1546,10 +1548,87 @@ async def search_knowledge_files( try: from open_webui.models.knowledge import Knowledges + from open_webui.models.files import Files + from open_webui.models.access_grants import AccessGrants user_id = __user__.get('id') + user_role = __user__.get('role', 'user') user_group_ids = [group.id for group in Groups.get_groups_by_member_id(user_id)] + # When model has attached knowledge, scope to attached KBs/files only + if __model_knowledge__: + attached_kb_ids = set() + attached_file_ids = set() + + for item in __model_knowledge__: + item_type = item.get('type') + item_id = item.get('id') + if item_type == 'collection': + attached_kb_ids.add(item_id) + elif item_type == 'file': + attached_file_ids.add(item_id) + + # If knowledge_id specified, verify it's in the attached set + if knowledge_id: + if knowledge_id not in attached_kb_ids: + return json.dumps({'error': f'Knowledge base {knowledge_id} is not attached to this model'}) + attached_kb_ids = {knowledge_id} + + all_files = [] + + # Search within attached KBs + for kb_id in attached_kb_ids: + knowledge = Knowledges.get_knowledge_by_id(kb_id) + if not knowledge: + continue + + if not ( + user_role == 'admin' + or knowledge.user_id == user_id + or AccessGrants.has_access( + user_id=user_id, + resource_type='knowledge', + resource_id=knowledge.id, + permission='read', + user_group_ids=set(user_group_ids), + ) + ): + continue + + result = Knowledges.search_files_by_id( + knowledge_id=kb_id, + user_id=user_id, + filter={'query': query}, + skip=0, + limit=count + skip, + ) + + for file in result.items: + all_files.append({ + 'id': file.id, + 'filename': file.filename, + 'knowledge_id': knowledge.id, + 'knowledge_name': knowledge.name, + 'updated_at': file.updated_at, + }) + + # Search within directly attached files (filename match) + if not knowledge_id and attached_file_ids: + query_lower = query.lower() if query else '' + for file_id in attached_file_ids: + file = Files.get_file_by_id(file_id) + if file and (not query_lower or query_lower in file.filename.lower()): + all_files.append({ + 'id': file.id, + 'filename': file.filename, + 'updated_at': file.updated_at, + }) + + # Apply pagination across combined results + all_files = all_files[skip:skip + count] + return json.dumps(all_files, ensure_ascii=False) + + # No attached knowledge - search all accessible KBs if knowledge_id: result = Knowledges.search_files_by_id( knowledge_id=knowledge_id, @@ -1793,7 +1872,7 @@ async def view_knowledge_file( return json.dumps({'error': str(e)}) -async def list_attached_knowledge( +async def list_knowledge( __request__: Request = None, __user__: dict = None, __model_knowledge__: Optional[list[dict]] = None, @@ -1895,121 +1974,7 @@ async def list_attached_knowledge( 'notes': notes, }, ensure_ascii=False) except Exception as e: - log.exception(f'list_attached_knowledge error: {e}') - return json.dumps({'error': str(e)}) - - -async def search_attached_files( - query: str, - knowledge_id: Optional[str] = None, - count: int = 10, - skip: int = 0, - __request__: Request = None, - __user__: dict = None, - __model_knowledge__: Optional[list[dict]] = None, -) -> str: - """ - Search files by filename within the attached knowledge scope. - Only searches knowledge bases and files that are attached to the current model. - - :param query: The filename search query - :param knowledge_id: Optional KB id to limit search to a specific attached knowledge base - :param count: Maximum number of results to return (default: 10) - :param skip: Number of results to skip for pagination (default: 0) - :return: JSON with matching files containing id, filename, knowledge_id, and knowledge_name - """ - if __request__ is None: - return json.dumps({'error': 'Request context not available'}) - - if not __user__: - return json.dumps({'error': 'User context not available'}) - - if not __model_knowledge__: - return json.dumps([]) - - try: - from open_webui.models.knowledge import Knowledges - from open_webui.models.files import Files - from open_webui.models.access_grants import AccessGrants - - user_id = __user__.get('id') - user_role = __user__.get('role', 'user') - user_group_ids = [group.id for group in Groups.get_groups_by_member_id(user_id)] - - # Collect attached KB IDs and direct file IDs - attached_kb_ids = set() - attached_file_ids = set() - - for item in __model_knowledge__: - item_type = item.get('type') - item_id = item.get('id') - if item_type == 'collection': - attached_kb_ids.add(item_id) - elif item_type == 'file': - attached_file_ids.add(item_id) - - # If knowledge_id is specified, verify it's in the attached set - if knowledge_id: - if knowledge_id not in attached_kb_ids: - return json.dumps({'error': f'Knowledge base {knowledge_id} is not attached to this model'}) - attached_kb_ids = {knowledge_id} - - all_files = [] - - # Search within attached KBs - for kb_id in attached_kb_ids: - knowledge = Knowledges.get_knowledge_by_id(kb_id) - if not knowledge: - continue - - if not ( - user_role == 'admin' - or knowledge.user_id == user_id - or AccessGrants.has_access( - user_id=user_id, - resource_type='knowledge', - resource_id=knowledge.id, - permission='read', - user_group_ids=set(user_group_ids), - ) - ): - continue - - result = Knowledges.search_files_by_id( - knowledge_id=kb_id, - user_id=user_id, - filter={'query': query}, - skip=0, - limit=count + skip, # Fetch enough for pagination across KBs - ) - - for file in result.items: - all_files.append({ - 'id': file.id, - 'filename': file.filename, - 'knowledge_id': knowledge.id, - 'knowledge_name': knowledge.name, - 'updated_at': file.updated_at, - }) - - # Search within directly attached files (filename match) - if not knowledge_id and attached_file_ids: - query_lower = query.lower() if query else '' - for file_id in attached_file_ids: - file = Files.get_file_by_id(file_id) - if file and (not query_lower or query_lower in file.filename.lower()): - all_files.append({ - 'id': file.id, - 'filename': file.filename, - 'updated_at': file.updated_at, - }) - - # Apply pagination across combined results - all_files = all_files[skip:skip + count] - - return json.dumps(all_files, ensure_ascii=False) - except Exception as e: - log.exception(f'search_attached_files error: {e}') + log.exception(f'list_knowledge error: {e}') return json.dumps({'error': str(e)}) diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index 6f24739b0d..5d88facce2 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -79,8 +79,7 @@ from open_webui.tools.builtin import ( query_knowledge_bases, search_knowledge_files, query_knowledge_files, - list_attached_knowledge, - search_attached_files, + list_knowledge, view_file, view_knowledge_file, view_skill, @@ -408,8 +407,8 @@ def get_builtin_tools( if is_builtin_tool_enabled('knowledge'): if model_knowledge: # Model has attached knowledge - provide discovery, search and semantic tools - builtin_functions.append(list_attached_knowledge) - builtin_functions.append(search_attached_files) + builtin_functions.append(list_knowledge) + builtin_functions.append(search_knowledge_files) builtin_functions.append(query_knowledge_files) knowledge_types = {item.get('type') for item in model_knowledge} From 36c3fc58b54953785715e0d141eee3e344572859 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Mon, 23 Mar 2026 19:29:34 -0500 Subject: [PATCH 004/125] refac --- src/lib/components/chat/ChatControls.svelte | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/components/chat/ChatControls.svelte b/src/lib/components/chat/ChatControls.svelte index aea9fa83cf..d531449e7b 100644 --- a/src/lib/components/chat/ChatControls.svelte +++ b/src/lib/components/chat/ChatControls.svelte @@ -95,10 +95,12 @@ showControls.set(true); } - // Auto-open Files tab when a terminal is selected + // Auto-open Files tab when a terminal is selected (suppress panel open when full-screen) $: if ($selectedTerminalId) { activeTab = 'files'; - showControls.set(true); + if (largeScreen) { + showControls.set(true); + } } // Attach a terminal file to the chat input From 1c25b06dca83ad491b4dc3d373b1c215a7a8fd3e Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Mon, 23 Mar 2026 19:46:24 -0500 Subject: [PATCH 005/125] refac --- backend/open_webui/utils/files.py | 2 +- backend/open_webui/utils/middleware.py | 72 ++++++++++++++++++++++---- backend/open_webui/utils/misc.py | 34 +++++++++--- backend/open_webui/utils/tools.py | 27 ++++++++-- 4 files changed, 110 insertions(+), 25 deletions(-) diff --git a/backend/open_webui/utils/files.py b/backend/open_webui/utils/files.py index 3bb918e8da..06bec33250 100644 --- a/backend/open_webui/utils/files.py +++ b/backend/open_webui/utils/files.py @@ -148,7 +148,7 @@ def get_audio_url_from_base64(request, base64_audio_string, metadata, user): def get_file_url_from_base64(request, base64_file_string, metadata, user): - if 'data:image/png;base64' in base64_file_string: + if BASE64_IMAGE_URL_PREFIX.match(base64_file_string): return get_image_url_from_base64(request, base64_file_string, metadata, user) elif 'data:audio/wav;base64' in base64_file_string: return get_audio_url_from_base64(request, base64_file_string, metadata, user) diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index cbf644b53c..058c8db169 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -1049,6 +1049,13 @@ def process_tool_result( tool_result_files = [] + # Detect base64 image data URIs from tool results (e.g. binary image + # responses from execute_tool_server). Move the data URI to + # tool_result_files and replace tool_result with a text summary. + if isinstance(tool_result, str) and tool_result.startswith('data:image/'): + tool_result_files.append({'type': 'image', 'url': tool_result}) + tool_result = f'{tool_function_name}: Image file read successfully.' + if isinstance(tool_result, list): if tool_type == 'mcp': # MCP tool_response = [] @@ -4181,19 +4188,27 @@ async def streaming_chat_response_handler(response, ctx): break for result in results: + output_parts = [{'type': 'input_text', 'text': result.get('content', '')}] + + # Separate image data URIs (for LLM via input_image) from + # other files (for frontend display via files attribute). + display_files = [] + for file_item in result.get('files', []): + if file_item.get('type') == 'image' and file_item.get('url', '').startswith('data:'): + # LLM-only: add as input_image part (invisible to serialize_output) + output_parts.append({'type': 'input_image', 'image_url': file_item['url']}) + else: + # Frontend display (MCP images, audio, etc.) + display_files.append(file_item) + output.append( { 'type': 'function_call_output', 'id': output_id('fco'), 'call_id': result.get('tool_call_id', ''), - 'output': [ - { - 'type': 'input_text', - 'text': result.get('content', ''), - } - ], + 'output': output_parts, 'status': 'completed', - **({'files': result.get('files')} if result.get('files') else {}), + **({'files': display_files} if display_files else {}), **({'embeds': result.get('embeds')} if result.get('embeds') else {}), } ) @@ -4262,12 +4277,23 @@ async def streaming_chat_response_handler(response, ctx): ) tool_call_sources.clear() + # Strip input_image parts (large base64 data URIs) from the + # output sent to the frontend — they're only for LLM consumption + # via convert_output_to_messages. + frontend_output = [] + for item in output: + if item.get('type') == 'function_call_output': + parts = item.get('output', []) + if any(p.get('type') == 'input_image' for p in parts): + item = {**item, 'output': [p for p in parts if p.get('type') != 'input_image']} + frontend_output.append(item) + await event_emitter( { 'type': 'chat:completion', 'data': { 'content': serialize_output(output), - 'output': output, + 'output': frontend_output, }, } ) @@ -4287,11 +4313,35 @@ async def streaming_chat_response_handler(response, ctx): ) new_form_data['previous_response_id'] = last_response_id else: + tool_messages = convert_output_to_messages(output, raw=True) + + # Chat Completions providers don't support multimodal + # tool messages. Extract images into a user message. + image_urls = [] + for message in tool_messages: + if message.get('role') == 'tool' and isinstance(message.get('content'), list): + text_parts = [] + for part in message['content']: + if part.get('type') == 'input_text': + text_parts.append(part.get('text', '')) + elif part.get('type') == 'input_image': + image_urls.append(part.get('image_url', '')) + message['content'] = ''.join(text_parts) + new_form_data['messages'] = [ *form_data['messages'], - *convert_output_to_messages(output, raw=True), + *tool_messages, ] + if image_urls: + new_form_data['messages'].append({ + 'role': 'user', + 'content': [ + {'type': 'text', 'text': 'Here are the images from the tool results above. Please analyze them.'}, + *[{'type': 'image_url', 'image_url': {'url': url}} for url in image_urls], + ], + }) + res = await generate_chat_completion( request, new_form_data, @@ -4416,7 +4466,7 @@ async def streaming_chat_response_handler(response, ctx): if isinstance(stdout, str): stdoutLines = stdout.split('\n') for idx, line in enumerate(stdoutLines): - if 'data:image/png;base64' in line: + if re.match(r'data:image/\w+;base64', line): image_url = get_image_url_from_base64( request, line, @@ -4433,7 +4483,7 @@ async def streaming_chat_response_handler(response, ctx): if isinstance(result, str): resultLines = result.split('\n') for idx, line in enumerate(resultLines): - if 'data:image/png;base64' in line: + if re.match(r'data:image/\w+;base64', line): image_url = get_image_url_from_base64( request, line, diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index 91217b3c23..060d5c4a6a 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -196,21 +196,39 @@ def convert_output_to_messages(output: list, raw: bool = False) -> list[dict]: # Flush any pending content/tool_calls before adding tool result flush_pending() - # Extract text from output content parts + # Extract text and images from output content parts output_parts = item.get('output', []) content = '' + image_urls = [] for part in output_parts: if part.get('type') == 'input_text': output_text = part.get('text', '') content += str(output_text) if not isinstance(output_text, str) else output_text + elif part.get('type') == 'input_image': + url = part.get('image_url', '') + if url: + image_urls.append(url) - messages.append( - { - 'role': 'tool', - 'tool_call_id': item.get('call_id', ''), - 'content': content, - } - ) + if image_urls: + # Multimodal tool content with image(s) + messages.append( + { + 'role': 'tool', + 'tool_call_id': item.get('call_id', ''), + 'content': [ + {'type': 'input_text', 'text': content}, + *[{'type': 'input_image', 'image_url': url} for url in image_urls], + ], + } + ) + else: + messages.append( + { + 'role': 'tool', + 'tool_call_id': item.get('call_id', ''), + 'content': content, + } + ) elif item_type == 'reasoning': if raw: diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index 5d88facce2..e95942f94d 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -1,3 +1,4 @@ +import base64 import inspect import logging import re @@ -1236,9 +1237,13 @@ async def execute_tool_server( if param_name in params: if param_in == 'path': path_params[param_name] = params[param_name] - elif param_in == 'query': - if params[param_name] is not None: - query_params[param_name] = params[param_name] + if param_in == 'query': + value = params[param_name] + # Skip empty values for optional params (LLMs sometimes + # pass "" instead of omitting optional parameters). + if value is None or (value == '' and not param.get('required')): + continue + query_params[param_name] = value final_url = f'{url.rstrip("/")}{route_path}' for key, value in path_params.items(): @@ -1273,7 +1278,13 @@ async def execute_tool_server( try: response_data = await response.json() except Exception: - response_data = await response.text() + content_type = response.headers.get('Content-Type', '').split(';')[0].strip() + if content_type.startswith('text/') or not content_type: + response_data = await response.text() + else: + raw = await response.read() + b64 = base64.b64encode(raw).decode() + response_data = f'data:{content_type};base64,{b64}' response_headers = response.headers return (response_data, response_headers) @@ -1292,7 +1303,13 @@ async def execute_tool_server( try: response_data = await response.json() except Exception: - response_data = await response.text() + content_type = response.headers.get('Content-Type', '').split(';')[0].strip() + if content_type.startswith('text/') or not content_type: + response_data = await response.text() + else: + raw = await response.read() + b64 = base64.b64encode(raw).decode() + response_data = f'data:{content_type};base64,{b64}' response_headers = response.headers return (response_data, response_headers) From 108a019cb8e63a533250abe84f2b6f2b7c2131c4 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Mon, 23 Mar 2026 19:58:32 -0500 Subject: [PATCH 006/125] refac --- backend/open_webui/routers/openai.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/open_webui/routers/openai.py b/backend/open_webui/routers/openai.py index 3aa121913e..c42b9163f9 100644 --- a/backend/open_webui/routers/openai.py +++ b/backend/open_webui/routers/openai.py @@ -1124,6 +1124,15 @@ async def generate_chat_completion( request_url = f'{url}/responses' else: request_url = f'{url}/chat/completions' + # For Chat Completions, strip image parts from multimodal tool messages + # (Chat Completions doesn't support images in tool content). + if not is_responses and 'messages' in payload: + for message in payload['messages']: + if message.get('role') == 'tool' and isinstance(message.get('content'), list): + message['content'] = ''.join( + part.get('text', '') for part in message['content'] + if part.get('type') in ('input_text', 'text') + ) payload = json.dumps(payload) From a3238aa79f344765f5b62cb64eba71ffd001abaf Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Mon, 23 Mar 2026 20:42:48 -0500 Subject: [PATCH 007/125] refac --- backend/open_webui/env.py | 10 ++++++++++ backend/open_webui/utils/tools.py | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index e891f1d39f..f0dbf9c114 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -760,6 +760,16 @@ AIOHTTP_CLIENT_SESSION_TOOL_SERVER_SSL = ( os.environ.get('AIOHTTP_CLIENT_SESSION_TOOL_SERVER_SSL', 'True').lower() == 'true' ) +AIOHTTP_CLIENT_TIMEOUT_TOOL_SERVER = os.environ.get('AIOHTTP_CLIENT_TIMEOUT_TOOL_SERVER', '') + +if AIOHTTP_CLIENT_TIMEOUT_TOOL_SERVER == '': + AIOHTTP_CLIENT_TIMEOUT_TOOL_SERVER = AIOHTTP_CLIENT_TIMEOUT +else: + try: + AIOHTTP_CLIENT_TIMEOUT_TOOL_SERVER = int(AIOHTTP_CLIENT_TIMEOUT_TOOL_SERVER) + except Exception: + AIOHTTP_CLIENT_TIMEOUT_TOOL_SERVER = AIOHTTP_CLIENT_TIMEOUT + RAG_EMBEDDING_TIMEOUT = os.environ.get('RAG_EMBEDDING_TIMEOUT', '') diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index e95942f94d..d4d5e9e49f 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -45,6 +45,7 @@ from open_webui.utils.access_control import has_access, has_connection_access from open_webui.config import BYPASS_ADMIN_ACCESS_CONTROL from open_webui.env import ( AIOHTTP_CLIENT_TIMEOUT, + AIOHTTP_CLIENT_TIMEOUT_TOOL_SERVER, AIOHTTP_CLIENT_TIMEOUT_TOOL_SERVER_DATA, AIOHTTP_CLIENT_SESSION_TOOL_SERVER_SSL, ENABLE_FORWARD_USER_INFO_HEADERS, @@ -1187,6 +1188,7 @@ async def get_tool_servers_data(servers: List[Dict[str, Any]]) -> List[Dict[str, return results + async def execute_tool_server( url: str, headers: Dict[str, str], @@ -1258,7 +1260,7 @@ async def execute_tool_server( body_params = params async with aiohttp.ClientSession( - trust_env=True, timeout=aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT) + trust_env=True, timeout=aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT_TOOL_SERVER) ) as session: request_method = getattr(session, http_method.lower()) From 689061822173e561a153290b2bb816f4cb6f4959 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Mon, 23 Mar 2026 21:14:22 -0500 Subject: [PATCH 008/125] refac --- backend/open_webui/routers/terminals.py | 6 + src/lib/components/chat/FileNav.svelte | 25 +- .../components/chat/FileNav/PortList.svelte | 47 ++- .../chat/FileNav/PortPreview.svelte | 321 ++++++++++++++++++ 4 files changed, 378 insertions(+), 21 deletions(-) create mode 100644 src/lib/components/chat/FileNav/PortPreview.svelte diff --git a/backend/open_webui/routers/terminals.py b/backend/open_webui/routers/terminals.py index 49c39c8bf7..59f1f3ab48 100644 --- a/backend/open_webui/routers/terminals.py +++ b/backend/open_webui/routers/terminals.py @@ -31,14 +31,20 @@ def _sanitize_proxy_path(path: str) -> str | None: """Sanitize a proxy path to prevent directory traversal / SSRF. Returns the cleaned path, or None if the path is invalid. + Trailing slashes are preserved — many upstream frameworks treat + ``/path`` and ``/path/`` differently. """ decoded = unquote(path) + had_trailing_slash = decoded.endswith('/') normalized = posixpath.normpath(decoded) # Remove any leading slashes that would reset the base cleaned = normalized.lstrip('/') # Reject if normpath resolved to parent traversal or current-dir only if cleaned.startswith('..') or cleaned == '.': return None + # Restore trailing slash if the original path had one + if had_trailing_slash and cleaned and not cleaned.endswith('/'): + cleaned += '/' return cleaned diff --git a/src/lib/components/chat/FileNav.svelte b/src/lib/components/chat/FileNav.svelte index 770d2b270c..adb8bdf6d4 100644 --- a/src/lib/components/chat/FileNav.svelte +++ b/src/lib/components/chat/FileNav.svelte @@ -40,6 +40,7 @@ import FilePreview from './FileNav/FilePreview.svelte'; import FileEntryRow from './FileNav/FileEntryRow.svelte'; import PortList from './FileNav/PortList.svelte'; + import PortPreview from './FileNav/PortPreview.svelte'; import XTerminal from './XTerminal.svelte'; const i18n = getContext('i18n'); @@ -90,6 +91,7 @@ // ── File preview state ─────────────────────────────────────────────── let selectedFile: string | null = null; + let previewPort: number | null = null; let fileContent: string | null = null; let fileImageUrl: string | null = null; let fileVideoUrl: string | null = null; @@ -253,6 +255,7 @@ loading = true; error = null; selectedFile = null; + previewPort = null; clearFilePreview(); currentPath = path; savedPath = path; @@ -632,6 +635,7 @@ {/if} + {#if previewPort === null} + {/if}
- {#if selectedFile !== null} + {#if previewPort !== null} + { previewPort = null; }} + /> + {:else if selectedFile !== null} - {#if selectedTerminal && !selectedFile} + {#if selectedTerminal && !selectedFile && previewPort === null}
- + { + selectedFile = null; + clearFilePreview(); + previewPort = e.detail; + }} + />
{/if} diff --git a/src/lib/components/chat/FileNav/PortList.svelte b/src/lib/components/chat/FileNav/PortList.svelte index 8a5dfb3cd2..9e85951126 100644 --- a/src/lib/components/chat/FileNav/PortList.svelte +++ b/src/lib/components/chat/FileNav/PortList.svelte @@ -1,10 +1,11 @@ + +
+ +
+ + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + + +
+ + + {#if isLoading} +
+
+
+ {/if} + + + {#key iframeKey} +