From 088830cf82f6ac430653079668fc2c66d830f164 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Sat, 10 Oct 2026 22:43:52 +0400 Subject: [PATCH] feat: Dynamic Tool Loading / Lazy Loading Co-Authored-By: Taylor Wilsdon <6508528+taylorwilsdon@users.noreply.github.com> --- backend/open_webui/config.py | 14 ++ backend/open_webui/routers/chats.py | 12 ++ backend/open_webui/tools/builtin.py | 30 ++++ backend/open_webui/utils/middleware.py | 7 +- backend/open_webui/utils/subagents.py | 5 +- backend/open_webui/utils/timers.py | 5 +- backend/open_webui/utils/tool_search.py | 159 ++++++++++++++++++ .../admin/Settings/Interface.svelte | 70 +++++++- src/lib/i18n/locales/en-US/translation.json | 9 + 9 files changed, 306 insertions(+), 5 deletions(-) create mode 100644 backend/open_webui/utils/tool_search.py diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index c828e847ca..5726327747 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -2224,6 +2224,16 @@ CONTEXT_COMPACTION_RETENTION_PERCENTAGE = min( CONTEXT_COMPACTION_PROMPT_TEMPLATE = os.getenv('CONTEXT_COMPACTION_PROMPT_TEMPLATE', '') +ENABLE_TOOL_SEARCH = os.getenv('ENABLE_TOOL_SEARCH', 'False').lower() == 'true' + +TOOL_SEARCH_DEFER_THRESHOLD = int(os.getenv('TOOL_SEARCH_DEFER_THRESHOLD', '400')) + +TOOL_SEARCH_ALWAYS_LOADED = [ + item.strip() for item in os.getenv('TOOL_SEARCH_ALWAYS_LOADED', '').split(',') if item.strip() +] + +TOOL_SEARCH_DEFER_BUILTIN_TOOLS = os.getenv('TOOL_SEARCH_DEFER_BUILTIN_TOOLS', 'True').lower() == 'true' + TITLE_GENERATION_PROMPT_TEMPLATE = os.getenv('TITLE_GENERATION_PROMPT_TEMPLATE', '') DEFAULT_TITLE_GENERATION_PROMPT_TEMPLATE = """### Task: @@ -3185,6 +3195,10 @@ DEFAULT_CONFIG = { 'chat.context_compaction.retention_percentage': CONTEXT_COMPACTION_RETENTION_PERCENTAGE, 'chat.context_compaction.prompt_template': CONTEXT_COMPACTION_PROMPT_TEMPLATE, 'chat.tool_permissions.enable': ENABLE_TOOL_PERMISSIONS, + 'chat.tool_search.enable': ENABLE_TOOL_SEARCH, + 'chat.tool_search.defer_threshold': TOOL_SEARCH_DEFER_THRESHOLD, + 'chat.tool_search.always_loaded': TOOL_SEARCH_ALWAYS_LOADED, + 'chat.tool_search.defer_builtin_tools': TOOL_SEARCH_DEFER_BUILTIN_TOOLS, 'task.title.prompt_template': TITLE_GENERATION_PROMPT_TEMPLATE, 'task.tags.prompt_template': TAGS_GENERATION_PROMPT_TEMPLATE, 'task.image.prompt_template': IMAGE_PROMPT_GENERATION_PROMPT_TEMPLATE, diff --git a/backend/open_webui/routers/chats.py b/backend/open_webui/routers/chats.py index 9f469fabf9..2a4e8ca033 100644 --- a/backend/open_webui/routers/chats.py +++ b/backend/open_webui/routers/chats.py @@ -62,6 +62,10 @@ CHAT_CONFIG_KEYS = { 'CONTEXT_COMPACTION_RETENTION_PERCENTAGE': 'chat.context_compaction.retention_percentage', 'CONTEXT_COMPACTION_PROMPT_TEMPLATE': 'chat.context_compaction.prompt_template', 'ENABLE_TOOL_PERMISSIONS': 'chat.tool_permissions.enable', + 'ENABLE_TOOL_SEARCH': 'chat.tool_search.enable', + 'TOOL_SEARCH_DEFER_THRESHOLD': 'chat.tool_search.defer_threshold', + 'TOOL_SEARCH_ALWAYS_LOADED': 'chat.tool_search.always_loaded', + 'TOOL_SEARCH_DEFER_BUILTIN_TOOLS': 'chat.tool_search.defer_builtin_tools', } @@ -202,6 +206,10 @@ class ChatConfigForm(BaseModel): CONTEXT_COMPACTION_RETENTION_PERCENTAGE: int | None = None CONTEXT_COMPACTION_PROMPT_TEMPLATE: str ENABLE_TOOL_PERMISSIONS: bool = False + ENABLE_TOOL_SEARCH: bool = False + TOOL_SEARCH_DEFER_THRESHOLD: int = 400 + TOOL_SEARCH_ALWAYS_LOADED: list[str] = [] + TOOL_SEARCH_DEFER_BUILTIN_TOOLS: bool = True class CompactChatForm(BaseModel): @@ -892,6 +900,8 @@ async def set_chat_config(form_data: ChatConfigForm, user=Depends(get_admin_user if retention_percentage is None: retention_percentage = CONTEXT_COMPACTION_RETENTION_PERCENTAGE retention_percentage = min(50, max(10, int(retention_percentage))) + tool_search_defer_threshold = max(0, int(form_data.TOOL_SEARCH_DEFER_THRESHOLD)) + tool_search_always_loaded = [item.strip() for item in form_data.TOOL_SEARCH_ALWAYS_LOADED if item.strip()] await Config.upsert( chat_config_updates( { @@ -900,6 +910,8 @@ async def set_chat_config(form_data: ChatConfigForm, user=Depends(get_admin_user 'CONTEXT_COMPACTION_TOKEN_THRESHOLD': threshold, 'CONTEXT_COMPACTION_TOKEN_CAP': token_cap, 'CONTEXT_COMPACTION_RETENTION_PERCENTAGE': retention_percentage, + 'TOOL_SEARCH_DEFER_THRESHOLD': tool_search_defer_threshold, + 'TOOL_SEARCH_ALWAYS_LOADED': tool_search_always_loaded, } ) ) diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py index b191326d9e..8fd7cfa487 100644 --- a/backend/open_webui/tools/builtin.py +++ b/backend/open_webui/tools/builtin.py @@ -3734,6 +3734,36 @@ async def update_skill_files( return JSONCodec.dumps({'error': getattr(error, 'detail', str(error))}) +# ============================================================================= +# TOOL SEARCH +# ============================================================================= + + +async def search_tools( + query: str, + count: int = 5, + __metadata__: dict = None, +) -> str: + """ + Search the tools listed in and return their full definitions. + Pass the exact tool name when you already know it. + + :param query: Keywords describing the capability you need (e.g. "jira create issue"), or an exact tool name + :param count: Maximum number of results to return (default: 5, max: 20) + :return: JSON with the definitions of the matching tools, which can then be called by name + """ + from open_webui.utils.tool_search import search_deferred_tools + + tools = __metadata__['tools'] + candidates = {name: tools[name]['spec'] for name in __metadata__['deferred_tools']} + matches = search_deferred_tools(query, candidates, count) + if not matches: + return JSONCodec.dumps( + {'tools': [], 'message': 'No matching tools found. Try different keywords or the exact tool name.'} + ) + return JSONCodec.dumps({'tools': [candidates[name] for name in matches]}) + + # ============================================================================= # TASK MANAGEMENT TOOLS # ============================================================================= diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index 80b364f574..e8a12e3e35 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -144,6 +144,7 @@ from open_webui.utils.task import ( tools_function_calling_generation_template, ) from open_webui.utils.tool_approval import complete_tool_call, pending_tool_calls +from open_webui.utils.tool_search import apply_tool_search from open_webui.utils.tools import ( connect_mcp_server, get_attached_knowledge, @@ -3281,9 +3282,13 @@ async def process_chat_payload(request, form_data, user, metadata, model): metadata['tools'] = tools_dict if metadata.get('params', {}).get('function_calling') != 'legacy': + deferred = await apply_tool_search(form_data, metadata, tools_dict) + # If the function calling is native, then call the tools function calling handler form_data['tools'] = [ - {'type': 'function', 'function': tool.get('spec', {})} for tool in tools_dict.values() + {'type': 'function', 'function': tool.get('spec', {})} + for name, tool in tools_dict.items() + if name not in deferred ] if inlet_filter_tools: form_data['tools'].extend(inlet_filter_tools) diff --git a/backend/open_webui/utils/subagents.py b/backend/open_webui/utils/subagents.py index 916c078cc0..f047ecabec 100644 --- a/backend/open_webui/utils/subagents.py +++ b/backend/open_webui/utils/subagents.py @@ -19,6 +19,7 @@ from open_webui.tasks import create_task, has_active_tasks from open_webui.utils.auth import VERIFIED_USER_ROLES, create_token from open_webui.utils.json_codec import JSONCodec from open_webui.utils.misc import add_or_update_system_message, get_message_list +from open_webui.utils.tool_search import strip_deferred_tools_manifest from sqlalchemy import select from starlette.datastructures import Headers @@ -249,7 +250,9 @@ async def process_pending_internal_messages( chat_context = copy.deepcopy(run.get('chat_context') or {}) # Runs saved before chat_context was introduced only have the assembled prompt. if run.get('chat_context') is None and run.get('system_prompt'): - chat_context['messages'] = [{'role': 'system', 'content': run['system_prompt']}] + chat_context['messages'] = [ + {'role': 'system', 'content': strip_deferred_tools_manifest(run['system_prompt'])} + ] form_data = { 'model': model_id, 'params': chat_context.get('params') or {}, diff --git a/backend/open_webui/utils/timers.py b/backend/open_webui/utils/timers.py index bfd1829c2f..0b8ccb076f 100644 --- a/backend/open_webui/utils/timers.py +++ b/backend/open_webui/utils/timers.py @@ -20,6 +20,7 @@ from open_webui.tasks import has_active_tasks from open_webui.utils.auth import VERIFIED_USER_ROLES from open_webui.utils.json_codec import JSONCodec from open_webui.utils.misc import get_message_list +from open_webui.utils.tool_search import strip_deferred_tools_manifest from sqlalchemy import select from starlette.datastructures import Headers @@ -377,7 +378,9 @@ async def execute_due_timer(app, timer_id: str, claim_id: str | None = None) -> chat_context = copy.deepcopy(run.get('chat_context') or {}) # Timers saved before chat_context was introduced only have the assembled prompt. if run.get('chat_context') is None and run.get('system_prompt'): - chat_context['messages'] = [{'role': 'system', 'content': run['system_prompt']}] + chat_context['messages'] = [ + {'role': 'system', 'content': strip_deferred_tools_manifest(run['system_prompt'])} + ] form_data = { 'model': model_id, 'params': chat_context.get('params') or {}, diff --git a/backend/open_webui/utils/tool_search.py b/backend/open_webui/utils/tool_search.py new file mode 100644 index 0000000000..8bff23e62c --- /dev/null +++ b/backend/open_webui/utils/tool_search.py @@ -0,0 +1,159 @@ +""" +Tool search: large tool schemas are left out of the provider `tools` array and listed by name in an +`` system prompt block. `search_tools` returns the full definitions of matching tools, +which the model then calls by name. The `tools` array never changes during a chat, so prompt caches hold. +""" + +import fnmatch +import re +import unicodedata + +import regex +from open_webui.models.config import Config +from open_webui.utils.json_codec import JSONCodec +from open_webui.utils.misc import add_or_update_system_message + +SEARCH_TOOL_NAME = 'search_tools' +MANIFEST_DESCRIPTION_MAX_CHARS = 100 +MAX_SEARCH_COUNT = 20 +DEFAULT_SEARCH_COUNT = 5 + +_MANIFEST_RE = re.compile(r'\n?\n.*?', re.DOTALL) + + +async def get_tool_search_config() -> dict | None: + """Tool search settings, or None when the feature is off.""" + if not await Config.get('chat.tool_search.enable'): + return None + values = await Config.get_many( + 'chat.tool_search.defer_threshold', + 'chat.tool_search.always_loaded', + 'chat.tool_search.defer_builtin_tools', + ) + return { + 'defer_threshold': values['chat.tool_search.defer_threshold'], + 'always_loaded': values['chat.tool_search.always_loaded'], + 'defer_builtin_tools': values['chat.tool_search.defer_builtin_tools'], + } + + +def select_deferred_tools(tools_dict: dict[str, dict], config: dict) -> list[str]: + """Sorted names of tools whose schema exceeds the threshold and that are not always loaded.""" + patterns = config['always_loaded'] + return sorted( + name + for name, tool in tools_dict.items() + if (config['defer_builtin_tools'] or tool.get('type') != 'builtin') + and not any(fnmatch.fnmatchcase(name, pattern) for pattern in patterns) + and len(JSONCodec.dumps(tool.get('spec') or {})) > config['defer_threshold'] + ) + + +def _truncate_description(description: str | None) -> str: + text = ' '.join(str(description or '').split()) + if len(text) > MANIFEST_DESCRIPTION_MAX_CHARS: + text = text[: MANIFEST_DESCRIPTION_MAX_CHARS - 1].rstrip() + '…' + return text + + +def build_deferred_tools_manifest(tools_dict: dict[str, dict], deferred: list[str]) -> str: + entries = '' + for name in deferred: + description = _truncate_description(tools_dict[name].get('spec', {}).get('description')) + entries += f'\n{name}\n{description}\n\n' + + return ( + '\n' + 'The following tools are available but their definitions are not loaded. ' + f'To use one, call `{SEARCH_TOOL_NAME}` with a short keyword query or the exact tool name to get its ' + 'definition, then call the tool by name with the parameters it defines. ' + 'Never tell the user a capability is unavailable without searching first.\n' + f'{entries}' + ) + + +async def apply_tool_search(form_data: dict, metadata: dict, tools_dict: dict[str, dict]) -> set[str]: + """Add search_tools and the manifest when tools can be deferred; returns the names to leave out of `tools`.""" + config = SEARCH_TOOL_NAME not in tools_dict and await get_tool_search_config() + deferred = select_deferred_tools(tools_dict, config) if config else [] + if not deferred: + return set() + + from open_webui.tools.builtin import search_tools + from open_webui.utils.tools import ( + get_async_tool_function_and_apply_extra_params, + get_builtin_function_introspection, + get_builtin_tool_spec, + ) + + metadata['deferred_tools'] = deferred + form_data['messages'] = add_or_update_system_message( + build_deferred_tools_manifest(tools_dict, deferred), + form_data['messages'], + append=True, + ) + tools_dict[SEARCH_TOOL_NAME] = { + 'tool_id': f'builtin:{SEARCH_TOOL_NAME}', + 'callable': await get_async_tool_function_and_apply_extra_params( + search_tools, {'__metadata__': metadata}, get_builtin_function_introspection(search_tools) + ), + 'spec': get_builtin_tool_spec(search_tools), + 'type': 'builtin', + } + return set(deferred) + + +def strip_deferred_tools_manifest(system_prompt: str | None) -> str | None: + """Remove the manifest from a system prompt that is reused by a request which builds its own.""" + return _MANIFEST_RE.sub('', system_prompt) if system_prompt else system_prompt + + +# Letters, digits and combining marks, so vowel signs in scripts like Devanagari stay attached to their word. +_WORD_RE = regex.compile(r'[\p{L}\p{N}\p{M}]+') +_CAMEL_RE = re.compile(r'(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])') +# Scripts written without spaces between words: Han, Hiragana, Katakana and Hangul. +_CJK_RE = re.compile(r'([\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af]+)') + + +def tokenize(text: str | None) -> list[str]: + """Lowercase word tokens split on camelCase boundaries, with CJK runs split into character bigrams.""" + tokens = [] + for word in _WORD_RE.findall(unicodedata.normalize('NFKC', text or '')): + for part in _CAMEL_RE.split(word): + for chunk in _CJK_RE.split(part.casefold()): + if _CJK_RE.fullmatch(chunk): + tokens.extend([chunk[i : i + 2] for i in range(len(chunk) - 1)] or [chunk]) + elif chunk: + tokens.append(chunk) + return tokens + + +def _document_text(name: str, spec: dict) -> str: + properties = (spec.get('parameters') or {}).get('properties') or {} + return ' '.join([name, str(spec.get('description') or ''), *properties.keys()]) + + +def search_deferred_tools(query: str, candidates: dict[str, dict], count: int = DEFAULT_SEARCH_COUNT) -> list[str]: + """Rank candidate tools (name -> spec) against the query; an exact tool name wins outright.""" + from rank_bm25 import BM25Okapi + + query = (query or '').strip() + if query in candidates: + return [query] + + query_tokens = tokenize(query) + if not candidates or not query_tokens: + return [] + + names = list(candidates) + documents = [tokenize(_document_text(name, candidates[name])) or ['_'] for name in names] + # BM25 IDF is zero for a term found in half the corpus, common with small tool sets, + # so raw token overlap keeps every matching term rankable. + bm25 = BM25Okapi(documents).get_scores(query_tokens) + scores = { + name: max(0.0, float(bm25[index])) + len(set(query_tokens) & set(documents[index])) + for index, name in enumerate(names) + } + + ranked = sorted((name for name in names if scores[name] > 0), key=lambda name: (-scores[name], name)) + return ranked[: max(1, min(count, MAX_SEARCH_COUNT))] diff --git a/src/lib/components/admin/Settings/Interface.svelte b/src/lib/components/admin/Settings/Interface.svelte index 99daa73e5e..f3e581e9c3 100644 --- a/src/lib/components/admin/Settings/Interface.svelte +++ b/src/lib/components/admin/Settings/Interface.svelte @@ -50,8 +50,13 @@ CONTEXT_COMPACTION_TOKEN_CAP: 80000, CONTEXT_COMPACTION_RETENTION_PERCENTAGE: 40, CONTEXT_COMPACTION_PROMPT_TEMPLATE: '', - ENABLE_TOOL_PERMISSIONS: false + ENABLE_TOOL_PERMISSIONS: false, + ENABLE_TOOL_SEARCH: false, + TOOL_SEARCH_DEFER_THRESHOLD: 400, + TOOL_SEARCH_ALWAYS_LOADED: [] as string[], + TOOL_SEARCH_DEFER_BUILTIN_TOOLS: true }; + let toolSearchAlwaysLoaded = ''; let showTaskParameters = false; const configuredParams = (params: Record = {}) => @@ -70,7 +75,13 @@ try { [taskConfig, chatConfig] = await Promise.all([ updateTaskConfig(localStorage.token, taskConfigPayload), - updateChatConfig(localStorage.token, chatConfig) + updateChatConfig(localStorage.token, { + ...chatConfig, + TOOL_SEARCH_ALWAYS_LOADED: toolSearchAlwaysLoaded + .split(',') + .map((item) => item.trim()) + .filter((item) => item !== '') + }) ]); } catch (error) { toast.error( @@ -78,6 +89,7 @@ ); return; } + toolSearchAlwaysLoaded = (chatConfig.TOOL_SEARCH_ALWAYS_LOADED ?? []).join(', '); appConfig.update((current) => current ? { @@ -131,6 +143,7 @@ getChatConfig(localStorage.token) ]); taskConfig.TASK_MODEL_PARAMS = taskConfig.TASK_MODEL_PARAMS ?? {}; + toolSearchAlwaysLoaded = (chatConfig.TOOL_SEARCH_ALWAYS_LOADED ?? []).join(', '); workspaceModels = await getBaseModels(localStorage.token); baseModels = await getModels(localStorage.token, null, false); @@ -369,6 +382,59 @@ {/if} + + +
+ {$i18n.t('settings.admin.interface.toolSearch.label')} + +
+ + + + {#if chatConfig.ENABLE_TOOL_SEARCH} + + + + + + + + + + + + {/if} diff --git a/src/lib/i18n/locales/en-US/translation.json b/src/lib/i18n/locales/en-US/translation.json index 413f72e4fb..dd6464db62 100644 --- a/src/lib/i18n/locales/en-US/translation.json +++ b/src/lib/i18n/locales/en-US/translation.json @@ -2793,6 +2793,15 @@ "settings.admin.interface.tokenThreshold.label": "Token Threshold", "settings.admin.interface.toolPermissions.description": "Show Full access and Ask for approval in the chat input menu.", "settings.admin.interface.toolPermissions.label": "Tool Permissions", + "settings.admin.interface.toolSearch.description": "Leave large tool definitions out of the request and let the model look them up on demand. Reduces prompt size when many tools are enabled.", + "settings.admin.interface.toolSearch.label": "Tool Search", + "settings.admin.interface.toolSearchAlwaysLoaded.description": "Comma-separated tool names or glob patterns that are never deferred.", + "settings.admin.interface.toolSearchAlwaysLoaded.label": "Always Loaded Tools", + "settings.admin.interface.toolSearchAlwaysLoaded.placeholder": "e.g. jira_create_issue, github_*", + "settings.admin.interface.toolSearchDeferBuiltinTools.description": "Defer built-in tools like other tools. When off, built-in tools are always sent in full.", + "settings.admin.interface.toolSearchDeferBuiltinTools.label": "Defer Built-in Tools", + "settings.admin.interface.toolSearchThreshold.description": "Tools whose definition is longer than this many characters are deferred.", + "settings.admin.interface.toolSearchThreshold.label": "Deferral Threshold (characters)", "settings.admin.interface.toolsFunctionCallingPrompt.description": "Guides how the assistant formats tool and function calls.", "settings.admin.interface.toolsFunctionCallingPrompt.label": "Tools Function Calling Prompt", "settings.admin.interface.voiceModePrompt.description": "Apply voice-specific instructions while voice mode is active.",