feat: Dynamic Tool Loading / Lazy Loading

Co-Authored-By: Taylor Wilsdon <6508528+taylorwilsdon@users.noreply.github.com>

backend/open_webui/config.py

@@ -2224,6 +2224,16 @@ CONTEXT_COMPACTION_RETENTION_PERCENTAGE = min(
 
 CONTEXT_COMPACTION_PROMPT_TEMPLATE = os.getenv('CONTEXT_COMPACTION_PROMPT_TEMPLATE', '')
 
+ENABLE_TOOL_SEARCH = os.getenv('ENABLE_TOOL_SEARCH', 'False').lower() == 'true'
+
+TOOL_SEARCH_DEFER_THRESHOLD = int(os.getenv('TOOL_SEARCH_DEFER_THRESHOLD', '400'))
+
+TOOL_SEARCH_ALWAYS_LOADED = [
+    item.strip() for item in os.getenv('TOOL_SEARCH_ALWAYS_LOADED', '').split(',') if item.strip()
+]
+
+TOOL_SEARCH_DEFER_BUILTIN_TOOLS = os.getenv('TOOL_SEARCH_DEFER_BUILTIN_TOOLS', 'True').lower() == 'true'
+
 TITLE_GENERATION_PROMPT_TEMPLATE = os.getenv('TITLE_GENERATION_PROMPT_TEMPLATE', '')
 
 DEFAULT_TITLE_GENERATION_PROMPT_TEMPLATE = """### Task:
@@ -3185,6 +3195,10 @@ DEFAULT_CONFIG = {
     'chat.context_compaction.retention_percentage': CONTEXT_COMPACTION_RETENTION_PERCENTAGE,
     'chat.context_compaction.prompt_template': CONTEXT_COMPACTION_PROMPT_TEMPLATE,
     'chat.tool_permissions.enable': ENABLE_TOOL_PERMISSIONS,
+    'chat.tool_search.enable': ENABLE_TOOL_SEARCH,
+    'chat.tool_search.defer_threshold': TOOL_SEARCH_DEFER_THRESHOLD,
+    'chat.tool_search.always_loaded': TOOL_SEARCH_ALWAYS_LOADED,
+    'chat.tool_search.defer_builtin_tools': TOOL_SEARCH_DEFER_BUILTIN_TOOLS,
     'task.title.prompt_template': TITLE_GENERATION_PROMPT_TEMPLATE,
     'task.tags.prompt_template': TAGS_GENERATION_PROMPT_TEMPLATE,
     'task.image.prompt_template': IMAGE_PROMPT_GENERATION_PROMPT_TEMPLATE,

backend/open_webui/routers/chats.py

@@ -62,6 +62,10 @@ CHAT_CONFIG_KEYS = {
     'CONTEXT_COMPACTION_RETENTION_PERCENTAGE': 'chat.context_compaction.retention_percentage',
     'CONTEXT_COMPACTION_PROMPT_TEMPLATE': 'chat.context_compaction.prompt_template',
     'ENABLE_TOOL_PERMISSIONS': 'chat.tool_permissions.enable',
+    'ENABLE_TOOL_SEARCH': 'chat.tool_search.enable',
+    'TOOL_SEARCH_DEFER_THRESHOLD': 'chat.tool_search.defer_threshold',
+    'TOOL_SEARCH_ALWAYS_LOADED': 'chat.tool_search.always_loaded',
+    'TOOL_SEARCH_DEFER_BUILTIN_TOOLS': 'chat.tool_search.defer_builtin_tools',
 }
 
 
@@ -202,6 +206,10 @@ class ChatConfigForm(BaseModel):
     CONTEXT_COMPACTION_RETENTION_PERCENTAGE: int | None = None
     CONTEXT_COMPACTION_PROMPT_TEMPLATE: str
     ENABLE_TOOL_PERMISSIONS: bool = False
+    ENABLE_TOOL_SEARCH: bool = False
+    TOOL_SEARCH_DEFER_THRESHOLD: int = 400
+    TOOL_SEARCH_ALWAYS_LOADED: list[str] = []
+    TOOL_SEARCH_DEFER_BUILTIN_TOOLS: bool = True
 
 
 class CompactChatForm(BaseModel):
@@ -892,6 +900,8 @@ async def set_chat_config(form_data: ChatConfigForm, user=Depends(get_admin_user
     if retention_percentage is None:
         retention_percentage = CONTEXT_COMPACTION_RETENTION_PERCENTAGE
     retention_percentage = min(50, max(10, int(retention_percentage)))
+    tool_search_defer_threshold = max(0, int(form_data.TOOL_SEARCH_DEFER_THRESHOLD))
+    tool_search_always_loaded = [item.strip() for item in form_data.TOOL_SEARCH_ALWAYS_LOADED if item.strip()]
     await Config.upsert(
         chat_config_updates(
             {
@@ -900,6 +910,8 @@ async def set_chat_config(form_data: ChatConfigForm, user=Depends(get_admin_user
                 'CONTEXT_COMPACTION_TOKEN_THRESHOLD': threshold,
                 'CONTEXT_COMPACTION_TOKEN_CAP': token_cap,
                 'CONTEXT_COMPACTION_RETENTION_PERCENTAGE': retention_percentage,
+                'TOOL_SEARCH_DEFER_THRESHOLD': tool_search_defer_threshold,
+                'TOOL_SEARCH_ALWAYS_LOADED': tool_search_always_loaded,
             }
         )
     )

backend/open_webui/tools/builtin.py

@@ -3734,6 +3734,36 @@ async def update_skill_files(
         return JSONCodec.dumps({'error': getattr(error, 'detail', str(error))})
 
 
+# =============================================================================
+# TOOL SEARCH
+# =============================================================================
+
+
+async def search_tools(
+    query: str,
+    count: int = 5,
+    __metadata__: dict = None,
+) -> str:
+    """
+    Search the tools listed in <available_tools> and return their full definitions.
+    Pass the exact tool name when you already know it.
+
+    :param query: Keywords describing the capability you need (e.g. "jira create issue"), or an exact tool name
+    :param count: Maximum number of results to return (default: 5, max: 20)
+    :return: JSON with the definitions of the matching tools, which can then be called by name
+    """
+    from open_webui.utils.tool_search import search_deferred_tools
+
+    tools = __metadata__['tools']
+    candidates = {name: tools[name]['spec'] for name in __metadata__['deferred_tools']}
+    matches = search_deferred_tools(query, candidates, count)
+    if not matches:
+        return JSONCodec.dumps(
+            {'tools': [], 'message': 'No matching tools found. Try different keywords or the exact tool name.'}
+        )
+    return JSONCodec.dumps({'tools': [candidates[name] for name in matches]})
+
+
 # =============================================================================
 # TASK MANAGEMENT TOOLS
 # =============================================================================

backend/open_webui/utils/middleware.py

@@ -144,6 +144,7 @@ from open_webui.utils.task import (
     tools_function_calling_generation_template,
 )
 from open_webui.utils.tool_approval import complete_tool_call, pending_tool_calls
+from open_webui.utils.tool_search import apply_tool_search
 from open_webui.utils.tools import (
     connect_mcp_server,
     get_attached_knowledge,
@@ -3281,9 +3282,13 @@ async def process_chat_payload(request, form_data, user, metadata, model):
             metadata['tools'] = tools_dict
 
             if metadata.get('params', {}).get('function_calling') != 'legacy':
+                deferred = await apply_tool_search(form_data, metadata, tools_dict)
+
                 # If the function calling is native, then call the tools function calling handler
                 form_data['tools'] = [
-                    {'type': 'function', 'function': tool.get('spec', {})} for tool in tools_dict.values()
+                    {'type': 'function', 'function': tool.get('spec', {})}
+                    for name, tool in tools_dict.items()
+                    if name not in deferred
                 ]
                 if inlet_filter_tools:
                     form_data['tools'].extend(inlet_filter_tools)

backend/open_webui/utils/subagents.py

@@ -19,6 +19,7 @@ from open_webui.tasks import create_task, has_active_tasks
 from open_webui.utils.auth import VERIFIED_USER_ROLES, create_token
 from open_webui.utils.json_codec import JSONCodec
 from open_webui.utils.misc import add_or_update_system_message, get_message_list
+from open_webui.utils.tool_search import strip_deferred_tools_manifest
 from sqlalchemy import select
 from starlette.datastructures import Headers
 
@@ -249,7 +250,9 @@ async def process_pending_internal_messages(
         chat_context = copy.deepcopy(run.get('chat_context') or {})
         # Runs saved before chat_context was introduced only have the assembled prompt.
         if run.get('chat_context') is None and run.get('system_prompt'):
-            chat_context['messages'] = [{'role': 'system', 'content': run['system_prompt']}]
+            chat_context['messages'] = [
+                {'role': 'system', 'content': strip_deferred_tools_manifest(run['system_prompt'])}
+            ]
         form_data = {
             'model': model_id,
             'params': chat_context.get('params') or {},

backend/open_webui/utils/timers.py

@@ -20,6 +20,7 @@ from open_webui.tasks import has_active_tasks
 from open_webui.utils.auth import VERIFIED_USER_ROLES
 from open_webui.utils.json_codec import JSONCodec
 from open_webui.utils.misc import get_message_list
+from open_webui.utils.tool_search import strip_deferred_tools_manifest
 from sqlalchemy import select
 from starlette.datastructures import Headers
 
@@ -377,7 +378,9 @@ async def execute_due_timer(app, timer_id: str, claim_id: str | None = None) ->
         chat_context = copy.deepcopy(run.get('chat_context') or {})
         # Timers saved before chat_context was introduced only have the assembled prompt.
         if run.get('chat_context') is None and run.get('system_prompt'):
-            chat_context['messages'] = [{'role': 'system', 'content': run['system_prompt']}]
+            chat_context['messages'] = [
+                {'role': 'system', 'content': strip_deferred_tools_manifest(run['system_prompt'])}
+            ]
         form_data = {
             'model': model_id,
             'params': chat_context.get('params') or {},

backend/open_webui/utils/tool_search.py

@@ -0,0 +1,159 @@
+"""
+Tool search: large tool schemas are left out of the provider `tools` array and listed by name in an
+`<available_tools>` system prompt block. `search_tools` returns the full definitions of matching tools,
+which the model then calls by name. The `tools` array never changes during a chat, so prompt caches hold.
+"""
+
+import fnmatch
+import re
+import unicodedata
+
+import regex
+from open_webui.models.config import Config
+from open_webui.utils.json_codec import JSONCodec
+from open_webui.utils.misc import add_or_update_system_message
+
+SEARCH_TOOL_NAME = 'search_tools'
+MANIFEST_DESCRIPTION_MAX_CHARS = 100
+MAX_SEARCH_COUNT = 20
+DEFAULT_SEARCH_COUNT = 5
+
+_MANIFEST_RE = re.compile(r'\n?<available_tools>\n.*?</available_tools>', re.DOTALL)
+
+
+async def get_tool_search_config() -> dict | None:
+    """Tool search settings, or None when the feature is off."""
+    if not await Config.get('chat.tool_search.enable'):
+        return None
+    values = await Config.get_many(
+        'chat.tool_search.defer_threshold',
+        'chat.tool_search.always_loaded',
+        'chat.tool_search.defer_builtin_tools',
+    )
+    return {
+        'defer_threshold': values['chat.tool_search.defer_threshold'],
+        'always_loaded': values['chat.tool_search.always_loaded'],
+        'defer_builtin_tools': values['chat.tool_search.defer_builtin_tools'],
+    }
+
+
+def select_deferred_tools(tools_dict: dict[str, dict], config: dict) -> list[str]:
+    """Sorted names of tools whose schema exceeds the threshold and that are not always loaded."""
+    patterns = config['always_loaded']
+    return sorted(
+        name
+        for name, tool in tools_dict.items()
+        if (config['defer_builtin_tools'] or tool.get('type') != 'builtin')
+        and not any(fnmatch.fnmatchcase(name, pattern) for pattern in patterns)
+        and len(JSONCodec.dumps(tool.get('spec') or {})) > config['defer_threshold']
+    )
+
+
+def _truncate_description(description: str | None) -> str:
+    text = ' '.join(str(description or '').split())
+    if len(text) > MANIFEST_DESCRIPTION_MAX_CHARS:
+        text = text[: MANIFEST_DESCRIPTION_MAX_CHARS - 1].rstrip() + '…'
+    return text
+
+
+def build_deferred_tools_manifest(tools_dict: dict[str, dict], deferred: list[str]) -> str:
+    entries = ''
+    for name in deferred:
+        description = _truncate_description(tools_dict[name].get('spec', {}).get('description'))
+        entries += f'<tool>\n<name>{name}</name>\n<description>{description}</description>\n</tool>\n'
+
+    return (
+        '<available_tools>\n'
+        'The following tools are available but their definitions are not loaded. '
+        f'To use one, call `{SEARCH_TOOL_NAME}` with a short keyword query or the exact tool name to get its '
+        'definition, then call the tool by name with the parameters it defines. '
+        'Never tell the user a capability is unavailable without searching first.\n'
+        f'{entries}</available_tools>'
+    )
+
+
+async def apply_tool_search(form_data: dict, metadata: dict, tools_dict: dict[str, dict]) -> set[str]:
+    """Add search_tools and the manifest when tools can be deferred; returns the names to leave out of `tools`."""
+    config = SEARCH_TOOL_NAME not in tools_dict and await get_tool_search_config()
+    deferred = select_deferred_tools(tools_dict, config) if config else []
+    if not deferred:
+        return set()
+
+    from open_webui.tools.builtin import search_tools
+    from open_webui.utils.tools import (
+        get_async_tool_function_and_apply_extra_params,
+        get_builtin_function_introspection,
+        get_builtin_tool_spec,
+    )
+
+    metadata['deferred_tools'] = deferred
+    form_data['messages'] = add_or_update_system_message(
+        build_deferred_tools_manifest(tools_dict, deferred),
+        form_data['messages'],
+        append=True,
+    )
+    tools_dict[SEARCH_TOOL_NAME] = {
+        'tool_id': f'builtin:{SEARCH_TOOL_NAME}',
+        'callable': await get_async_tool_function_and_apply_extra_params(
+            search_tools, {'__metadata__': metadata}, get_builtin_function_introspection(search_tools)
+        ),
+        'spec': get_builtin_tool_spec(search_tools),
+        'type': 'builtin',
+    }
+    return set(deferred)
+
+
+def strip_deferred_tools_manifest(system_prompt: str | None) -> str | None:
+    """Remove the manifest from a system prompt that is reused by a request which builds its own."""
+    return _MANIFEST_RE.sub('', system_prompt) if system_prompt else system_prompt
+
+
+# Letters, digits and combining marks, so vowel signs in scripts like Devanagari stay attached to their word.
+_WORD_RE = regex.compile(r'[\p{L}\p{N}\p{M}]+')
+_CAMEL_RE = re.compile(r'(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])')
+# Scripts written without spaces between words: Han, Hiragana, Katakana and Hangul.
+_CJK_RE = re.compile(r'([\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af]+)')
+
+
+def tokenize(text: str | None) -> list[str]:
+    """Lowercase word tokens split on camelCase boundaries, with CJK runs split into character bigrams."""
+    tokens = []
+    for word in _WORD_RE.findall(unicodedata.normalize('NFKC', text or '')):
+        for part in _CAMEL_RE.split(word):
+            for chunk in _CJK_RE.split(part.casefold()):
+                if _CJK_RE.fullmatch(chunk):
+                    tokens.extend([chunk[i : i + 2] for i in range(len(chunk) - 1)] or [chunk])
+                elif chunk:
+                    tokens.append(chunk)
+    return tokens
+
+
+def _document_text(name: str, spec: dict) -> str:
+    properties = (spec.get('parameters') or {}).get('properties') or {}
+    return ' '.join([name, str(spec.get('description') or ''), *properties.keys()])
+
+
+def search_deferred_tools(query: str, candidates: dict[str, dict], count: int = DEFAULT_SEARCH_COUNT) -> list[str]:
+    """Rank candidate tools (name -> spec) against the query; an exact tool name wins outright."""
+    from rank_bm25 import BM25Okapi
+
+    query = (query or '').strip()
+    if query in candidates:
+        return [query]
+
+    query_tokens = tokenize(query)
+    if not candidates or not query_tokens:
+        return []
+
+    names = list(candidates)
+    documents = [tokenize(_document_text(name, candidates[name])) or ['_'] for name in names]
+    # BM25 IDF is zero for a term found in half the corpus, common with small tool sets,
+    # so raw token overlap keeps every matching term rankable.
+    bm25 = BM25Okapi(documents).get_scores(query_tokens)
+    scores = {
+        name: max(0.0, float(bm25[index])) + len(set(query_tokens) & set(documents[index]))
+        for index, name in enumerate(names)
+    }
+
+    ranked = sorted((name for name in names if scores[name] > 0), key=lambda name: (-scores[name], name))
+    return ranked[: max(1, min(count, MAX_SEARCH_COUNT))]

src/lib/components/admin/Settings/Interface.svelte

@@ -50,8 +50,13 @@
 		CONTEXT_COMPACTION_TOKEN_CAP: 80000,
 		CONTEXT_COMPACTION_RETENTION_PERCENTAGE: 40,
 		CONTEXT_COMPACTION_PROMPT_TEMPLATE: '',
-		ENABLE_TOOL_PERMISSIONS: false
+		ENABLE_TOOL_PERMISSIONS: false,
+		ENABLE_TOOL_SEARCH: false,
+		TOOL_SEARCH_DEFER_THRESHOLD: 400,
+		TOOL_SEARCH_ALWAYS_LOADED: [] as string[],
+		TOOL_SEARCH_DEFER_BUILTIN_TOOLS: true
 	};
+	let toolSearchAlwaysLoaded = '';
 	let showTaskParameters = false;
 
 	const configuredParams = (params: Record<string, any> = {}) =>
@@ -70,7 +75,13 @@
 		try {
 			[taskConfig, chatConfig] = await Promise.all([
 				updateTaskConfig(localStorage.token, taskConfigPayload),
-				updateChatConfig(localStorage.token, chatConfig)
+				updateChatConfig(localStorage.token, {
+					...chatConfig,
+					TOOL_SEARCH_ALWAYS_LOADED: toolSearchAlwaysLoaded
+						.split(',')
+						.map((item) => item.trim())
+						.filter((item) => item !== '')
+				})
 			]);
 		} catch (error) {
 			toast.error(
@@ -78,6 +89,7 @@
 			);
 			return;
 		}
+		toolSearchAlwaysLoaded = (chatConfig.TOOL_SEARCH_ALWAYS_LOADED ?? []).join(', ');
 		appConfig.update((current) =>
 			current
 				? {
@@ -131,6 +143,7 @@
 				getChatConfig(localStorage.token)
 			]);
 			taskConfig.TASK_MODEL_PARAMS = taskConfig.TASK_MODEL_PARAMS ?? {};
+			toolSearchAlwaysLoaded = (chatConfig.TOOL_SEARCH_ALWAYS_LOADED ?? []).join(', ');
 
 			workspaceModels = await getBaseModels(localStorage.token);
 			baseModels = await getModels(localStorage.token, null, false);
@@ -369,6 +382,59 @@
 						</div>
 					</AdminSettingField>
 				{/if}
+
+				<AdminSettingRow
+					label={$i18n.t('settings.admin.interface.toolSearch.label')}
+					description={$i18n.t('settings.admin.interface.toolSearch.description')}
+					let:labelId
+				>
+					<div slot="label" class="flex items-center gap-2">
+						<span>{$i18n.t('settings.admin.interface.toolSearch.label')}</span>
+						<ExperimentalBadge />
+					</div>
+					<Switch bind:state={chatConfig.ENABLE_TOOL_SEARCH} ariaLabelledbyId={labelId} />
+				</AdminSettingRow>
+
+				{#if chatConfig.ENABLE_TOOL_SEARCH}
+					<AdminSettingRow
+						label={$i18n.t('settings.admin.interface.toolSearchDeferBuiltinTools.label')}
+						description={$i18n.t(
+							'settings.admin.interface.toolSearchDeferBuiltinTools.description'
+						)}
+						let:labelId
+					>
+						<Switch
+							bind:state={chatConfig.TOOL_SEARCH_DEFER_BUILTIN_TOOLS}
+							ariaLabelledbyId={labelId}
+						/>
+					</AdminSettingRow>
+
+					<AdminSettingField
+						label={$i18n.t('settings.admin.interface.toolSearchThreshold.label')}
+						description={$i18n.t('settings.admin.interface.toolSearchThreshold.description')}
+					>
+						<input
+							type="number"
+							min="0"
+							step="1"
+							class={inputClass}
+							bind:value={chatConfig.TOOL_SEARCH_DEFER_THRESHOLD}
+						/>
+					</AdminSettingField>
+
+					<AdminSettingField
+						label={$i18n.t('settings.admin.interface.toolSearchAlwaysLoaded.label')}
+						description={$i18n.t('settings.admin.interface.toolSearchAlwaysLoaded.description')}
+					>
+						<input
+							class={inputClass}
+							type="text"
+							placeholder={$i18n.t('settings.admin.interface.toolSearchAlwaysLoaded.placeholder')}
+							bind:value={toolSearchAlwaysLoaded}
+							autocomplete="off"
+						/>
+					</AdminSettingField>
+				{/if}
 			</AdminSettingSection>
 
 			<AdminSettingSection title={$i18n.t('settings.admin.interface.sections.generation.title')}>

src/lib/i18n/locales/en-US/translation.json

@@ -2793,6 +2793,15 @@
 	"settings.admin.interface.tokenThreshold.label": "Token Threshold",
 	"settings.admin.interface.toolPermissions.description": "Show Full access and Ask for approval in the chat input menu.",
 	"settings.admin.interface.toolPermissions.label": "Tool Permissions",
+	"settings.admin.interface.toolSearch.description": "Leave large tool definitions out of the request and let the model look them up on demand. Reduces prompt size when many tools are enabled.",
+	"settings.admin.interface.toolSearch.label": "Tool Search",
+	"settings.admin.interface.toolSearchAlwaysLoaded.description": "Comma-separated tool names or glob patterns that are never deferred.",
+	"settings.admin.interface.toolSearchAlwaysLoaded.label": "Always Loaded Tools",
+	"settings.admin.interface.toolSearchAlwaysLoaded.placeholder": "e.g. jira_create_issue, github_*",
+	"settings.admin.interface.toolSearchDeferBuiltinTools.description": "Defer built-in tools like other tools. When off, built-in tools are always sent in full.",
+	"settings.admin.interface.toolSearchDeferBuiltinTools.label": "Defer Built-in Tools",
+	"settings.admin.interface.toolSearchThreshold.description": "Tools whose definition is longer than this many characters are deferred.",
+	"settings.admin.interface.toolSearchThreshold.label": "Deferral Threshold (characters)",
 	"settings.admin.interface.toolsFunctionCallingPrompt.description": "Guides how the assistant formats tool and function calls.",
 	"settings.admin.interface.toolsFunctionCallingPrompt.label": "Tools Function Calling Prompt",
 	"settings.admin.interface.voiceModePrompt.description": "Apply voice-specific instructions while voice mode is active.",