diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index b17df7340a..f603c6c1ab 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -1018,6 +1018,13 @@ DEFAULT_GROUP_SHARE_PERMISSION = 'members' if _default_group_share == 'members' ENABLE_CUSTOM_MODEL_FALLBACK = os.getenv('ENABLE_CUSTOM_MODEL_FALLBACK', 'False').lower() == 'true' +# Seconds a connection's last successful model list stays usable when a later +# refresh of that connection fails. 0 serves no stale model list. +try: + MODEL_LIST_FALLBACK_TTL = int(os.getenv('MODEL_LIST_FALLBACK_TTL', '300')) +except ValueError: + MODEL_LIST_FALLBACK_TTL = 300 + MODELS_CACHE_TTL = os.getenv('MODELS_CACHE_TTL', '1') if MODELS_CACHE_TTL == '': MODELS_CACHE_TTL = None diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index c84a394b87..6729c8a27f 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -538,6 +538,7 @@ if ENABLE_OTEL: app.state.OLLAMA_MODELS = {} +app.state.OLLAMA_MODEL_LIST_FALLBACK = {} ######################################## # @@ -547,6 +548,7 @@ app.state.OLLAMA_MODELS = {} app.state.OPENAI_MODELS = {} +app.state.OPENAI_MODEL_LIST_FALLBACK = {} ######################################## # diff --git a/backend/open_webui/routers/ollama.py b/backend/open_webui/routers/ollama.py index 00f41c01b3..2fd4a566c8 100644 --- a/backend/open_webui/routers/ollama.py +++ b/backend/open_webui/routers/ollama.py @@ -37,7 +37,7 @@ from open_webui.utils.access_control import check_model_access from open_webui.utils.auth import get_admin_user, get_verified_user from open_webui.utils.headers import get_custom_headers, include_user_info_headers from open_webui.utils.json_codec import JSONCodec -from open_webui.utils.misc import calculate_sha256 +from open_webui.utils.misc import apply_model_list_fallback, calculate_sha256 from open_webui.utils.model_ids import strip_provider_model_prefix from open_webui.utils.payload import ( apply_model_params_to_body_ollama, @@ -320,6 +320,7 @@ async def update_config( await get_all_models.cache.clear() request.app.state.BASE_MODELS = [] + request.app.state.OLLAMA_MODEL_LIST_FALLBACK.clear() request.app.state.OLLAMA_MODELS = {} models = getattr(request.app.state, 'MODELS', None) if hasattr(models, 'clear'): @@ -394,11 +395,15 @@ async def get_all_models(request: Request, user: UserModel | None = None): tasks = [] base_urls = await Config.get('ollama.base_urls', []) api_configs = await Config.get('ollama.api_configs', {}) + # Backends we query, so a failed request can reuse the models we last saw there. + reusable_idxs = set() for idx, url in enumerate(base_urls): api_config = resolve_api_config(api_configs, idx, url) if not api_config: + reusable_idxs.add(idx) tasks.append(send_get_request(f'{url}/api/tags', user=user)) elif api_config.get('enable', True): + reusable_idxs.add(idx) tasks.append(send_get_request(f'{url}/api/tags', api_config.get('key'), user=user)) else: tasks.append(asyncio.ensure_future(asyncio.sleep(0, None))) @@ -434,6 +439,13 @@ async def get_all_models(request: Request, user: UserModel | None = None): if connection_type: m['connection_type'] = connection_type + # After the loop above, so a reused response is not prefixed and tagged twice. Forwarded user + # info can make a backend answer per user, and one user's list must not be reused for another. + if not ENABLE_FORWARD_USER_INFO_HEADERS: + apply_model_list_fallback( + request.app.state.OLLAMA_MODEL_LIST_FALLBACK, responses, reusable_idxs, base_urls, 'models' + ) + models_dict = {'models': merge_models_lists(r.get('models', []) if r else None for r in responses)} # Annotate with expiry info from loaded-model state diff --git a/backend/open_webui/routers/openai.py b/backend/open_webui/routers/openai.py index 0f21defdd5..6a2042ab4c 100644 --- a/backend/open_webui/routers/openai.py +++ b/backend/open_webui/routers/openai.py @@ -43,7 +43,7 @@ from open_webui.utils.anthropic import ANTHROPIC_VERSION, get_anthropic_models, from open_webui.utils.auth import get_admin_user, get_verified_user from open_webui.utils.headers import get_custom_headers, include_user_info_headers from open_webui.utils.json_codec import JSONCodec -from open_webui.utils.misc import convert_logit_bias_input_to_json +from open_webui.utils.misc import apply_model_list_fallback, convert_logit_bias_input_to_json from open_webui.utils.model_ids import strip_provider_model_prefix from open_webui.utils.payload import ( apply_model_params_to_body_openai, @@ -346,6 +346,7 @@ async def get_openai_connection(idx: int) -> tuple[str, str, dict]: async def clear_openai_model_cache(request: Request): await get_all_models.cache.clear() request.app.state.BASE_MODELS = [] + request.app.state.OPENAI_MODEL_LIST_FALLBACK.clear() request.app.state.OPENAI_MODELS = {} models = getattr(request.app.state, 'MODELS', None) if hasattr(models, 'clear'): @@ -571,14 +572,7 @@ async def update_config(request: Request, form_data: OpenAIConfigForm, user=Depe } ) - await get_all_models.cache.clear() - request.app.state.BASE_MODELS = [] - request.app.state.OPENAI_MODELS = {} - models = getattr(request.app.state, 'MODELS', None) - if hasattr(models, 'clear'): - models.clear() - else: - request.app.state.MODELS = {} + await clear_openai_model_cache(request) await publish_event( request, @@ -677,6 +671,13 @@ async def speech(request: Request, user=Depends(get_verified_user)): raise HTTPException(status_code=401, detail=ERROR_MESSAGES.OPENAI_NOT_FOUND) +def is_user_scoped_connection(api_config: dict) -> bool: + """True when a connection's model list can differ per user, so it must not be reused for another one.""" + if api_config.get('auth_type') in ('session', 'system_oauth'): + return True + return any('{{USER_' in str(value) for value in (api_config.get('headers') or {}).values()) + + async def get_all_models_responses(request: Request, user: UserModel) -> list: enable_openai_api, api_base_urls, api_keys, api_configs = await get_openai_runtime_config() if not enable_openai_api: @@ -689,8 +690,11 @@ async def get_all_models_responses(request: Request, user: UserModel) -> list: api_keys = await normalize_openai_api_keys(api_base_urls, api_keys) request_tasks = [] + # Connections whose model list is the same for every user, so a failed request can reuse it. + reusable_idxs = set() for idx, url in enumerate(api_base_urls): if (str(idx) not in api_configs) and (url not in api_configs): # Legacy support + reusable_idxs.add(idx) request_tasks.append(get_models_request(request, url, api_keys[idx], user=user)) else: api_config = api_configs.get( @@ -703,6 +707,8 @@ async def get_all_models_responses(request: Request, user: UserModel) -> list: if enable: if len(model_ids) == 0: + if not is_user_scoped_connection(api_config): + reusable_idxs.add(idx) request_tasks.append(get_models_request(request, url, api_keys[idx], user=user, config=api_config)) else: model_list = { @@ -762,6 +768,13 @@ async def get_all_models_responses(request: Request, user: UserModel) -> list: if provider: model['provider'] = provider + # After the loop above, so a reused response is not prefixed and tagged twice. Forwarded user + # info can make any upstream answer per user, and one user's list must not be reused for another. + if not ENABLE_FORWARD_USER_INFO_HEADERS: + apply_model_list_fallback( + request.app.state.OPENAI_MODEL_LIST_FALLBACK, responses, reusable_idxs, api_base_urls, 'data' + ) + log.debug('get_all_models:responses() %s', responses) return responses diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index ea15adf75e..327c73616a 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -15,7 +15,7 @@ from typing import Callable, Optional, Sequence, Union import aiohttp import mimeparse -from open_webui.env import CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE +from open_webui.env import CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE, MODEL_LIST_FALLBACK_TTL from open_webui.utils.json_codec import JSONCodec log = logging.getLogger(__name__) @@ -1310,3 +1310,28 @@ def stream_chunks_handler(stream: aiohttp.StreamReader): yield bytes(buffer) return yield_safe_stream_chunks() + + +def apply_model_list_fallback(last_good: dict, responses: list, reusable_idxs: set, urls: list, key: str) -> None: + """Cache each connection's model list, and put the last good one back when its request failed.""" + if MODEL_LIST_FALLBACK_TTL <= 0: + return + + for idx in reusable_idxs: + response = responses[idx] + # Some providers report a failure as an error field next to an empty list. + if isinstance(response, dict) and response.get('error'): + models = None + else: + models = response if isinstance(response, list) else (response or {}).get(key) + + if isinstance(models, list): + last_good[idx] = (time.monotonic(), urls[idx], [{**model} for model in models]) + continue + + # The url has to match too: connections are identified by index, and another + # worker may still hold the models of a connection that has since been removed. + cached_at, cached_url, cached_models = last_good.get(idx, (0, None, None)) + if cached_models and cached_url == urls[idx] and time.monotonic() - cached_at <= MODEL_LIST_FALLBACK_TTL: + log.warning('Model list request to %s failed, reusing the last known good one', urls[idx]) + responses[idx] = {key: [{**model} for model in cached_models]}