diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index c84a394b87..7d47d12eda 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -417,8 +417,10 @@ async def lifespan(app: FastAPI): except Exception as e: log.warning(f'Failed to pre-fetch models at startup: {e}') - # Pre-fetch tool server specs so the first request doesn't pay the latency cost - if len(await Config.get('tool_server.connections', []) or []) > 0: + # Pre-fetch tool/terminal server specs so the first request doesn't pay the latency cost + has_tool_servers = len(await Config.get('tool_server.connections', []) or []) > 0 + has_terminal_servers = len(await Config.get('terminal_server.connections', []) or []) > 0 + if has_tool_servers or has_terminal_servers: mock_request = Request( { 'type': 'http', @@ -435,18 +437,20 @@ async def lifespan(app: FastAPI): } ) - log.info('Initializing tool servers...') - try: - await set_tool_servers(mock_request) - log.info('Initialized %s tool server(s)', len(app.state.TOOL_SERVERS)) - except Exception as e: - log.warning(f'Failed to initialize tool servers at startup: {e}') + if has_tool_servers: + log.info('Initializing tool servers...') + try: + await set_tool_servers(mock_request) + log.info('Initialized %s tool server(s)', len(app.state.TOOL_SERVERS)) + except Exception as e: + log.warning(f'Failed to initialize tool servers at startup: {e}') - try: - await set_terminal_servers(mock_request) - log.info('Initialized %s terminal server(s)', len(app.state.TERMINAL_SERVERS)) - except Exception as e: - log.warning(f'Failed to initialize terminal servers at startup: {e}') + if has_terminal_servers: + try: + await set_terminal_servers(mock_request) + log.info('Initialized %s terminal server(s)', len(app.state.TERMINAL_SERVERS)) + except Exception as e: + log.warning(f'Failed to initialize terminal servers at startup: {e}') # Mark application as ready to accept traffic from a startup perspective. if license_task: diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index 4ab5c96504..8ed6c4ae67 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -1299,7 +1299,36 @@ async def set_terminal_servers(request: Request): } ) - request.app.state.TERMINAL_SERVERS = await get_tool_servers_data(server_configs) + terminal_servers = await get_tool_servers_data(server_configs) + + # A failed spec fetch (e.g. cold orchestrator right after a restart) must + # not evict a previously cached spec: keep the last known good entry for + # any enabled connection that dropped out instead of caching its absence. + expected_urls = { + server_config['info']['id'] or str(idx): server_config['url'] + for idx, server_config in enumerate(server_configs) + if server_config['config']['enable'] + } + missing_ids = set(expected_urls) - {server.get('id') for server in terminal_servers} + if missing_ids: + previous_servers = getattr(request.app.state, 'TERMINAL_SERVERS', None) or [] + if request.app.state.redis is not None: + try: + data = await request.app.state.redis.get(f'{REDIS_KEY_PREFIX}:terminal_servers') + if data is not None: + previous_servers = JSONCodec.loads(data) + except Exception as e: + log.error(f'Error fetching terminal_servers from Redis: {e}') + for server in previous_servers: + server_id = server.get('id') + if server_id in missing_ids and server.get('url') == expected_urls[server_id]: + log.warning( + "Spec fetch for terminal server '%s' failed; keeping previously cached specs", + server_id, + ) + terminal_servers.append(server) + + request.app.state.TERMINAL_SERVERS = terminal_servers # Fetch system prompts concurrently (runs at cache time, not per-request) connections_by_id = {c.get('id'): c for c in connections if c.get('id')} @@ -1323,9 +1352,12 @@ async def set_terminal_servers(request: Request): ) if request.app.state.redis is not None: - await request.app.state.redis.set( - f'{REDIS_KEY_PREFIX}:terminal_servers', JSONCodec.dumps(request.app.state.TERMINAL_SERVERS) - ) + try: + await request.app.state.redis.set( + f'{REDIS_KEY_PREFIX}:terminal_servers', JSONCodec.dumps(request.app.state.TERMINAL_SERVERS) + ) + except Exception as e: + log.error(f'Error caching terminal_servers to Redis: {e}') return request.app.state.TERMINAL_SERVERS @@ -1375,6 +1407,11 @@ async def get_terminal_tools( # Find the cached spec data for this terminal terminal_servers = await get_terminal_servers(request) server_data = next((s for s in terminal_servers if s.get('id') == terminal_id), None) + if server_data is None: + # The cache can lack this server when its spec fetch failed earlier + # (e.g. it was still starting up); rebuild once before giving up. + terminal_servers = await set_terminal_servers(request) + server_data = next((s for s in terminal_servers if s.get('id') == terminal_id), None) if server_data is None: raise RuntimeError(f"Terminal server '{terminal_id}' is unavailable")