From e6eb4c86bbcdba76ffca77d86cf13c48b6d7dea9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 27 Aug 2026 19:14:29 +0000 Subject: [PATCH] fix: prevent terminal server spec cache poisoning after restart A spec fetch that failed while the terminals orchestrator was still cold (e.g. right after a stack restart, while it removes conflicting stopped containers and re-provisions) used to overwrite the cached terminal server list in app state and Redis with an empty list. Since nothing on the message path ever refreshed that cache, every chat request then failed with "Terminal server '' is unavailable" until an admin re-saved the connection settings. - set_terminal_servers: keep the last known good cached entry (same id and URL) for any enabled connection whose spec fetch failed, instead of caching its absence; wrap the Redis write in try/except so a Redis blip doesn't fail the request. - get_terminal_tools: rebuild the cache once when the requested terminal id is missing before raising "unavailable", so a stale or cold cache self-heals on the next message. - main.py: warm terminal server specs at startup whenever terminal connections exist; this was previously gated on classic tool server connections being configured. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01N972k9EbZ4vR3JZfHSNqmK --- backend/open_webui/main.py | 30 ++++++++++++--------- backend/open_webui/utils/tools.py | 45 ++++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 17 deletions(-) diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index c84a394b87..7d47d12eda 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -417,8 +417,10 @@ async def lifespan(app: FastAPI): except Exception as e: log.warning(f'Failed to pre-fetch models at startup: {e}') - # Pre-fetch tool server specs so the first request doesn't pay the latency cost - if len(await Config.get('tool_server.connections', []) or []) > 0: + # Pre-fetch tool/terminal server specs so the first request doesn't pay the latency cost + has_tool_servers = len(await Config.get('tool_server.connections', []) or []) > 0 + has_terminal_servers = len(await Config.get('terminal_server.connections', []) or []) > 0 + if has_tool_servers or has_terminal_servers: mock_request = Request( { 'type': 'http', @@ -435,18 +437,20 @@ async def lifespan(app: FastAPI): } ) - log.info('Initializing tool servers...') - try: - await set_tool_servers(mock_request) - log.info('Initialized %s tool server(s)', len(app.state.TOOL_SERVERS)) - except Exception as e: - log.warning(f'Failed to initialize tool servers at startup: {e}') + if has_tool_servers: + log.info('Initializing tool servers...') + try: + await set_tool_servers(mock_request) + log.info('Initialized %s tool server(s)', len(app.state.TOOL_SERVERS)) + except Exception as e: + log.warning(f'Failed to initialize tool servers at startup: {e}') - try: - await set_terminal_servers(mock_request) - log.info('Initialized %s terminal server(s)', len(app.state.TERMINAL_SERVERS)) - except Exception as e: - log.warning(f'Failed to initialize terminal servers at startup: {e}') + if has_terminal_servers: + try: + await set_terminal_servers(mock_request) + log.info('Initialized %s terminal server(s)', len(app.state.TERMINAL_SERVERS)) + except Exception as e: + log.warning(f'Failed to initialize terminal servers at startup: {e}') # Mark application as ready to accept traffic from a startup perspective. if license_task: diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index 4ab5c96504..8ed6c4ae67 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -1299,7 +1299,36 @@ async def set_terminal_servers(request: Request): } ) - request.app.state.TERMINAL_SERVERS = await get_tool_servers_data(server_configs) + terminal_servers = await get_tool_servers_data(server_configs) + + # A failed spec fetch (e.g. cold orchestrator right after a restart) must + # not evict a previously cached spec: keep the last known good entry for + # any enabled connection that dropped out instead of caching its absence. + expected_urls = { + server_config['info']['id'] or str(idx): server_config['url'] + for idx, server_config in enumerate(server_configs) + if server_config['config']['enable'] + } + missing_ids = set(expected_urls) - {server.get('id') for server in terminal_servers} + if missing_ids: + previous_servers = getattr(request.app.state, 'TERMINAL_SERVERS', None) or [] + if request.app.state.redis is not None: + try: + data = await request.app.state.redis.get(f'{REDIS_KEY_PREFIX}:terminal_servers') + if data is not None: + previous_servers = JSONCodec.loads(data) + except Exception as e: + log.error(f'Error fetching terminal_servers from Redis: {e}') + for server in previous_servers: + server_id = server.get('id') + if server_id in missing_ids and server.get('url') == expected_urls[server_id]: + log.warning( + "Spec fetch for terminal server '%s' failed; keeping previously cached specs", + server_id, + ) + terminal_servers.append(server) + + request.app.state.TERMINAL_SERVERS = terminal_servers # Fetch system prompts concurrently (runs at cache time, not per-request) connections_by_id = {c.get('id'): c for c in connections if c.get('id')} @@ -1323,9 +1352,12 @@ async def set_terminal_servers(request: Request): ) if request.app.state.redis is not None: - await request.app.state.redis.set( - f'{REDIS_KEY_PREFIX}:terminal_servers', JSONCodec.dumps(request.app.state.TERMINAL_SERVERS) - ) + try: + await request.app.state.redis.set( + f'{REDIS_KEY_PREFIX}:terminal_servers', JSONCodec.dumps(request.app.state.TERMINAL_SERVERS) + ) + except Exception as e: + log.error(f'Error caching terminal_servers to Redis: {e}') return request.app.state.TERMINAL_SERVERS @@ -1375,6 +1407,11 @@ async def get_terminal_tools( # Find the cached spec data for this terminal terminal_servers = await get_terminal_servers(request) server_data = next((s for s in terminal_servers if s.get('id') == terminal_id), None) + if server_data is None: + # The cache can lack this server when its spec fetch failed earlier + # (e.g. it was still starting up); rebuild once before giving up. + terminal_servers = await set_terminal_servers(request) + server_data = next((s for s in terminal_servers if s.get('id') == terminal_id), None) if server_data is None: raise RuntimeError(f"Terminal server '{terminal_id}' is unavailable")