fix: prevent terminal server spec cache poisoning after restart

A spec fetch that failed while the terminals orchestrator was still cold
(e.g. right after a stack restart, while it removes conflicting stopped
containers and re-provisions) used to overwrite the cached terminal server
list in app state and Redis with an empty list. Since nothing on the
message path ever refreshed that cache, every chat request then failed
with "Terminal server '<id>' is unavailable" until an admin re-saved the
connection settings.

- set_terminal_servers: keep the last known good cached entry (same id
  and URL) for any enabled connection whose spec fetch failed, instead of
  caching its absence; wrap the Redis write in try/except so a Redis blip
  doesn't fail the request.
- get_terminal_tools: rebuild the cache once when the requested terminal
  id is missing before raising "unavailable", so a stale or cold cache
  self-heals on the next message.
- main.py: warm terminal server specs at startup whenever terminal
  connections exist; this was previously gated on classic tool server
  connections being configured.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N972k9EbZ4vR3JZfHSNqmK

backend/open_webui/main.py

@@ -417,8 +417,10 @@ async def lifespan(app: FastAPI):
         except Exception as e:
             log.warning(f'Failed to pre-fetch models at startup: {e}')
 
-    # Pre-fetch tool server specs so the first request doesn't pay the latency cost
-    if len(await Config.get('tool_server.connections', []) or []) > 0:
+    # Pre-fetch tool/terminal server specs so the first request doesn't pay the latency cost
+    has_tool_servers = len(await Config.get('tool_server.connections', []) or []) > 0
+    has_terminal_servers = len(await Config.get('terminal_server.connections', []) or []) > 0
+    if has_tool_servers or has_terminal_servers:
         mock_request = Request(
             {
                 'type': 'http',
@@ -435,18 +437,20 @@ async def lifespan(app: FastAPI):
             }
         )
 
-        log.info('Initializing tool servers...')
-        try:
-            await set_tool_servers(mock_request)
-            log.info('Initialized %s tool server(s)', len(app.state.TOOL_SERVERS))
-        except Exception as e:
-            log.warning(f'Failed to initialize tool servers at startup: {e}')
+        if has_tool_servers:
+            log.info('Initializing tool servers...')
+            try:
+                await set_tool_servers(mock_request)
+                log.info('Initialized %s tool server(s)', len(app.state.TOOL_SERVERS))
+            except Exception as e:
+                log.warning(f'Failed to initialize tool servers at startup: {e}')
 
-        try:
-            await set_terminal_servers(mock_request)
-            log.info('Initialized %s terminal server(s)', len(app.state.TERMINAL_SERVERS))
-        except Exception as e:
-            log.warning(f'Failed to initialize terminal servers at startup: {e}')
+        if has_terminal_servers:
+            try:
+                await set_terminal_servers(mock_request)
+                log.info('Initialized %s terminal server(s)', len(app.state.TERMINAL_SERVERS))
+            except Exception as e:
+                log.warning(f'Failed to initialize terminal servers at startup: {e}')
 
     # Mark application as ready to accept traffic from a startup perspective.
     if license_task:

backend/open_webui/utils/tools.py

@@ -1299,7 +1299,36 @@ async def set_terminal_servers(request: Request):
             }
         )
 
-    request.app.state.TERMINAL_SERVERS = await get_tool_servers_data(server_configs)
+    terminal_servers = await get_tool_servers_data(server_configs)
+
+    # A failed spec fetch (e.g. cold orchestrator right after a restart) must
+    # not evict a previously cached spec: keep the last known good entry for
+    # any enabled connection that dropped out instead of caching its absence.
+    expected_urls = {
+        server_config['info']['id'] or str(idx): server_config['url']
+        for idx, server_config in enumerate(server_configs)
+        if server_config['config']['enable']
+    }
+    missing_ids = set(expected_urls) - {server.get('id') for server in terminal_servers}
+    if missing_ids:
+        previous_servers = getattr(request.app.state, 'TERMINAL_SERVERS', None) or []
+        if request.app.state.redis is not None:
+            try:
+                data = await request.app.state.redis.get(f'{REDIS_KEY_PREFIX}:terminal_servers')
+                if data is not None:
+                    previous_servers = JSONCodec.loads(data)
+            except Exception as e:
+                log.error(f'Error fetching terminal_servers from Redis: {e}')
+        for server in previous_servers:
+            server_id = server.get('id')
+            if server_id in missing_ids and server.get('url') == expected_urls[server_id]:
+                log.warning(
+                    "Spec fetch for terminal server '%s' failed; keeping previously cached specs",
+                    server_id,
+                )
+                terminal_servers.append(server)
+
+    request.app.state.TERMINAL_SERVERS = terminal_servers
 
     # Fetch system prompts concurrently (runs at cache time, not per-request)
     connections_by_id = {c.get('id'): c for c in connections if c.get('id')}
@@ -1323,9 +1352,12 @@ async def set_terminal_servers(request: Request):
     )
 
     if request.app.state.redis is not None:
-        await request.app.state.redis.set(
-            f'{REDIS_KEY_PREFIX}:terminal_servers', JSONCodec.dumps(request.app.state.TERMINAL_SERVERS)
-        )
+        try:
+            await request.app.state.redis.set(
+                f'{REDIS_KEY_PREFIX}:terminal_servers', JSONCodec.dumps(request.app.state.TERMINAL_SERVERS)
+            )
+        except Exception as e:
+            log.error(f'Error caching terminal_servers to Redis: {e}')
 
     return request.app.state.TERMINAL_SERVERS
 
@@ -1375,6 +1407,11 @@ async def get_terminal_tools(
     # Find the cached spec data for this terminal
     terminal_servers = await get_terminal_servers(request)
     server_data = next((s for s in terminal_servers if s.get('id') == terminal_id), None)
+    if server_data is None:
+        # The cache can lack this server when its spec fetch failed earlier
+        # (e.g. it was still starting up); rebuild once before giving up.
+        terminal_servers = await set_terminal_servers(request)
+        server_data = next((s for s in terminal_servers if s.get('id') == terminal_id), None)
     if server_data is None:
         raise RuntimeError(f"Terminal server '{terminal_id}' is unavailable")