Fix: Handle PostgreSQL cached plan errors during rolling deployments

- Add _query_first_with_cached_plan_fallback() helper function to handle
  'cached plan must not change result type' errors
- This error occurs when schema changes (migrations) are applied during
  rolling deployments while old pods have cached query plans
- The fix automatically retries queries with modified SQL to force
  PostgreSQL to create fresh plans based on the new schema
- Addresses issue caused by migration 20260107111013_add_router_settings_to_keys_teams
  which added router_settings column to LiteLLM_VerificationToken table
- The combined_view query uses SELECT v.* which is vulnerable to schema changes
This commit is contained in:
Alexsander Hamir 2026-01-20 10:31:57 -08:00
parent ce37729da4
commit 20423248df

View file

@ -2214,6 +2214,45 @@ class PrismaClient:
raise e
async def _query_first_with_cached_plan_fallback(
self, sql_query: str
) -> Optional[dict]:
"""
Execute a query with automatic fallback for PostgreSQL cached plan errors.
This handles the "cached plan must not change result type" error that occurs
during rolling deployments when schema changes are applied while old pods
still have cached query plans expecting the old schema.
Args:
sql_query: SQL query string to execute
Returns:
Query result or None
Raises:
Original exception if not a cached plan error
"""
try:
return await self.db.query_first(query=sql_query)
except Exception as e:
error_str = str(e)
if "cached plan must not change result type" in error_str:
# Force PostgreSQL to re-plan by invalidating the cache
# Add a unique comment to make the query different
sql_query_retry = sql_query.replace(
"SELECT",
f"SELECT /* cache_invalidated_{int(time.time() * 1000)} */"
)
verbose_proxy_logger.warning(
"PostgreSQL cached plan error detected for token lookup, "
"retrying with fresh plan. This may occur during rolling deployments "
"when schema changes are applied."
)
return await self.db.query_first(query=sql_query_retry)
else:
raise
@backoff.on_exception(
backoff.expo,
Exception, # base exception to catch for the backoff
@ -2545,7 +2584,7 @@ class PrismaClient:
WHERE v.token = '{token}'
"""
response = await self.db.query_first(query=sql_query)
response = await self._query_first_with_cached_plan_fallback(sql_query)
if response is not None:
if response["team_models"] is None: