diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 2ab4f64ad39..e247842fe61 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -211,6 +211,10 @@ async def _perform_health_check( healthy_endpoints = [] unhealthy_endpoints = [] + # Exceptions keyed by model_id; returned separately so callers can use + # them for cooldown integration without risking JSON-serialization errors + # in the /health response. + exceptions_by_model_id: dict = {} for is_healthy, model in zip(results, model_list): litellm_params = model["litellm_params"] @@ -225,14 +229,18 @@ async def _perform_health_check( cleaned = _clean_endpoint_data({**litellm_params, **is_healthy}, details) if _model_id: cleaned["model_id"] = _model_id + if "exception" in is_healthy: + exceptions_by_model_id[_model_id] = is_healthy["exception"] unhealthy_endpoints.append(cleaned) else: cleaned = _clean_endpoint_data(litellm_params, details) if _model_id: cleaned["model_id"] = _model_id + if isinstance(is_healthy, Exception): + exceptions_by_model_id[_model_id] = is_healthy unhealthy_endpoints.append(cleaned) - return healthy_endpoints, unhealthy_endpoints + return healthy_endpoints, unhealthy_endpoints, exceptions_by_model_id def build_deployment_health_states( @@ -413,7 +421,11 @@ async def perform_health_check( ) try: - healthy_endpoints, unhealthy_endpoints = await _perform_health_check( + ( + healthy_endpoints, + unhealthy_endpoints, + exceptions_by_model_id, + ) = await _perform_health_check( model_list, details, max_concurrency=max_concurrency, @@ -445,4 +457,4 @@ async def perform_health_check( _rss_mb_for_log(), ) - return healthy_endpoints, unhealthy_endpoints + return healthy_endpoints, unhealthy_endpoints, exceptions_by_model_id diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py index ae18a42c02b..5a65268b6fe 100644 --- a/litellm/proxy/health_check_utils/shared_health_check_manager.py +++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py @@ -231,7 +231,11 @@ class SharedHealthCheckManager: len(model_list), ) - healthy_endpoints, unhealthy_endpoints = await perform_health_check( + ( + healthy_endpoints, + unhealthy_endpoints, + exceptions_by_model_id, + ) = await perform_health_check( model_list=model_list, details=details, max_concurrency=max_concurrency, @@ -242,7 +246,7 @@ class SharedHealthCheckManager: healthy_endpoints, unhealthy_endpoints ) - return healthy_endpoints, unhealthy_endpoints + return healthy_endpoints, unhealthy_endpoints, exceptions_by_model_id finally: # Always release the lock diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index ef9436f2d8c..8a09edfd4c4 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -771,7 +771,7 @@ async def _perform_health_check_and_save( max_concurrency=None, ): """Helper function to perform health check and save results to database""" - healthy_endpoints, unhealthy_endpoints = await perform_health_check( + healthy_endpoints, unhealthy_endpoints, _ = await perform_health_check( model_list=model_list, cli_model=cli_model, model=target_model, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 839fcb8e8e2..fdcbd6fba25 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2287,6 +2287,7 @@ async def _run_background_health_check(): ( healthy_endpoints, unhealthy_endpoints, + _exceptions_by_model_id, ) = await shared_health_manager.perform_shared_health_check( model_list=_llm_model_list, details=details_bool, @@ -2300,6 +2301,7 @@ async def _run_background_health_check(): ( healthy_endpoints, unhealthy_endpoints, + _exceptions_by_model_id, ) = await _run_direct_health_check_with_instrumentation( _llm_model_list, health_check_details, @@ -2310,6 +2312,7 @@ async def _run_background_health_check(): ( healthy_endpoints, unhealthy_endpoints, + _exceptions_by_model_id, ) = await _run_direct_health_check_with_instrumentation( _llm_model_list, health_check_details, @@ -2352,7 +2355,9 @@ async def _run_background_health_check(): ) # Write health state to router cache for health-check-driven routing - _write_health_state_to_router_cache(healthy_endpoints, unhealthy_endpoints) + _write_health_state_to_router_cache( + healthy_endpoints, unhealthy_endpoints, _exceptions_by_model_id + ) await asyncio.sleep(health_check_interval)