From fd1d1c9011cc44893d41650d1360ff6dfa2b99a8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:25:21 -0700 Subject: [PATCH] fix(health): keep bad values from stopping checks and restore the skip flag 503 A bad model_info value raised inside the skip rule and ended the background loop, so the rule now fails open and the deployment fails its own check as it does on the base. A media deployment with disable_background_health_check true keeps its 503 when health_check_skip_disabled_background_models is on. The per-clip price lookup also reads the provider set apart from the model, and the dashboard shows the skip message as neutral text --- litellm/proxy/health_check.py | 57 +++++++++++++------ .../health_endpoints/_health_endpoints.py | 4 +- .../health_endpoints/test_health_endpoints.py | 40 +++++++++++++ .../proxy/test_health_check_max_tokens.py | 38 +++++++++++++ .../HealthCheckComponent.test.tsx | 1 + .../HealthChecksTableColumns.tsx | 9 +++ 6 files changed, 131 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index d258fd0e59c..f6e3556794a 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -197,7 +197,7 @@ def _get_process_rss_mb() -> float | None: if sys.platform == "darwin": return float(ru_maxrss) / (1024 * 1024) return float(ru_maxrss) / 1024 - except Exception: + except (ImportError, OSError): return None @@ -346,47 +346,70 @@ def _fallback_probe_mode(probe_params: Mapping[str, object]) -> str | None: ) -def _priced_per_generated_clip(model: object) -> bool: - entry: Final = cost_map_entry(model) if isinstance(model, str) else MappingProxyType({}) +def _bills_per_generated_clip(entry: Mapping[str, object]) -> bool: return bool(entry.get("output_cost_per_image")) and not entry.get("output_cost_per_token") +def _priced_per_generated_clip(probe_params: Mapping[str, object]) -> bool: + model: Final = probe_params.get("model") + provider: Final = probe_params.get("custom_llm_provider") + if not isinstance(model, str): + return False + names: Final = (model, f"{provider}/{model}") if isinstance(provider, str) else (model,) + return any(_bills_per_generated_clip(cost_map_entry(name)) for name in names) + + def _probe_generates_billed_media(model_info: Mapping[str, object], litellm_params: Mapping[str, object]) -> bool: mode: Final = resolve_health_check_mode(model_info, litellm_params) if mode in _BILLED_MEDIA_GENERATION_MODES: return True if not isinstance(litellm_params.get("model"), str): return False - try: - probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python( - update_litellm_params_for_health_check(dict(model_info), dict(litellm_params)) - ) - except (TypeError, ValueError): - return False + probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python( + update_litellm_params_for_health_check(dict(model_info), dict(litellm_params)) + ) return (mode is None and _fallback_probe_mode(probe_params) in _BILLED_MEDIA_GENERATION_MODES) or ( - _priced_per_generated_clip(probe_params.get("model")) + _priced_per_generated_clip(probe_params) ) +def _disable_background_flag(deployment: Mapping[str, object]) -> object: + model_info: Final = deployment.get("model_info") + return model_info.get("disable_background_health_check") if isinstance(model_info, Mapping) else None + + def bills_media_on_probe(deployment: Mapping[str, object]) -> bool: - model_info: Final = _str_key_mapping(deployment.get("model_info")) - explicit: Final = model_info.get("disable_background_health_check") + explicit: Final = _disable_background_flag(deployment) if explicit is not None and not explicit: return False - return _probe_generates_billed_media(model_info, _str_key_mapping(deployment.get("litellm_params"))) + try: + return _probe_generates_billed_media( + _str_key_mapping(deployment.get("model_info")), _str_key_mapping(deployment.get("litellm_params")) + ) + except Exception: + logger.debug("Could not work out the health check for %s", _deployment_id(deployment)) + return False def skips_background_health_check(deployment: Mapping[str, object]) -> bool: - disabled: Final = _str_key_mapping(deployment.get("model_info")).get("disable_background_health_check") - return bool(disabled) or bills_media_on_probe(deployment) + return bool(_disable_background_flag(deployment)) or bills_media_on_probe(deployment) def background_check_deployments(model_list: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: return tuple(deployment for deployment in model_list if not skips_background_health_check(deployment)) -def all_bill_media_on_probe(deployments: Sequence[Mapping[str, object]], ids: AbstractSet[object]) -> bool: - targets: Final = tuple(d for d in deployments if _deployment_id(d) in ids) +def all_bill_media_on_probe( + deployments: Sequence[Mapping[str, object]], + ids: AbstractSet[object], + health_check_skip_disabled_background_models: bool, +) -> bool: + targets: Final = tuple( + d + for d in deployments + if _deployment_id(d) in ids + and not (health_check_skip_disabled_background_models and _disable_background_flag(d)) + ) return bool(targets) and all(bills_media_on_probe(d) for d in targets) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 7d2a7a390a9..f21cc19b5f6 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -1694,7 +1694,9 @@ async def health_endpoint( }, ) if targeted_ids is not None and all_bill_media_on_probe( - DEPLOYMENTS_ADAPTER.validate_python(_llm_model_list), MODEL_IDS_ADAPTER.validate_python(targeted_ids) + DEPLOYMENTS_ADAPTER.validate_python(_llm_model_list), + MODEL_IDS_ADAPTER.validate_python(targeted_ids), + **_hc_filter, ): return { "healthy_endpoints": [], diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 6d1fa84f5bd..0454987ea65 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -2721,6 +2721,46 @@ async def test_targeted_health_answers_without_probing_billed_media(use_backgrou assert "warnings" not in results[checked] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_info", "expected_status"), + [ + ({"id": "image-1", "mode": "image_generation", "disable_background_health_check": True}, 503), + ({"id": "image-1", "mode": "image_generation"}, 200), + ], +) +async def test_skip_disabled_setting_keeps_the_503_for_an_opted_out_media_model(model_info, expected_status): + from fastapi import Response + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + model_list = [ + {"model_name": "image", "litellm_params": {"model": "gemini/gemini-3-pro-image"}, "model_info": model_info} + ] + response = Response() + with ( + _proxy_health_globals( + model_list, + litellm.Router(model_list=copy.deepcopy(model_list)), + use_background_health_checks=True, + health_check_results={"healthy_endpoints": [], "unhealthy_endpoints": []}, + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.general_settings", + {"health_check_skip_disabled_background_models": True}, + ), + ): + await health_endpoint( + response=response, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token"), + model=None, + model_id="image-1", + ) + + assert response.status_code == expected_status + + @pytest.mark.asyncio async def test_health_endpoint_filters_background_cache_by_user_access(): """ diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index c9830bcd65c..11c54786ad9 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -1271,6 +1271,44 @@ def test_health_checks_skip_billed_media_by_the_mode_the_probe_would_run(): ) +@pytest.mark.parametrize( + ("litellm_params", "billed"), + [ + ({"model": "lyria-002", "custom_llm_provider": "vertex_ai"}, True), + ({"model": "lyria-3-clip-preview", "custom_llm_provider": "gemini"}, True), + ({"model": "gemini-3.1-pro-preview", "custom_llm_provider": "gemini"}, False), + ({"model": "my-fine-tune", "custom_llm_provider": "vertex_ai"}, False), + ], +) +def test_per_clip_price_is_found_when_the_provider_is_set_apart_from_the_model( + litellm_params: dict[str, str], billed: bool +): + """Prices as model_prices_and_context_window.json listed them on 2026-10-31: the Lyria entries bill per clip with + no token price under their provider prefix alone, and gemini/gemini-3.1-pro-preview also has a token price.""" + deployment = {"litellm_params": litellm_params, "model_info": {"id": "clip", "mode": "audio_speech"}} + + assert hc_module.bills_media_on_probe(deployment) is billed + + +@pytest.mark.parametrize( + "model_info", + [ + {"health_check_max_tokens": float("inf")}, + {"health_check_max_tokens": "many"}, + {"health_check_model": 123}, + {1: "not a string key"}, + ], +) +def test_one_bad_model_info_value_cannot_stop_the_health_checks(model_info: dict[object, object]): + bad = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "bad", **model_info}} + control = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "control"}} + + assert hc_module.background_check_deployments([bad, control]) == (bad, control) + assert hc_module._health_check_eligible([bad, control], skip_disabled=True) == (bad, control) + assert hc_module._health_check_eligible([bad, control], skip_disabled=False) == (bad, control) + assert hc_module.all_bill_media_on_probe([bad, control], {"bad"}, False) is False + + def test_narrowing_by_an_id_that_matches_nothing_keeps_the_whole_list(): """Pinned because the disabled-dependency fix moved this filter into its own helper.""" deployments = [{"model_name": "a", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a-1"}}] diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx index 657b9049ed5..d6fde177be2 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx @@ -131,6 +131,7 @@ describe("HealthCheckComponent", () => { expect(screen.getByText("skipped")).toBeInTheDocument(); expect(screen.queryByText("healthy")).not.toBeInTheDocument(); expect(screen.getByText(warning)).toBeInTheDocument(); + expect(screen.getByText(warning)).not.toHaveClass("text-destructive"); } }); diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx index 4ee0eae5d89..e343992328e 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx @@ -340,6 +340,15 @@ export const getHealthChecksTableColumns = ({ } const cleanedError = healthStatus.error; + + if (healthStatus.status === "skipped") { + return ( + + {cleanedError} + + ); + } + const fullError = healthStatus.fullError || healthStatus.error; const displayName = getDisplayModelName(model) || model.model_name;