From 51368f8d1d2e6c4635618862cbc4e7302ebb6071 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:57:52 -0700 Subject: [PATCH 1/7] fix(health)!: skip billed media generation in background health checks by default The background health check probe for image_generation, image_edit and video_generation deployments is a real generation call, so every interval billed an image or video with no traffic. The loop now skips those modes unless the deployment sets disable_background_health_check: false. health_check_skip_disabled_background_models applies the same skip to GET /health. --- litellm/proxy/_types.py | 5 +- litellm/proxy/health_check.py | 31 ++++--- .../health_endpoints/_health_endpoints.py | 5 +- litellm/proxy/proxy_server.py | 5 +- .../proxy_server/test_background_health.py | 82 +++++++++++++++++++ .../proxy/test_health_check_max_tokens.py | 17 ++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 7 +- 7 files changed, 132 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 0a09fe4a855..09f26542439 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2694,8 +2694,9 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): health_check_skip_disabled_background_models: bool = Field( False, description=( - "When true, deployments with model_info.disable_background_health_check " - "are skipped for on-demand GET /health as well as the background health loop." + "When true, deployments the background health loop skips (model_info.disable_background_health_check, " + "which defaults to true for image_generation, image_edit, and video_generation deployments) " + "are skipped for on-demand GET /health as well." ), ) background_health_check_model_groups: tuple[str, ...] | None = Field( diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 73c2fc35492..80b4dd21133 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -230,9 +230,9 @@ def health_check_filter_kwargs_from_general_settings( """ Build kwargs for ``perform_health_check`` from ``general_settings``. - When ``health_check_skip_disabled_background_models`` is true, deployments with - ``model_info.disable_background_health_check`` are omitted from health runs - (including on-demand ``GET /health``), matching the background loop behavior. + When ``health_check_skip_disabled_background_models`` is true, deployments the + background loop skips are omitted from health runs (including on-demand + ``GET /health``). """ g: Final = general_settings or {} return { @@ -307,9 +307,21 @@ async def run_with_timeout(task, timeout): return {"error": "Timeout exceeded", "exception": timeout_exception} -def _skips_health_checks(deployment: Mapping[str, object]) -> bool: - info: Final = deployment.get("model_info") - return bool(info.get("disable_background_health_check", False)) if isinstance(info, Mapping) else False +_BILLED_MEDIA_GENERATION_MODES: Final = frozenset({"image_generation", "image_edit", "video_generation"}) +_STR_KEY_MAPPING: Final = TypeAdapter(Mapping[str, object]) + + +def _str_key_mapping(value: object) -> Mapping[str, object]: + return _STR_KEY_MAPPING.validate_python(value) if isinstance(value, Mapping) else MappingProxyType({}) + + +def skips_background_health_check(deployment: Mapping[str, object]) -> bool: + model_info: Final = _str_key_mapping(deployment.get("model_info")) + explicit: Final = model_info.get("disable_background_health_check") + if explicit is not None: + return bool(explicit) + litellm_params: Final = _str_key_mapping(deployment.get("litellm_params")) + return resolve_health_check_mode(model_info, litellm_params) in _BILLED_MEDIA_GENERATION_MODES def _health_check_eligible( @@ -321,7 +333,7 @@ def _health_check_eligible( dependencies are drawn from alike, so an opted-out deployment cannot re-enter through a router that depends on it. """ - return tuple(x for x in model_list if not (skip_disabled and _skips_health_checks(x))) + return tuple(x for x in model_list if not (skip_disabled and skips_background_health_check(x))) def _deployment_model(deployment: Mapping[str, object]) -> str | None: @@ -970,9 +982,8 @@ async def perform_health_check( When ``health_check_skip_disabled_background_models`` is True (via ``general_settings.health_check_skip_disabled_background_models``), deployments - with ``model_info.disable_background_health_check: true`` are omitted from - this run (including targeted ``/health`` queries), consistent with the - background health loop. + the background health loop skips are omitted from this run (including targeted + ``/health`` queries). Returns: (bool): True if the health check passes, False otherwise. diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 59c63a18f0f..4deade5d7de 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -1238,8 +1238,9 @@ async def health_endpoint( ``` else, the health checks will be run on models when /health is called. - To skip deployments that set ``model_info.disable_background_health_check: true`` - on ``GET /health`` as well as in the background loop, set + The background loop skips deployments that set ``model_info.disable_background_health_check: true``, + and image_generation, image_edit, and video_generation deployments unless they set it to ``false``, + since those probes generate billed media. To skip the same deployments on ``GET /health``, set ``general_settings.health_check_skip_disabled_background_models: true``. """ import time diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b0ccf74c63b..eb55cb991c4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -652,6 +652,7 @@ from litellm.proxy.health_check import ( health_check_filter_kwargs_from_general_settings, parse_background_health_check_model_groups, perform_health_check, + skips_background_health_check, ) from litellm.proxy.health_endpoints._health_endpoints import router as health_router from litellm.proxy.health_endpoints.deployment_warning_endpoints import router as deployment_warning_router @@ -4819,9 +4820,7 @@ async def _run_background_health_check(): model_count_total = len(_llm_model_list) # filter out models that have disabled background health checks - _llm_model_list = [ - m for m in _llm_model_list if not m.get("model_info", {}).get("disable_background_health_check", False) - ] + _llm_model_list = [m for m in _llm_model_list if not skips_background_health_check(m)] scoped_model_groups = ( llm_router.settings().background_health_check_model_groups if llm_router is not None else None ) diff --git a/tests/unit/proxy/proxy_server/test_background_health.py b/tests/unit/proxy/proxy_server/test_background_health.py index ebc33b1c788..337c147f2cb 100644 --- a/tests/unit/proxy/proxy_server/test_background_health.py +++ b/tests/unit/proxy/proxy_server/test_background_health.py @@ -772,3 +772,85 @@ async def test_run_background_health_check_probes_only_listed_model_groups(monke await _run_background_health_check() assert probed["ids"] == ["listed-1", "listed-2"] + + +@pytest.mark.asyncio +async def test_run_background_health_check_skips_billed_media_generation_unless_opted_in(monkeypatch): + monkeypatch.setattr(proxy_server, "health_check_interval", 60) + monkeypatch.setattr(proxy_server, "health_check_concurrency", 1) + monkeypatch.setattr(proxy_server, "health_check_details", True) + monkeypatch.setattr(proxy_server, "use_shared_health_check", False) + monkeypatch.setattr(proxy_server, "redis_usage_cache", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "background_health_check_loop_active", False) + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr( + proxy_server, + "llm_model_list", + [ + {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat"}}, + { + "model_name": "image", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": "image", "mode": "image_generation"}, + }, + { + "model_name": "image-edit", + "litellm_params": {"model": "openai/gpt-image-1"}, + "model_info": {"id": "image-edit", "mode": "image_edit"}, + }, + { + "model_name": "video", + "litellm_params": {"model": "openai/sora-2"}, + "model_info": {"id": "video", "mode": "video_generation"}, + }, + { + "model_name": "speech", + "litellm_params": {"model": "openai/tts-1"}, + "model_info": {"id": "speech", "mode": "audio_speech"}, + }, + { + "model_name": "image-opted-in", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": { + "id": "image-opted-in", + "mode": "image_generation", + "disable_background_health_check": False, + }, + }, + ], + ) + monkeypatch.setattr( + proxy_server, + "health_check_results", + {"healthy_endpoints": [], "unhealthy_endpoints": []}, + ) + + probed = {} + + async def _fake_direct(model_list, *_a, **_kw): + probed["ids"] = [m["model_info"]["id"] for m in model_list] + return ([], [], {}) + + monkeypatch.setattr( + proxy_server, + "_run_direct_health_check_with_instrumentation", + _fake_direct, + ) + monkeypatch.setattr(proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None) + monkeypatch.setattr( + proxy_server, + "health_check_filter_kwargs_from_general_settings", + lambda _gs: {}, + ) + + async def _stop_sleep(_seconds): + raise asyncio.CancelledError() + + monkeypatch.setattr(proxy_server.asyncio, "sleep", _stop_sleep) + + with pytest.raises(asyncio.CancelledError): + await _run_background_health_check() + + assert probed["ids"] == ["chat", "speech", "image-opted-in"] diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index e48bf4edc09..a8d13e4286a 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -1226,6 +1226,23 @@ def test_a_dependency_that_opted_out_of_health_checks_is_never_probed(): assert [d["model_info"]["id"] for d in eligible] == ["router-1"] +def test_health_skip_flag_also_skips_media_deployments_whose_mode_comes_from_the_cost_map(): + """gemini/gemini-3-pro-image is mode image_generation in model_prices_and_context_window.json.""" + image = { + "model_name": "image", + "litellm_params": {"model": "gemini/gemini-3-pro-image", "api_key": "sk-x"}, + "model_info": {"id": "image-1"}, + } + chat = { + "model_name": "chat", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}, + "model_info": {"id": "chat-1"}, + } + + assert hc_module._health_check_eligible([image, chat], skip_disabled=True) == (chat,) + assert hc_module._health_check_eligible([image, chat], skip_disabled=False) == (image, chat) + + def test_narrowing_by_an_id_that_matches_nothing_keeps_the_whole_list(): """Pinned because the disabled-dependency fix moved this filter into its own helper.""" deployments = [{"model_name": "a", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a-1"}}] diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 13a064517a4..ce143703648 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7555,8 +7555,9 @@ export interface paths { * ``` * else, the health checks will be run on models when /health is called. * - * To skip deployments that set ``model_info.disable_background_health_check: true`` - * on ``GET /health`` as well as in the background loop, set + * The background loop skips deployments that set ``model_info.disable_background_health_check: true``, + * and image_generation, image_edit, and video_generation deployments unless they set it to ``false``, + * since those probes generate billed media. To skip the same deployments on ``GET /health``, set * ``general_settings.health_check_skip_disabled_background_models: true``. */ get: operations["health_endpoint_health_get"]; @@ -30106,7 +30107,7 @@ export interface components { health_check_interval: number; /** * Health Check Skip Disabled Background Models - * @description When true, deployments with model_info.disable_background_health_check are skipped for on-demand GET /health as well as the background health loop. + * @description When true, deployments the background health loop skips (model_info.disable_background_health_check, which defaults to true for image_generation, image_edit, and video_generation deployments) are skipped for on-demand GET /health as well. * @default false */ health_check_skip_disabled_background_models: boolean; From 54cffe3477e764ac47424a0803d5addbeb302752 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:22:38 -0700 Subject: [PATCH 2/7] fix(health): judge the skip by the mode the probe runs and keep targeted /health answering The skip now uses the same mode the probe falls back to, so a health_check_model or an azure_ai gpt-image deployment that resolves to an image model is skipped too. A targeted GET /health on a media deployment the background loop skips runs the live check instead of reading an empty cache and answering 503. --- litellm/proxy/_types.py | 5 +- litellm/proxy/health_check.py | 76 ++++++++++++++----- .../shared_health_check_manager.py | 2 +- .../health_endpoints/_health_endpoints.py | 13 +++- litellm/proxy/proxy_server.py | 6 +- .../health_endpoints/test_health_endpoints.py | 66 ++++++++++++++++ .../proxy_server/test_background_health.py | 13 +++- .../proxy/test_health_check_max_tokens.py | 47 +++++++----- ui/litellm-dashboard/src/lib/http/schema.d.ts | 7 +- 9 files changed, 189 insertions(+), 46 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 09f26542439..7f821cd0413 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2707,8 +2707,9 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): "whose model_name is listed, and enable_health_check_routing filters unhealthy " "deployments only within the listed groups; every other group, including newly " "added deployments, is skipped and keeps its configured routing strategy. " - "When unset, all deployments participate (opt out per deployment via " - "model_info.disable_background_health_check)." + "When unset, every deployment participates except those that set " + "model_info.disable_background_health_check, which defaults to true for " + "image_generation, image_edit, and video_generation deployments." ), ) model_list_healthy_only: bool | None = Field( diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 80b4dd21133..0ddb4e3753e 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -9,7 +9,7 @@ import time from collections.abc import Mapping, Sequence from collections.abc import Set as AbstractSet from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeVar +from typing import TYPE_CHECKING, Final, Literal, TypeVar from pydantic import TypeAdapter, ValidationError @@ -26,7 +26,7 @@ from litellm.constants import ( DEFAULT_HEALTH_CHECK_PROMPT, HEALTH_CHECK_TIMEOUT_SECONDS, ) -from litellm.litellm_core_utils.health_check_helpers import native_health_check_mode +from litellm.litellm_core_utils.health_check_helpers import default_health_check_mode, native_health_check_mode from litellm.router_strategy.auto_router.auto_router_model_naming import ( StrategyRouterDependency, classify_strategy_router_model, @@ -308,20 +308,63 @@ async def run_with_timeout(task, timeout): _BILLED_MEDIA_GENERATION_MODES: Final = frozenset({"image_generation", "image_edit", "video_generation"}) -_STR_KEY_MAPPING: Final = TypeAdapter(Mapping[str, object]) +DEPLOYMENT_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +DEPLOYMENTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +MODEL_IDS_ADAPTER: Final = TypeAdapter(frozenset[object]) +_STR: Final = TypeAdapter(str) def _str_key_mapping(value: object) -> Mapping[str, object]: - return _STR_KEY_MAPPING.validate_python(value) if isinstance(value, Mapping) else MappingProxyType({}) + return DEPLOYMENT_ADAPTER.validate_python(value) if isinstance(value, Mapping) else MappingProxyType({}) -def skips_background_health_check(deployment: Mapping[str, object]) -> bool: +def _fallback_probe_mode(probe_params: Mapping[str, object]) -> str | None: + """The mode `litellm.ahealth_check` picks for these probe params when the caller passes none.""" + provider: Final = probe_params.get("custom_llm_provider") + api_base: Final = probe_params.get("api_base") + api_key: Final = probe_params.get("api_key") + try: + model: Final = _STR.validate_python(probe_params.get("model")) + resolved_model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model, + custom_llm_provider=provider if isinstance(provider, str) else None, + api_base=api_base if isinstance(api_base, str) else None, + api_key=api_key if isinstance(api_key, str) else None, + ) + except (ValidationError, litellm.BadRequestError): + return None + return default_health_check_mode( + requested_model=model, model=resolved_model, custom_llm_provider=custom_llm_provider + ) + + +def _probe_mode(model_info: Mapping[str, object], litellm_params: Mapping[str, object]) -> str | None: + resolved: Final = resolve_health_check_mode(model_info, litellm_params) + if resolved is not None or not isinstance(litellm_params.get("model"), str): + return resolved + return _fallback_probe_mode( + _PROBE_PARAMS_ADAPTER.validate_python( + update_litellm_params_for_health_check(dict(model_info), dict(litellm_params)) + ) + ) + + +def background_health_check_skip(deployment: Mapping[str, object]) -> Literal["opted_out", "billed_media"] | None: model_info: Final = _str_key_mapping(deployment.get("model_info")) explicit: Final = model_info.get("disable_background_health_check") if explicit is not None: - return bool(explicit) + return "opted_out" if explicit else None litellm_params: Final = _str_key_mapping(deployment.get("litellm_params")) - return resolve_health_check_mode(model_info, litellm_params) in _BILLED_MEDIA_GENERATION_MODES + return "billed_media" if _probe_mode(model_info, litellm_params) in _BILLED_MEDIA_GENERATION_MODES else None + + +def skips_background_health_check(deployment: Mapping[str, object]) -> bool: + return background_health_check_skip(deployment) is not None + + +def all_skipped_as_billed_media(deployments: Sequence[Mapping[str, object]], ids: AbstractSet[object]) -> bool: + targets: Final = tuple(d for d in deployments if _deployment_id(d) in ids) + return bool(targets) and all(background_health_check_skip(d) == "billed_media" for d in targets) def _health_check_eligible( @@ -329,9 +372,9 @@ def _health_check_eligible( ) -> tuple[Mapping[str, object], ...]: """Deployments this run is allowed to contact. - The one eligibility gate, applied to the requested set and to the pool a router's - dependencies are drawn from alike, so an opted-out deployment cannot re-enter through a - router that depends on it. + The one eligibility gate, applied to the requested set and to each round of a router's + dependencies alike, so an opted-out deployment cannot re-enter through a router that + depends on it. """ return tuple(x for x in model_list if not (skip_disabled and skips_background_health_check(x))) @@ -483,6 +526,7 @@ def _dependency_deployments_to_probe( checked: Sequence[Mapping[str, object]], universe: Sequence[Mapping[str, object]], router: "Router", + skip_disabled: bool = False, ) -> tuple[Mapping[str, object], ...]: """Deployments backing the checked routers' dependencies that are not already checked. @@ -490,7 +534,7 @@ def _dependency_deployments_to_probe( `/health?model_id=` call the dashboard makes per deployment that needs them, since a router's verdict is a statement about models the request never named. Drawn from `universe`, the caller's access-filtered list, so no deployment is probed that the caller - was not already granted. Expansion follows routers through routers, one hop per round, + was not already granted, and only those `_health_check_eligible` passes. Expansion follows routers through routers, one hop per round, because a child router's own models must be probed for the parent to fail; stopping when a round adds nothing is what makes a router cycle terminate. """ @@ -510,9 +554,9 @@ def _dependency_deployments_to_probe( ) if not fresh_ids: break - frontier = _deployments_by_id(universe, fresh_ids) + frontier = _health_check_eligible(_deployments_by_id(universe, fresh_ids), skip_disabled) reached = reached | fresh_ids - return _deployments_by_id(universe, reached - checked_ids) + return _health_check_eligible(_deployments_by_id(universe, reached - checked_ids), skip_disabled) def _strategy_router_verdicts( @@ -616,7 +660,7 @@ async def _run_model_health_check(model: dict): return await run_with_timeout( litellm.ahealth_check( probe_params, - mode=mode, + mode=mode or _fallback_probe_mode(_PROBE_PARAMS_ADAPTER.validate_python(probe_params)), prompt=DEFAULT_HEALTH_CHECK_PROMPT, input=["test from litellm"], ), @@ -1023,9 +1067,7 @@ async def perform_health_check( deduped_model_count: Final = len(requested) dependency_probes: Final = ( - _dependency_deployments_to_probe(requested, _health_check_eligible(model_list, skip_disabled), router) - if router is not None - else () + _dependency_deployments_to_probe(requested, model_list, router, skip_disabled) if router is not None else () ) checked: Final = requested + list(dependency_probes) diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py index d689ab2a537..81ad195bd0b 100644 --- a/litellm/proxy/health_check_utils/shared_health_check_manager.py +++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py @@ -225,7 +225,7 @@ class SharedHealthCheckManager: model_list: List of models to check details: Whether to include detailed information max_concurrency: Optional limit on concurrent health check requests - health_check_skip_disabled_background_models: Remove models with disable_background_health_check: true + health_check_skip_disabled_background_models: Remove models the background loop skips Returns: Tuple of (healthy_endpoints, unhealthy_endpoints) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 4deade5d7de..1be78fd9f0f 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -56,8 +56,11 @@ from litellm.proxy.db.health_check_latest import ( from litellm.proxy.db.proxy_worker_heartbeat import count_live_proxy_workers from litellm.proxy.health_check import ( # noqa: F401 # legacy module exports ADMIN_ONLY_HEALTH_DISPLAY_PARAMS, + DEPLOYMENTS_ADAPTER, + MODEL_IDS_ADAPTER, _clean_endpoint_data, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _update_litellm_params_for_health_check, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + all_skipped_as_billed_media, clean_endpoint_data, deployments_targeted_by_name, health_check_filter_kwargs_from_general_settings, @@ -1240,8 +1243,9 @@ async def health_endpoint( The background loop skips deployments that set ``model_info.disable_background_health_check: true``, and image_generation, image_edit, and video_generation deployments unless they set it to ``false``, - since those probes generate billed media. To skip the same deployments on ``GET /health``, set - ``general_settings.health_check_skip_disabled_background_models: true``. + since those probes generate billed media. A ``model`` or ``model_id`` that targets only such media + deployments runs the live check, since the background results never hold them. To skip the same + deployments on ``GET /health``, set ``general_settings.health_check_skip_disabled_background_models: true``. """ import time @@ -1323,7 +1327,10 @@ async def health_endpoint( else f"key not allowed to health-check model {model}" }, ) - if use_background_health_checks: + targets_only_skipped_media: Final = targeted_ids is not None and all_skipped_as_billed_media( + DEPLOYMENTS_ADAPTER.validate_python(_llm_model_list), MODEL_IDS_ADAPTER.validate_python(targeted_ids) + ) + if use_background_health_checks and not targets_only_skipped_media: # The cached background result covers every model. When the # caller targets a specific model/model_id we have to narrow the # cache to that deployment before _post_process evaluates diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index eb55cb991c4..05403a34189 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -648,6 +648,7 @@ from litellm.proxy.guardrails.init_guardrails import ( initialize_guardrails, ) from litellm.proxy.health_check import ( + DEPLOYMENT_ADAPTER, filter_deployments_to_model_groups, health_check_filter_kwargs_from_general_settings, parse_background_health_check_model_groups, @@ -4819,12 +4820,13 @@ async def _run_background_health_check(): _llm_model_list = copy.deepcopy(llm_model_list) or [] model_count_total = len(_llm_model_list) - # filter out models that have disabled background health checks - _llm_model_list = [m for m in _llm_model_list if not skips_background_health_check(m)] scoped_model_groups = ( llm_router.settings().background_health_check_model_groups if llm_router is not None else None ) _llm_model_list = list(filter_deployments_to_model_groups(_llm_model_list, scoped_model_groups)) + _llm_model_list = [ + m for m in _llm_model_list if not skips_background_health_check(DEPLOYMENT_ADAPTER.validate_python(m)) + ] if scoped_model_groups is not None and not _llm_model_list: verbose_proxy_logger.warning( "background_health_check_model_groups matched no deployments; groups=%s", diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 9847787efce..74068bbc3e2 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -2635,6 +2635,72 @@ async def test_health_endpoint_omits_model_id_warning_when_no_deployment_matches assert "warnings" not in result +@pytest.mark.asyncio +async def test_targeted_health_runs_live_for_media_the_background_loop_skips_by_default(): + from fastapi import Response + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + model_list = [ + { + "model_name": "image", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": "image-1", "mode": "image_generation"}, + }, + { + "model_name": "image-opted-out", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": "image-off", "mode": "image_generation", "disable_background_health_check": True}, + }, + {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat-1"}}, + ] + cached = { + "healthy_endpoints": [{"model": "openai/gpt-4o-mini", "model_id": "chat-1"}], + "unhealthy_endpoints": [], + "healthy_count": 1, + "unhealthy_count": 0, + } + live_model_ids: list[str | None] = [] + + async def fake_perform(**kwargs): + live_model_ids.append(kwargs["model_id"]) + return { + "healthy_endpoints": [{"model": "gemini/gemini-3-pro-image", "model_id": kwargs["model_id"]}], + "unhealthy_endpoints": [], + "healthy_count": 1, + "unhealthy_count": 0, + } + + admin = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token") + responses = {model_id: Response() for model_id in ("image-1", "image-off", "chat-1")} + with ( + _proxy_health_globals( + model_list, + litellm.Router(model_list=copy.deepcopy(model_list)), + use_background_health_checks=True, + health_check_results=cached, + ), + patch( # test-quality-ok: whether the live probe ran is the assertion; no injection seam + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", + side_effect=fake_perform, + ), + ): + results = { + model_id: await health_endpoint( + response=response, user_api_key_dict=admin, model=None, model_id=model_id + ) + for model_id, response in responses.items() + } + + assert live_model_ids == ["image-1"] + assert results["image-1"]["healthy_count"] == 1 + assert responses["image-1"].status_code == 200 + assert results["chat-1"]["healthy_count"] == 1 + assert results["image-off"]["healthy_count"] == 0 + assert responses["image-off"].status_code == 503 + + @pytest.mark.asyncio async def test_health_endpoint_filters_background_cache_by_user_access(): """ diff --git a/tests/unit/proxy/proxy_server/test_background_health.py b/tests/unit/proxy/proxy_server/test_background_health.py index 337c147f2cb..4c487ddd567 100644 --- a/tests/unit/proxy/proxy_server/test_background_health.py +++ b/tests/unit/proxy/proxy_server/test_background_health.py @@ -809,6 +809,17 @@ async def test_run_background_health_check_skips_billed_media_generation_unless_ "litellm_params": {"model": "openai/tts-1"}, "model_info": {"id": "speech", "mode": "audio_speech"}, }, + { + "model_name": "embedding", + "litellm_params": {"model": "openai/text-embedding-3-small"}, + "model_info": {"id": "embedding", "mode": "embedding"}, + }, + {"model_name": "fine-tune", "litellm_params": {"model": "openai/my-fine-tune"}, "model_info": {"id": "fine-tune"}}, + { + "model_name": "azure-ai-image", + "litellm_params": {"model": "azure_ai/gpt-image-1", "api_base": "https://example.services.ai.azure.com"}, + "model_info": {"id": "azure-ai-image"}, + }, { "model_name": "image-opted-in", "litellm_params": {"model": "gemini/gemini-3-pro-image"}, @@ -853,4 +864,4 @@ async def test_run_background_health_check_skips_billed_media_generation_unless_ with pytest.raises(asyncio.CancelledError): await _run_background_health_check() - assert probed["ids"] == ["chat", "speech", "image-opted-in"] + assert probed["ids"] == ["chat", "speech", "embedding", "fine-tune", "image-opted-in"] diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index a8d13e4286a..1b2e3597e5b 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -1219,28 +1219,41 @@ def test_a_dependency_that_opted_out_of_health_checks_is_never_probed(): ) marker = [d for d in router.model_list if d["model_info"]["id"] == "router-1"] - eligible = hc_module._health_check_eligible(router.model_list, skip_disabled=True) - probes = hc_module._dependency_deployments_to_probe(marker, eligible, router) + probes = hc_module._dependency_deployments_to_probe(marker, router.model_list, router, skip_disabled=True) assert probes == () - assert [d["model_info"]["id"] for d in eligible] == ["router-1"] + assert [d["model_info"]["id"] for d in hc_module._dependency_deployments_to_probe(marker, router.model_list, router)] == [ + "dead-1" + ] -def test_health_skip_flag_also_skips_media_deployments_whose_mode_comes_from_the_cost_map(): - """gemini/gemini-3-pro-image is mode image_generation in model_prices_and_context_window.json.""" - image = { - "model_name": "image", - "litellm_params": {"model": "gemini/gemini-3-pro-image", "api_key": "sk-x"}, - "model_info": {"id": "image-1"}, - } - chat = { - "model_name": "chat", - "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}, - "model_info": {"id": "chat-1"}, - } +def test_health_skip_flag_skips_media_by_the_mode_the_probe_would_run(): + """Modes as model_prices_and_context_window.json listed them on 2026-10-29: gemini/gemini-3-pro-image and + gpt-image-1 are image_generation, openai/text-embedding-3-small is embedding.""" - assert hc_module._health_check_eligible([image, chat], skip_disabled=True) == (chat,) - assert hc_module._health_check_eligible([image, chat], skip_disabled=False) == (image, chat) + def deployment(model_id: str, model: str, **model_info: object) -> dict[str, object]: + return { + "model_name": model_id, + "litellm_params": {"model": model, "api_key": "sk-x", "api_base": "https://example.services.ai.azure.com"}, + "model_info": {"id": model_id, **model_info}, + } + + skipped = ( + deployment("cost-map-image", "gemini/gemini-3-pro-image"), + deployment("azure-ai-image", "azure_ai/gpt-image-1"), + deployment("wildcard-probing-an-image", "openai/*", health_check_model="openai/gpt-image-1"), + ) + probed = ( + deployment("chat", "openai/gpt-4o-mini"), + deployment("embedding", "openai/text-embedding-3-small"), + deployment("wildcard", "openai/*"), + deployment("unknown", "openai/my-fine-tune"), + deployment("no-provider-prefix", "my-fine-tune"), + deployment("image-opted-in", "gemini/gemini-3-pro-image", disable_background_health_check=False), + ) + + assert hc_module._health_check_eligible([*skipped, *probed], skip_disabled=True) == probed + assert hc_module._health_check_eligible([*skipped, *probed], skip_disabled=False) == (*skipped, *probed) def test_narrowing_by_an_id_that_matches_nothing_keeps_the_whole_list(): diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ce143703648..03d53487712 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7557,8 +7557,9 @@ export interface paths { * * The background loop skips deployments that set ``model_info.disable_background_health_check: true``, * and image_generation, image_edit, and video_generation deployments unless they set it to ``false``, - * since those probes generate billed media. To skip the same deployments on ``GET /health``, set - * ``general_settings.health_check_skip_disabled_background_models: true``. + * since those probes generate billed media. A ``model`` or ``model_id`` that targets only such media + * deployments runs the live check, since the background results never hold them. To skip the same + * deployments on ``GET /health``, set ``general_settings.health_check_skip_disabled_background_models: true``. */ get: operations["health_endpoint_health_get"]; put?: never; @@ -29908,7 +29909,7 @@ export interface components { apply_user_budget_to_team_keys?: boolean | null; /** * Background Health Check Model Groups - * @description Opt-in allowlist of model group names for background health checks and health-check routing. When set, the background loop probes only deployments whose model_name is listed, and enable_health_check_routing filters unhealthy deployments only within the listed groups; every other group, including newly added deployments, is skipped and keeps its configured routing strategy. When unset, all deployments participate (opt out per deployment via model_info.disable_background_health_check). + * @description Opt-in allowlist of model group names for background health checks and health-check routing. When set, the background loop probes only deployments whose model_name is listed, and enable_health_check_routing filters unhealthy deployments only within the listed groups; every other group, including newly added deployments, is skipped and keeps its configured routing strategy. When unset, every deployment participates except those that set model_info.disable_background_health_check, which defaults to true for image_generation, image_edit, and video_generation deployments. */ background_health_check_model_groups?: string[] | null; /** From 8d36201dbe58fbe4254ecee9a75fc5df0b39381d Mon Sep 17 00:00:00 2001 From: Shayan12456 <136985581+Shayan12456@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:21:34 +0000 Subject: [PATCH 3/7] fix(health): skip background health checks on paid media modes by default Background health checks dispatched real, billed generation calls on image_generation, image_edit, video_generation, and audio_speech deployments. Unlike chat or completion, these modes have no max_tokens-style cap, so an unconditional probe on a tight polling interval runs up unbounded provider spend with no user-facing request involved. One reporter's proxy racked up thousands of dollars in a few days from gemini-3-pro-image health checks alone. Skip these modes by default in _run_model_health_check and report them healthy without calling the provider. model_info.health_check_paid_media: true opts a deployment back into the real probe, the same pattern already used for disable_background_health_check. Fixes #49089 --- litellm/proxy/health_check.py | 15 ++++ .../test_health_check_helpers.py | 1 + .../proxy/test_health_check_max_tokens.py | 81 +++++++++++++++++-- 3 files changed, 92 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 60d2abe0ab4..e522a0d72ee 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -79,6 +79,19 @@ HEALTH_DISPLAY_PARAMS: Final = ( # Per-deployment override: `model_info.health_check_supports_max_tokens`. _MAX_TOKEN_SUPPORT_MODES: Final[frozenset[str]] = frozenset({"chat", "completion", "responses", "anthropic_messages"}) +# Modes whose probe is a real, billed generation call with no cheap/free way to +# bound its cost (no `max_tokens`-style cap, unlike chat). A background health +# check loop polling every 30-60s against one of these can run up unbounded +# provider spend with no user-facing request involved. Skipped by default; +# per-deployment override: `model_info.health_check_paid_media: true`. +_PAID_MEDIA_HEALTH_CHECK_MODES: Final[frozenset[str]] = frozenset( + {"image_generation", "image_edit", "video_generation", "audio_speech"} +) + + +def _requires_paid_media_opt_in(mode: str | None, model_info: Mapping[str, object]) -> bool: + return mode in _PAID_MEDIA_HEALTH_CHECK_MODES and not model_info.get("health_check_paid_media", False) + def _native_health_check_mode(model: str, provider_param: object) -> str | None: try: @@ -593,6 +606,8 @@ async def _run_model_health_check(model: dict): model_info, litellm_params, # any-ok: untyped router config dict ) + if _requires_paid_media_opt_in(mode, model_info): # pyright: ignore[reportUnknownArgumentType] # model_info is the same untyped router dict every other call in this function already passes around + return {} # pyright: ignore[reportUnknownVariableType] # matches the untyped early return 2 lines above; this function has no return type annotation litellm_params = update_litellm_params_for_health_check( # rebind-ok: pre-existing rebinding on a rename-only line model_info, litellm_params ) diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 6979d72ff14..84ddf063564 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1264,6 +1264,7 @@ async def test_image_generation_health_check_prompt(monkeypatch): "litellm_params": {"model": "gpt-image-1", "api_key": "fake-key"}, "model_info": { "mode": "image_generation", + "health_check_paid_media": True, }, } ] diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index e48bf4edc09..ce029126435 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -760,6 +760,57 @@ async def test_run_model_health_check_skips_auto_router_deployment(): assert result == {} +@pytest.mark.asyncio +@pytest.mark.parametrize( + "mode", + ["image_generation", "image_edit", "video_generation", "audio_speech"], +) +async def test_run_model_health_check_skips_paid_media_mode_by_default(mode): + """A costly generative health check must never fire unless the deployment opts in. + + Regression for a proxy that billed thousands of dollars in real image generation + from its own background health check loop: unlike chat/completion, these modes have + no `max_tokens`-style cap, so an unconditional probe on a tight polling interval can + run up unbounded provider spend. + """ + fake_ahealth_check = AsyncMock(return_value={}) + model = { + "litellm_params": {"model": "openai/dall-e-3"}, + "model_info": {"mode": mode}, + } + + with patch.object(hc_module.litellm, "ahealth_check", fake_ahealth_check): + result = await hc_module._run_model_health_check(model) + + fake_ahealth_check.assert_not_called() + assert result == {} + + +@pytest.mark.asyncio +async def test_background_image_generation_health_check_probes_upstream_when_opted_in( + httpx_transport: None, respx_mock: respx.MockRouter +) -> None: + """`model_info.health_check_paid_media: true` lets the deployment's operator accept the cost.""" + upstream: Final = respx_mock.post("https://api.openai.com/v1/images/generations").respond( + 200, + json={"created": 1700000000, "data": [{"url": "https://images.test/result.png"}]}, + ) + + healthy, unhealthy, _ = await hc_module.perform_health_check( + [ + { + "litellm_params": {"model": "openai/dall-e-3", "api_key": "fake-key"}, + "model_info": {"id": "img", "mode": "image_generation", "health_check_paid_media": True}, + } + ], + max_concurrency=1, + ) + + assert len(healthy) == 1 + assert unhealthy == [] + assert upstream.called + + def test_health_check_params_merge_into_probe_params(): """health_check_params reach the probe request for the deployment that declares them.""" media_source = {"s3Location": {"uri": "s3://my-bucket/clip.mp4"}} @@ -1384,7 +1435,12 @@ async def test_background_audio_speech_health_check_uses_model_info_voice( "api_key": "fake-key", "api_base": "https://speech.example/v1", }, - "model_info": {"id": "speech", "mode": "audio_speech", "health_check_voice": "nova"}, + "model_info": { + "id": "speech", + "mode": "audio_speech", + "health_check_voice": "nova", + "health_check_paid_media": True, + }, } ], max_concurrency=1, @@ -1420,7 +1476,7 @@ async def test_background_health_check_observes_the_concurrency_limit_and_queue( "api_key": "fake-key", "api_base": "https://health.example/v1", }, - "model_info": {"id": f"audio-{index}", "mode": "audio_speech"}, + "model_info": {"id": f"audio-{index}", "mode": "audio_speech", "health_check_paid_media": True}, } for index in range(10) ] @@ -1465,7 +1521,12 @@ async def test_background_health_check_timeout_marks_a_blocked_provider_unhealth "api_key": "fake-key", "api_base": "https://health.example/v1", }, - "model_info": {"id": "blocked", "mode": "audio_speech", "health_check_timeout": 2}, + "model_info": { + "id": "blocked", + "mode": "audio_speech", + "health_check_timeout": 2, + "health_check_paid_media": True, + }, } ] @@ -1505,7 +1566,12 @@ async def test_background_health_check_timeout_does_not_cancel_a_sibling( "api_key": "fake-key", "api_base": "https://slow.example/v1", }, - "model_info": {"id": "slow", "mode": "audio_speech", "health_check_timeout": 1}, + "model_info": { + "id": "slow", + "mode": "audio_speech", + "health_check_timeout": 1, + "health_check_paid_media": True, + }, }, { "litellm_params": { @@ -1513,7 +1579,12 @@ async def test_background_health_check_timeout_does_not_cancel_a_sibling( "api_key": "fake-key", "api_base": "https://fast.example/v1", }, - "model_info": {"id": "fast", "mode": "audio_speech", "health_check_timeout": 2}, + "model_info": { + "id": "fast", + "mode": "audio_speech", + "health_check_timeout": 2, + "health_check_paid_media": True, + }, }, ] From a3b1604afc02b16bcc3cfdedd49ae4cda86e9209 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:14:21 -0700 Subject: [PATCH 4/7] fix(proxy): never run a billed media probe without an explicit opt-in Background checks, GET /health and targeted /health calls all skip a deployment whose probe would generate billed media, unless it sets model_info.disable_background_health_check: false. A targeted call answers 200 with zero counts and a warning, and the dashboard shows it as skipped. Clip-priced models such as Lyria count as billed media. An invalid health_check_max_tokens no longer stops the background loop. --- .../health_check_helpers.py | 10 ++- .../litellm_core_utils/health_check_utils.py | 1 + litellm/proxy/_types.py | 11 ++- litellm/proxy/health_check.py | 71 +++++++++++++------ .../shared_health_check_manager.py | 2 +- .../health_endpoints/_health_endpoints.py | 27 ++++--- litellm/proxy/proxy_server.py | 15 ++-- .../health_endpoints/test_health_endpoints.py | 38 ++++++---- .../proxy_server/test_background_health.py | 70 +++++++++++++++++- .../proxy/test_health_check_max_tokens.py | 23 ++++-- .../HealthCheckComponent.test.tsx | 29 ++++++++ .../model_dashboard/HealthCheckComponent.tsx | 35 +++++++++ .../HealthChecksTableColumns.tsx | 3 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 15 ++-- 14 files changed, 277 insertions(+), 73 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 940730eeb9f..d0931c8cdc6 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -45,11 +45,17 @@ def native_health_check_mode( return mantle_health_check_mode(model) -def _cost_map_mode(model: str) -> str | None: +def cost_map_entry(model: str) -> Mapping[str, object]: import litellm + from litellm.litellm_core_utils.health_check_utils import STR_KEY_MAPPING + + return STR_KEY_MAPPING.validate_python(litellm.model_cost.get(model, {})) + + +def _cost_map_mode(model: str) -> str | None: from litellm.litellm_core_utils.health_check_utils import OPTIONAL_STR - return OPTIONAL_STR.validate_python(litellm.model_cost.get(model, {}).get("mode")) + return OPTIONAL_STR.validate_python(cost_map_entry(model).get("mode")) def default_health_check_mode(requested_model: str, model: str, custom_llm_provider: str) -> str: diff --git a/litellm/litellm_core_utils/health_check_utils.py b/litellm/litellm_core_utils/health_check_utils.py index 74b6f6dea5d..840c4ed4010 100644 --- a/litellm/litellm_core_utils/health_check_utils.py +++ b/litellm/litellm_core_utils/health_check_utils.py @@ -13,6 +13,7 @@ from litellm.types.extraction import ExtractionCallParams DECISIONS_CALL_PARAMS: Final[TypeAdapter[DecisionsCallParams]] = TypeAdapter(DecisionsCallParams) EXTRACTION_CALL_PARAMS: Final[TypeAdapter[ExtractionCallParams]] = TypeAdapter(ExtractionCallParams) OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) +STR_KEY_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) def _filter_model_params(model_params: dict) -> dict: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7f821cd0413..776bf0a6412 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2694,9 +2694,8 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): health_check_skip_disabled_background_models: bool = Field( False, description=( - "When true, deployments the background health loop skips (model_info.disable_background_health_check, " - "which defaults to true for image_generation, image_edit, and video_generation deployments) " - "are skipped for on-demand GET /health as well." + "When true, deployments with model_info.disable_background_health_check " + "are skipped for on-demand GET /health as well as the background health loop." ), ) background_health_check_model_groups: tuple[str, ...] | None = Field( @@ -2707,9 +2706,9 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): "whose model_name is listed, and enable_health_check_routing filters unhealthy " "deployments only within the listed groups; every other group, including newly " "added deployments, is skipped and keeps its configured routing strategy. " - "When unset, every deployment participates except those that set " - "model_info.disable_background_health_check, which defaults to true for " - "image_generation, image_edit, and video_generation deployments." + "When unset, all deployments participate (opt out per deployment via " + "model_info.disable_background_health_check). Set or unset, deployments whose check would " + "generate billed media take part only with model_info.disable_background_health_check: false." ), ) model_list_healthy_only: bool | None = Field( diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 0ddb4e3753e..0a23745cd27 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -9,7 +9,7 @@ import time from collections.abc import Mapping, Sequence from collections.abc import Set as AbstractSet from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal, TypeVar +from typing import TYPE_CHECKING, Final, TypeVar from pydantic import TypeAdapter, ValidationError @@ -26,7 +26,11 @@ from litellm.constants import ( DEFAULT_HEALTH_CHECK_PROMPT, HEALTH_CHECK_TIMEOUT_SECONDS, ) -from litellm.litellm_core_utils.health_check_helpers import default_health_check_mode, native_health_check_mode +from litellm.litellm_core_utils.health_check_helpers import ( + cost_map_entry, + default_health_check_mode, + native_health_check_mode, +) from litellm.router_strategy.auto_router.auto_router_model_naming import ( StrategyRouterDependency, classify_strategy_router_model, @@ -308,14 +312,18 @@ async def run_with_timeout(task, timeout): _BILLED_MEDIA_GENERATION_MODES: Final = frozenset({"image_generation", "image_edit", "video_generation"}) -DEPLOYMENT_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +BILLED_MEDIA_NOT_CHECKED_WARNING: Final = ( + "Not checked, because this health check would generate billed media. " + "Set model_info.disable_background_health_check: false to check this model." +) +_DEPLOYMENT_ADAPTER: Final = TypeAdapter(Mapping[str, object]) DEPLOYMENTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) MODEL_IDS_ADAPTER: Final = TypeAdapter(frozenset[object]) _STR: Final = TypeAdapter(str) def _str_key_mapping(value: object) -> Mapping[str, object]: - return DEPLOYMENT_ADAPTER.validate_python(value) if isinstance(value, Mapping) else MappingProxyType({}) + return _DEPLOYMENT_ADAPTER.validate_python(value) if isinstance(value, Mapping) else MappingProxyType({}) def _fallback_probe_mode(probe_params: Mapping[str, object]) -> str | None: @@ -338,33 +346,48 @@ def _fallback_probe_mode(probe_params: Mapping[str, object]) -> str | None: ) -def _probe_mode(model_info: Mapping[str, object], litellm_params: Mapping[str, object]) -> str | None: - resolved: Final = resolve_health_check_mode(model_info, litellm_params) - if resolved is not None or not isinstance(litellm_params.get("model"), str): - return resolved - return _fallback_probe_mode( - _PROBE_PARAMS_ADAPTER.validate_python( +def _priced_per_generated_clip(model: object) -> bool: + entry: Final = cost_map_entry(model) if isinstance(model, str) else MappingProxyType({}) + return bool(entry.get("output_cost_per_image")) and not entry.get("output_cost_per_token") + + +def _probe_generates_billed_media(model_info: Mapping[str, object], litellm_params: Mapping[str, object]) -> bool: + mode: Final = resolve_health_check_mode(model_info, litellm_params) + if mode in _BILLED_MEDIA_GENERATION_MODES: + return True + if not isinstance(litellm_params.get("model"), str): + return False + try: + probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python( update_litellm_params_for_health_check(dict(model_info), dict(litellm_params)) ) + except (TypeError, ValueError): + return False + return (mode is None and _fallback_probe_mode(probe_params) in _BILLED_MEDIA_GENERATION_MODES) or ( + _priced_per_generated_clip(probe_params.get("model")) ) -def background_health_check_skip(deployment: Mapping[str, object]) -> Literal["opted_out", "billed_media"] | None: +def bills_media_on_probe(deployment: Mapping[str, object]) -> bool: model_info: Final = _str_key_mapping(deployment.get("model_info")) explicit: Final = model_info.get("disable_background_health_check") - if explicit is not None: - return "opted_out" if explicit else None - litellm_params: Final = _str_key_mapping(deployment.get("litellm_params")) - return "billed_media" if _probe_mode(model_info, litellm_params) in _BILLED_MEDIA_GENERATION_MODES else None + if explicit is not None and not explicit: + return False + return _probe_generates_billed_media(model_info, _str_key_mapping(deployment.get("litellm_params"))) def skips_background_health_check(deployment: Mapping[str, object]) -> bool: - return background_health_check_skip(deployment) is not None + disabled: Final = _str_key_mapping(deployment.get("model_info")).get("disable_background_health_check") + return bool(disabled) or bills_media_on_probe(deployment) -def all_skipped_as_billed_media(deployments: Sequence[Mapping[str, object]], ids: AbstractSet[object]) -> bool: +def background_check_deployments(model_list: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + return tuple(deployment for deployment in model_list if not skips_background_health_check(deployment)) + + +def all_bill_media_on_probe(deployments: Sequence[Mapping[str, object]], ids: AbstractSet[object]) -> bool: targets: Final = tuple(d for d in deployments if _deployment_id(d) in ids) - return bool(targets) and all(background_health_check_skip(d) == "billed_media" for d in targets) + return bool(targets) and all(bills_media_on_probe(d) for d in targets) def _health_check_eligible( @@ -376,7 +399,8 @@ def _health_check_eligible( dependencies alike, so an opted-out deployment cannot re-enter through a router that depends on it. """ - return tuple(x for x in model_list if not (skip_disabled and skips_background_health_check(x))) + skipped: Final = skips_background_health_check if skip_disabled else bills_media_on_probe + return tuple(x for x in model_list if not skipped(x)) def _deployment_model(deployment: Mapping[str, object]) -> str | None: @@ -660,7 +684,7 @@ async def _run_model_health_check(model: dict): return await run_with_timeout( litellm.ahealth_check( probe_params, - mode=mode or _fallback_probe_mode(_PROBE_PARAMS_ADAPTER.validate_python(probe_params)), + mode=mode, prompt=DEFAULT_HEALTH_CHECK_PROMPT, input=["test from litellm"], ), @@ -1024,6 +1048,9 @@ async def perform_health_check( another team's deployment carries, else the deployments whose ``litellm_params.model`` is that string. + Deployments whose check would generate billed media are omitted unless they set + ``model_info.disable_background_health_check: false``. + When ``health_check_skip_disabled_background_models`` is True (via ``general_settings.health_check_skip_disabled_background_models``), deployments the background health loop skips are omitted from this run (including targeted @@ -1052,7 +1079,9 @@ async def perform_health_check( cycle_start_time: Final = time.monotonic() requested_model_count: Final = len(model_list) skip_disabled: Final = health_check_skip_disabled_background_models - narrowed: Final = _health_check_eligible(_narrow_to_target(model_list, model, model_id, team_id), skip_disabled) + narrowed: Final = await asyncio.to_thread( + _health_check_eligible, _narrow_to_target(model_list, model, model_id, team_id), skip_disabled + ) if not narrowed: if instrumentation_enabled: logger.debug( diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py index 81ad195bd0b..d689ab2a537 100644 --- a/litellm/proxy/health_check_utils/shared_health_check_manager.py +++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py @@ -225,7 +225,7 @@ class SharedHealthCheckManager: model_list: List of models to check details: Whether to include detailed information max_concurrency: Optional limit on concurrent health check requests - health_check_skip_disabled_background_models: Remove models the background loop skips + health_check_skip_disabled_background_models: Remove models with disable_background_health_check: true Returns: Tuple of (healthy_endpoints, unhealthy_endpoints) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 1be78fd9f0f..e49eb370165 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -56,11 +56,12 @@ from litellm.proxy.db.health_check_latest import ( from litellm.proxy.db.proxy_worker_heartbeat import count_live_proxy_workers from litellm.proxy.health_check import ( # noqa: F401 # legacy module exports ADMIN_ONLY_HEALTH_DISPLAY_PARAMS, + BILLED_MEDIA_NOT_CHECKED_WARNING, DEPLOYMENTS_ADAPTER, MODEL_IDS_ADAPTER, _clean_endpoint_data, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _update_litellm_params_for_health_check, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - all_skipped_as_billed_media, + all_bill_media_on_probe, clean_endpoint_data, deployments_targeted_by_name, health_check_filter_kwargs_from_general_settings, @@ -1241,11 +1242,12 @@ async def health_endpoint( ``` else, the health checks will be run on models when /health is called. - The background loop skips deployments that set ``model_info.disable_background_health_check: true``, - and image_generation, image_edit, and video_generation deployments unless they set it to ``false``, - since those probes generate billed media. A ``model`` or ``model_id`` that targets only such media - deployments runs the live check, since the background results never hold them. To skip the same - deployments on ``GET /health``, set ``general_settings.health_check_skip_disabled_background_models: true``. + Deployments whose check would generate billed media (image_generation, image_edit, video_generation, + and models priced per generated clip) are not checked here or by the background loop unless they set + ``model_info.disable_background_health_check: false``. A ``model`` or ``model_id`` that targets only + such deployments answers 200 with zero counts and a ``warnings`` entry. To also skip deployments that + set ``disable_background_health_check: true`` on ``GET /health``, set + ``general_settings.health_check_skip_disabled_background_models: true``. """ import time @@ -1327,10 +1329,17 @@ async def health_endpoint( else f"key not allowed to health-check model {model}" }, ) - targets_only_skipped_media: Final = targeted_ids is not None and all_skipped_as_billed_media( + if targeted_ids is not None and all_bill_media_on_probe( DEPLOYMENTS_ADAPTER.validate_python(_llm_model_list), MODEL_IDS_ADAPTER.validate_python(targeted_ids) - ) - if use_background_health_checks and not targets_only_skipped_media: + ): + return { + "healthy_endpoints": [], + "unhealthy_endpoints": [], + "healthy_count": 0, + "unhealthy_count": 0, + "warnings": [BILLED_MEDIA_NOT_CHECKED_WARNING], + } + if use_background_health_checks: # The cached background result covers every model. When the # caller targets a specific model/model_id we have to narrow the # cache to that deployment before _post_process evaluates diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 05403a34189..798c89eb7a7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -648,12 +648,12 @@ from litellm.proxy.guardrails.init_guardrails import ( initialize_guardrails, ) from litellm.proxy.health_check import ( - DEPLOYMENT_ADAPTER, + DEPLOYMENTS_ADAPTER, + background_check_deployments, filter_deployments_to_model_groups, health_check_filter_kwargs_from_general_settings, parse_background_health_check_model_groups, perform_health_check, - skips_background_health_check, ) from litellm.proxy.health_endpoints._health_endpoints import router as health_router from litellm.proxy.health_endpoints.deployment_warning_endpoints import router as deployment_warning_router @@ -4823,13 +4823,14 @@ async def _run_background_health_check(): scoped_model_groups = ( llm_router.settings().background_health_check_model_groups if llm_router is not None else None ) - _llm_model_list = list(filter_deployments_to_model_groups(_llm_model_list, scoped_model_groups)) - _llm_model_list = [ - m for m in _llm_model_list if not skips_background_health_check(DEPLOYMENT_ADAPTER.validate_python(m)) - ] + group_deployments = DEPLOYMENTS_ADAPTER.validate_python( + filter_deployments_to_model_groups(_llm_model_list, scoped_model_groups) + ) + _llm_model_list = list(await asyncio.to_thread(background_check_deployments, group_deployments)) if scoped_model_groups is not None and not _llm_model_list: verbose_proxy_logger.warning( - "background_health_check_model_groups matched no deployments; groups=%s", + "background_health_check_model_groups %s; groups=%s", + "matched only deployments the background loop skips" if group_deployments else "matched no deployments", sorted(scoped_model_groups), ) model_count_enabled = len(_llm_model_list) diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 74068bbc3e2..fd603bd4b43 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -2636,10 +2636,12 @@ async def test_health_endpoint_omits_model_id_warning_when_no_deployment_matches @pytest.mark.asyncio -async def test_targeted_health_runs_live_for_media_the_background_loop_skips_by_default(): +@pytest.mark.parametrize("use_background_health_checks", [True, False]) +async def test_targeted_health_answers_without_probing_billed_media(use_background_health_checks): from fastapi import Response from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.health_check import BILLED_MEDIA_NOT_CHECKED_WARNING from litellm.proxy.health_endpoints._health_endpoints import health_endpoint model_list = [ @@ -2653,12 +2655,20 @@ async def test_targeted_health_runs_live_for_media_the_background_loop_skips_by_ "litellm_params": {"model": "gemini/gemini-3-pro-image"}, "model_info": {"id": "image-off", "mode": "image_generation", "disable_background_health_check": True}, }, + { + "model_name": "image-opted-in", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": "image-on", "mode": "image_generation", "disable_background_health_check": False}, + }, {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat-1"}}, ] cached = { - "healthy_endpoints": [{"model": "openai/gpt-4o-mini", "model_id": "chat-1"}], + "healthy_endpoints": [ + {"model": "openai/gpt-4o-mini", "model_id": "chat-1"}, + {"model": "gemini/gemini-3-pro-image", "model_id": "image-on"}, + ], "unhealthy_endpoints": [], - "healthy_count": 1, + "healthy_count": 2, "unhealthy_count": 0, } live_model_ids: list[str | None] = [] @@ -2666,19 +2676,19 @@ async def test_targeted_health_runs_live_for_media_the_background_loop_skips_by_ async def fake_perform(**kwargs): live_model_ids.append(kwargs["model_id"]) return { - "healthy_endpoints": [{"model": "gemini/gemini-3-pro-image", "model_id": kwargs["model_id"]}], + "healthy_endpoints": [{"model": "probed", "model_id": kwargs["model_id"]}], "unhealthy_endpoints": [], "healthy_count": 1, "unhealthy_count": 0, } admin = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token") - responses = {model_id: Response() for model_id in ("image-1", "image-off", "chat-1")} + responses = {model_id: Response() for model_id in ("image-1", "image-off", "image-on", "chat-1")} with ( _proxy_health_globals( model_list, litellm.Router(model_list=copy.deepcopy(model_list)), - use_background_health_checks=True, + use_background_health_checks=use_background_health_checks, health_check_results=cached, ), patch( # test-quality-ok: whether the live probe ran is the assertion; no injection seam @@ -2693,12 +2703,16 @@ async def test_targeted_health_runs_live_for_media_the_background_loop_skips_by_ for model_id, response in responses.items() } - assert live_model_ids == ["image-1"] - assert results["image-1"]["healthy_count"] == 1 - assert responses["image-1"].status_code == 200 - assert results["chat-1"]["healthy_count"] == 1 - assert results["image-off"]["healthy_count"] == 0 - assert responses["image-off"].status_code == 503 + assert live_model_ids == ([] if use_background_health_checks else ["image-on", "chat-1"]) + for skipped in ("image-1", "image-off"): + assert responses[skipped].status_code == 200 + assert results[skipped]["healthy_endpoints"] == results[skipped]["unhealthy_endpoints"] == [] + assert results[skipped]["healthy_count"] == results[skipped]["unhealthy_count"] == 0 + assert results[skipped]["warnings"] == [BILLED_MEDIA_NOT_CHECKED_WARNING] + for checked in ("image-on", "chat-1"): + assert responses[checked].status_code == 200 + assert results[checked]["healthy_count"] == 1 + assert "warnings" not in results[checked] @pytest.mark.asyncio diff --git a/tests/unit/proxy/proxy_server/test_background_health.py b/tests/unit/proxy/proxy_server/test_background_health.py index 4c487ddd567..3e461a09f1f 100644 --- a/tests/unit/proxy/proxy_server/test_background_health.py +++ b/tests/unit/proxy/proxy_server/test_background_health.py @@ -774,6 +774,69 @@ async def test_run_background_health_check_probes_only_listed_model_groups(monke assert probed["ids"] == ["listed-1", "listed-2"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "listed_group, expected_reason", + [ + ("image-group", "matched only deployments the background loop skips"), + ("no-such-group", "matched no deployments"), + ], +) +async def test_run_background_health_check_names_why_listed_model_groups_probe_nothing( + monkeypatch, listed_group, expected_reason +): + monkeypatch.setattr(proxy_server, "health_check_interval", 60) + monkeypatch.setattr(proxy_server, "health_check_concurrency", 1) + monkeypatch.setattr(proxy_server, "health_check_details", True) + monkeypatch.setattr(proxy_server, "use_shared_health_check", False) + monkeypatch.setattr(proxy_server, "redis_usage_cache", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "background_health_check_loop_active", False) + monkeypatch.setattr( + proxy_server, + "llm_router", + SimpleNamespace( + settings=lambda: RouterSettings(background_health_check_model_groups=frozenset({listed_group})) + ), + ) + monkeypatch.setattr( + proxy_server, + "llm_model_list", + [ + { + "model_name": "image-group", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": "image", "mode": "image_generation"}, + }, + ], + ) + monkeypatch.setattr( + proxy_server, + "health_check_results", + {"healthy_endpoints": [], "unhealthy_endpoints": []}, + ) + monkeypatch.setattr( + proxy_server, + "_run_direct_health_check_with_instrumentation", + AsyncMock(return_value=([], [], {})), + ) + monkeypatch.setattr(proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "health_check_filter_kwargs_from_general_settings", lambda _gs: {}) + warn = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn) + + async def _stop_sleep(_seconds): + raise asyncio.CancelledError() + + monkeypatch.setattr(proxy_server.asyncio, "sleep", _stop_sleep) + + with pytest.raises(asyncio.CancelledError): + await _run_background_health_check() + + assert [call.args[1:] for call in warn.call_args_list] == [(expected_reason, [listed_group])] + + @pytest.mark.asyncio async def test_run_background_health_check_skips_billed_media_generation_unless_opted_in(monkeypatch): monkeypatch.setattr(proxy_server, "health_check_interval", 60) @@ -815,6 +878,11 @@ async def test_run_background_health_check_skips_billed_media_generation_unless_ "model_info": {"id": "embedding", "mode": "embedding"}, }, {"model_name": "fine-tune", "litellm_params": {"model": "openai/my-fine-tune"}, "model_info": {"id": "fine-tune"}}, + { + "model_name": "bad-max-tokens", + "litellm_params": {"model": "openai/my-fine-tune"}, + "model_info": {"id": "bad-max-tokens", "health_check_max_tokens": "abc"}, + }, { "model_name": "azure-ai-image", "litellm_params": {"model": "azure_ai/gpt-image-1", "api_base": "https://example.services.ai.azure.com"}, @@ -864,4 +932,4 @@ async def test_run_background_health_check_skips_billed_media_generation_unless_ with pytest.raises(asyncio.CancelledError): await _run_background_health_check() - assert probed["ids"] == ["chat", "speech", "embedding", "fine-tune", "image-opted-in"] + assert probed["ids"] == ["chat", "speech", "embedding", "fine-tune", "bad-max-tokens", "image-opted-in"] diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index 1b2e3597e5b..428b0aa87b3 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -1227,9 +1227,10 @@ def test_a_dependency_that_opted_out_of_health_checks_is_never_probed(): ] -def test_health_skip_flag_skips_media_by_the_mode_the_probe_would_run(): - """Modes as model_prices_and_context_window.json listed them on 2026-10-29: gemini/gemini-3-pro-image and - gpt-image-1 are image_generation, openai/text-embedding-3-small is embedding.""" +def test_health_checks_skip_billed_media_by_the_mode_the_probe_would_run(): + """Modes and prices as model_prices_and_context_window.json listed them on 2026-10-30: gemini/gemini-3-pro-image + and gpt-image-1 are image_generation, openai/text-embedding-3-small is embedding, gemini/lyria-3-clip-preview and + vertex_ai/lyria-002 bill per clip with no token price, and gemini/gemini-3.1-pro-preview also has a token price.""" def deployment(model_id: str, model: str, **model_info: object) -> dict[str, object]: return { @@ -1238,22 +1239,32 @@ def test_health_skip_flag_skips_media_by_the_mode_the_probe_would_run(): "model_info": {"id": model_id, **model_info}, } - skipped = ( + billed = ( deployment("cost-map-image", "gemini/gemini-3-pro-image"), deployment("azure-ai-image", "azure_ai/gpt-image-1"), deployment("wildcard-probing-an-image", "openai/*", health_check_model="openai/gpt-image-1"), + deployment("clip-priced-chat", "gemini/lyria-3-clip-preview"), + deployment("clip-priced-speech", "vertex_ai/lyria-002", mode="audio_speech"), + deployment("image-with-bad-max-tokens", "openai/x", mode="image_generation", health_check_max_tokens="abc"), + deployment("opted-out-image", "gemini/gemini-3-pro-image", disable_background_health_check=True), ) probed = ( deployment("chat", "openai/gpt-4o-mini"), deployment("embedding", "openai/text-embedding-3-small"), + deployment("token-priced-chat-with-image-price", "gemini/gemini-3.1-pro-preview"), deployment("wildcard", "openai/*"), deployment("unknown", "openai/my-fine-tune"), deployment("no-provider-prefix", "my-fine-tune"), + deployment("bad-max-tokens", "openai/my-fine-tune", health_check_max_tokens="abc"), deployment("image-opted-in", "gemini/gemini-3-pro-image", disable_background_health_check=False), ) + opted_out_chat = deployment("opted-out-chat", "openai/gpt-4o-mini", disable_background_health_check=True) - assert hc_module._health_check_eligible([*skipped, *probed], skip_disabled=True) == probed - assert hc_module._health_check_eligible([*skipped, *probed], skip_disabled=False) == (*skipped, *probed) + assert hc_module._health_check_eligible([*billed, *probed, opted_out_chat], skip_disabled=True) == probed + assert hc_module._health_check_eligible([*billed, *probed, opted_out_chat], skip_disabled=False) == ( + *probed, + opted_out_chat, + ) def test_narrowing_by_an_id_that_matches_nothing_keeps_the_whole_list(): diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx index 1b0194b6478..657b9049ed5 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx @@ -105,6 +105,35 @@ describe("HealthCheckComponent", () => { expect(mockIndividualModelHealthCheckCall).not.toHaveBeenCalledWith("token", "gpt-4"); }); + it("shows a model the proxy declined to check as skipped, never healthy, even with an old saved result", async () => { + const warning = "Not checked, because this health check would generate billed media."; + const declinedResponse = { + healthy_count: 0, + unhealthy_count: 0, + healthy_endpoints: [], + unhealthy_endpoints: [], + warnings: [warning], + }; + mockIndividualModelHealthCheckCall.mockResolvedValue(declinedResponse); + mockLatestHealthChecksCall.mockResolvedValue({ + latest_health_checks: { "image-1": { status: "healthy", checked_at: "2026-10-01T00:00:00Z" } }, + }); + await renderHealthCheck({ modelData: { data: [makeModel("image-1", "image")] }, allModelsOnProxy: ["image-1"] }); + + for (const runButton of ["run-health-check-btn", "run-health-checks"]) { + await act(async () => { + screen.getByTestId(runButton).click(); + }); + await act(async () => { + await new Promise((r) => setTimeout(r, 50)); + }); + + expect(screen.getByText("skipped")).toBeInTheDocument(); + expect(screen.queryByText("healthy")).not.toBeInTheDocument(); + expect(screen.getByText(warning)).toBeInTheDocument(); + } + }); + it("should page through results with the shared pagination footer", async () => { const onPageIndexChange = vi.fn(); await renderHealthCheck({ diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx index 041c8718133..e4d5191078c 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx @@ -17,6 +17,22 @@ import { individualModelHealthCheckCall, latestHealthChecksCall } from "../netwo import { HealthChecksTable } from "./HealthChecksTable"; import type { HealthCheckData, HealthStatus } from "./HealthChecksTableColumns"; +const isSkippedResponse = (response: { + healthy_count: number; + unhealthy_count: number; + warnings?: string[]; +}): response is { healthy_count: 0; unhealthy_count: 0; warnings: string[] } => + response.healthy_count === 0 && response.unhealthy_count === 0 && Boolean(response.warnings?.length); + +const skippedStatus = (warning: string, lastCheck: string, previous?: HealthStatus): HealthStatus => ({ + status: "skipped", + lastCheck, + lastSuccess: previous?.lastSuccess || "None", + loading: false, + error: warning, + fullError: warning, +}); + interface LatestHealthCheck { status?: string; checked_at?: string | null; @@ -261,6 +277,14 @@ const HealthCheckComponent: React.FC = ({ const response = await individualModelHealthCheckCall(accessToken, modelId); const currentTime = new Date().toLocaleString(); + if (isSkippedResponse(response)) { + setModelHealthStatuses((prev) => ({ + ...prev, + [modelId]: skippedStatus(response.warnings[0], currentTime, prev[modelId]), + })); + return; + } + if (response.unhealthy_count > 0 && response.unhealthy_endpoints && response.unhealthy_endpoints.length > 0) { const rawError = response.unhealthy_endpoints[0]?.error || "Health check failed"; const errorMessage = extractMeaningfulError(rawError); @@ -357,6 +381,14 @@ const HealthCheckComponent: React.FC = ({ const response = await individualModelHealthCheckCall(accessToken, modelId); const currentTime = new Date().toLocaleString(); + if (isSkippedResponse(response)) { + setModelHealthStatuses((prev) => ({ + ...prev, + [modelId]: skippedStatus(response.warnings[0], currentTime, prev[modelId]), + })); + return; + } + if (response.unhealthy_count > 0 && response.unhealthy_endpoints && response.unhealthy_endpoints.length > 0) { const rawError = response.unhealthy_endpoints[0]?.error || "Health check failed"; const errorMessage = extractMeaningfulError(rawError); @@ -416,6 +448,9 @@ const HealthCheckComponent: React.FC = ({ setModelHealthStatuses((prev) => { const currentStatus = prev[modelId]; + if (currentStatus?.status === "skipped") { + return prev; + } return { ...prev, [modelId]: { diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx index 8a9ddee9b7e..4ee0eae5d89 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx @@ -40,10 +40,11 @@ const HEALTH_STATUS_TONES: Record = { unhealthy: "error", checking: "info", none: "neutral", + skipped: "neutral", }; // healthy > checking > unknown > unhealthy, matching the legacy health table ordering. -const HEALTH_STATUS_ORDER: Record = { healthy: 0, checking: 1, unknown: 2, unhealthy: 3 }; +const HEALTH_STATUS_ORDER: Record = { healthy: 0, checking: 1, unknown: 2, skipped: 2, unhealthy: 3 }; const NEVER_CHECKED = "Never checked"; const CHECK_IN_PROGRESS = "Check in progress..."; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 03d53487712..6594481323a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7555,11 +7555,12 @@ export interface paths { * ``` * else, the health checks will be run on models when /health is called. * - * The background loop skips deployments that set ``model_info.disable_background_health_check: true``, - * and image_generation, image_edit, and video_generation deployments unless they set it to ``false``, - * since those probes generate billed media. A ``model`` or ``model_id`` that targets only such media - * deployments runs the live check, since the background results never hold them. To skip the same - * deployments on ``GET /health``, set ``general_settings.health_check_skip_disabled_background_models: true``. + * Deployments whose check would generate billed media (image_generation, image_edit, video_generation, + * and models priced per generated clip) are not checked here or by the background loop unless they set + * ``model_info.disable_background_health_check: false``. A ``model`` or ``model_id`` that targets only + * such deployments answers 200 with zero counts and a ``warnings`` entry. To also skip deployments that + * set ``disable_background_health_check: true`` on ``GET /health``, set + * ``general_settings.health_check_skip_disabled_background_models: true``. */ get: operations["health_endpoint_health_get"]; put?: never; @@ -29909,7 +29910,7 @@ export interface components { apply_user_budget_to_team_keys?: boolean | null; /** * Background Health Check Model Groups - * @description Opt-in allowlist of model group names for background health checks and health-check routing. When set, the background loop probes only deployments whose model_name is listed, and enable_health_check_routing filters unhealthy deployments only within the listed groups; every other group, including newly added deployments, is skipped and keeps its configured routing strategy. When unset, every deployment participates except those that set model_info.disable_background_health_check, which defaults to true for image_generation, image_edit, and video_generation deployments. + * @description Opt-in allowlist of model group names for background health checks and health-check routing. When set, the background loop probes only deployments whose model_name is listed, and enable_health_check_routing filters unhealthy deployments only within the listed groups; every other group, including newly added deployments, is skipped and keeps its configured routing strategy. When unset, all deployments participate (opt out per deployment via model_info.disable_background_health_check). Set or unset, deployments whose check would generate billed media take part only with model_info.disable_background_health_check: false. */ background_health_check_model_groups?: string[] | null; /** @@ -30108,7 +30109,7 @@ export interface components { health_check_interval: number; /** * Health Check Skip Disabled Background Models - * @description When true, deployments the background health loop skips (model_info.disable_background_health_check, which defaults to true for image_generation, image_edit, and video_generation deployments) are skipped for on-demand GET /health as well. + * @description When true, deployments with model_info.disable_background_health_check are skipped for on-demand GET /health as well as the background health loop. * @default false */ health_check_skip_disabled_background_models: boolean; From fd1d1c9011cc44893d41650d1360ff6dfa2b99a8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:25:21 -0700 Subject: [PATCH 5/7] fix(health): keep bad values from stopping checks and restore the skip flag 503 A bad model_info value raised inside the skip rule and ended the background loop, so the rule now fails open and the deployment fails its own check as it does on the base. A media deployment with disable_background_health_check true keeps its 503 when health_check_skip_disabled_background_models is on. The per-clip price lookup also reads the provider set apart from the model, and the dashboard shows the skip message as neutral text --- litellm/proxy/health_check.py | 57 +++++++++++++------ .../health_endpoints/_health_endpoints.py | 4 +- .../health_endpoints/test_health_endpoints.py | 40 +++++++++++++ .../proxy/test_health_check_max_tokens.py | 38 +++++++++++++ .../HealthCheckComponent.test.tsx | 1 + .../HealthChecksTableColumns.tsx | 9 +++ 6 files changed, 131 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index d258fd0e59c..f6e3556794a 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -197,7 +197,7 @@ def _get_process_rss_mb() -> float | None: if sys.platform == "darwin": return float(ru_maxrss) / (1024 * 1024) return float(ru_maxrss) / 1024 - except Exception: + except (ImportError, OSError): return None @@ -346,47 +346,70 @@ def _fallback_probe_mode(probe_params: Mapping[str, object]) -> str | None: ) -def _priced_per_generated_clip(model: object) -> bool: - entry: Final = cost_map_entry(model) if isinstance(model, str) else MappingProxyType({}) +def _bills_per_generated_clip(entry: Mapping[str, object]) -> bool: return bool(entry.get("output_cost_per_image")) and not entry.get("output_cost_per_token") +def _priced_per_generated_clip(probe_params: Mapping[str, object]) -> bool: + model: Final = probe_params.get("model") + provider: Final = probe_params.get("custom_llm_provider") + if not isinstance(model, str): + return False + names: Final = (model, f"{provider}/{model}") if isinstance(provider, str) else (model,) + return any(_bills_per_generated_clip(cost_map_entry(name)) for name in names) + + def _probe_generates_billed_media(model_info: Mapping[str, object], litellm_params: Mapping[str, object]) -> bool: mode: Final = resolve_health_check_mode(model_info, litellm_params) if mode in _BILLED_MEDIA_GENERATION_MODES: return True if not isinstance(litellm_params.get("model"), str): return False - try: - probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python( - update_litellm_params_for_health_check(dict(model_info), dict(litellm_params)) - ) - except (TypeError, ValueError): - return False + probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python( + update_litellm_params_for_health_check(dict(model_info), dict(litellm_params)) + ) return (mode is None and _fallback_probe_mode(probe_params) in _BILLED_MEDIA_GENERATION_MODES) or ( - _priced_per_generated_clip(probe_params.get("model")) + _priced_per_generated_clip(probe_params) ) +def _disable_background_flag(deployment: Mapping[str, object]) -> object: + model_info: Final = deployment.get("model_info") + return model_info.get("disable_background_health_check") if isinstance(model_info, Mapping) else None + + def bills_media_on_probe(deployment: Mapping[str, object]) -> bool: - model_info: Final = _str_key_mapping(deployment.get("model_info")) - explicit: Final = model_info.get("disable_background_health_check") + explicit: Final = _disable_background_flag(deployment) if explicit is not None and not explicit: return False - return _probe_generates_billed_media(model_info, _str_key_mapping(deployment.get("litellm_params"))) + try: + return _probe_generates_billed_media( + _str_key_mapping(deployment.get("model_info")), _str_key_mapping(deployment.get("litellm_params")) + ) + except Exception: + logger.debug("Could not work out the health check for %s", _deployment_id(deployment)) + return False def skips_background_health_check(deployment: Mapping[str, object]) -> bool: - disabled: Final = _str_key_mapping(deployment.get("model_info")).get("disable_background_health_check") - return bool(disabled) or bills_media_on_probe(deployment) + return bool(_disable_background_flag(deployment)) or bills_media_on_probe(deployment) def background_check_deployments(model_list: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: return tuple(deployment for deployment in model_list if not skips_background_health_check(deployment)) -def all_bill_media_on_probe(deployments: Sequence[Mapping[str, object]], ids: AbstractSet[object]) -> bool: - targets: Final = tuple(d for d in deployments if _deployment_id(d) in ids) +def all_bill_media_on_probe( + deployments: Sequence[Mapping[str, object]], + ids: AbstractSet[object], + health_check_skip_disabled_background_models: bool, +) -> bool: + targets: Final = tuple( + d + for d in deployments + if _deployment_id(d) in ids + and not (health_check_skip_disabled_background_models and _disable_background_flag(d)) + ) return bool(targets) and all(bills_media_on_probe(d) for d in targets) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 7d2a7a390a9..f21cc19b5f6 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -1694,7 +1694,9 @@ async def health_endpoint( }, ) if targeted_ids is not None and all_bill_media_on_probe( - DEPLOYMENTS_ADAPTER.validate_python(_llm_model_list), MODEL_IDS_ADAPTER.validate_python(targeted_ids) + DEPLOYMENTS_ADAPTER.validate_python(_llm_model_list), + MODEL_IDS_ADAPTER.validate_python(targeted_ids), + **_hc_filter, ): return { "healthy_endpoints": [], diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 6d1fa84f5bd..0454987ea65 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -2721,6 +2721,46 @@ async def test_targeted_health_answers_without_probing_billed_media(use_backgrou assert "warnings" not in results[checked] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_info", "expected_status"), + [ + ({"id": "image-1", "mode": "image_generation", "disable_background_health_check": True}, 503), + ({"id": "image-1", "mode": "image_generation"}, 200), + ], +) +async def test_skip_disabled_setting_keeps_the_503_for_an_opted_out_media_model(model_info, expected_status): + from fastapi import Response + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + model_list = [ + {"model_name": "image", "litellm_params": {"model": "gemini/gemini-3-pro-image"}, "model_info": model_info} + ] + response = Response() + with ( + _proxy_health_globals( + model_list, + litellm.Router(model_list=copy.deepcopy(model_list)), + use_background_health_checks=True, + health_check_results={"healthy_endpoints": [], "unhealthy_endpoints": []}, + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.general_settings", + {"health_check_skip_disabled_background_models": True}, + ), + ): + await health_endpoint( + response=response, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token"), + model=None, + model_id="image-1", + ) + + assert response.status_code == expected_status + + @pytest.mark.asyncio async def test_health_endpoint_filters_background_cache_by_user_access(): """ diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index c9830bcd65c..11c54786ad9 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -1271,6 +1271,44 @@ def test_health_checks_skip_billed_media_by_the_mode_the_probe_would_run(): ) +@pytest.mark.parametrize( + ("litellm_params", "billed"), + [ + ({"model": "lyria-002", "custom_llm_provider": "vertex_ai"}, True), + ({"model": "lyria-3-clip-preview", "custom_llm_provider": "gemini"}, True), + ({"model": "gemini-3.1-pro-preview", "custom_llm_provider": "gemini"}, False), + ({"model": "my-fine-tune", "custom_llm_provider": "vertex_ai"}, False), + ], +) +def test_per_clip_price_is_found_when_the_provider_is_set_apart_from_the_model( + litellm_params: dict[str, str], billed: bool +): + """Prices as model_prices_and_context_window.json listed them on 2026-10-31: the Lyria entries bill per clip with + no token price under their provider prefix alone, and gemini/gemini-3.1-pro-preview also has a token price.""" + deployment = {"litellm_params": litellm_params, "model_info": {"id": "clip", "mode": "audio_speech"}} + + assert hc_module.bills_media_on_probe(deployment) is billed + + +@pytest.mark.parametrize( + "model_info", + [ + {"health_check_max_tokens": float("inf")}, + {"health_check_max_tokens": "many"}, + {"health_check_model": 123}, + {1: "not a string key"}, + ], +) +def test_one_bad_model_info_value_cannot_stop_the_health_checks(model_info: dict[object, object]): + bad = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "bad", **model_info}} + control = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "control"}} + + assert hc_module.background_check_deployments([bad, control]) == (bad, control) + assert hc_module._health_check_eligible([bad, control], skip_disabled=True) == (bad, control) + assert hc_module._health_check_eligible([bad, control], skip_disabled=False) == (bad, control) + assert hc_module.all_bill_media_on_probe([bad, control], {"bad"}, False) is False + + def test_narrowing_by_an_id_that_matches_nothing_keeps_the_whole_list(): """Pinned because the disabled-dependency fix moved this filter into its own helper.""" deployments = [{"model_name": "a", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a-1"}}] diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx index 657b9049ed5..d6fde177be2 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx @@ -131,6 +131,7 @@ describe("HealthCheckComponent", () => { expect(screen.getByText("skipped")).toBeInTheDocument(); expect(screen.queryByText("healthy")).not.toBeInTheDocument(); expect(screen.getByText(warning)).toBeInTheDocument(); + expect(screen.getByText(warning)).not.toHaveClass("text-destructive"); } }); diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx index 4ee0eae5d89..e343992328e 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx @@ -340,6 +340,15 @@ export const getHealthChecksTableColumns = ({ } const cleanedError = healthStatus.error; + + if (healthStatus.status === "skipped") { + return ( + + {cleanedError} + + ); + } + const fullError = healthStatus.fullError || healthStatus.error; const displayName = getDisplayModelName(model) || model.model_name; From 3d7c0deacc45d50e449c6fba2d90ad05cba6d1a4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:48:40 -0700 Subject: [PATCH 6/7] test(health): cover the failed memory reading and drop the unreachable model guard --- litellm/proxy/health_check.py | 4 +--- tests/unit/proxy/test_health_check_max_tokens.py | 5 +++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index f6e3556794a..dffce4f03c0 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -351,10 +351,8 @@ def _bills_per_generated_clip(entry: Mapping[str, object]) -> bool: def _priced_per_generated_clip(probe_params: Mapping[str, object]) -> bool: - model: Final = probe_params.get("model") + model: Final = _STR.validate_python(probe_params.get("model")) provider: Final = probe_params.get("custom_llm_provider") - if not isinstance(model, str): - return False names: Final = (model, f"{provider}/{model}") if isinstance(provider, str) else (model,) return any(_bills_per_generated_clip(cost_map_entry(name)) for name in names) diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index 11c54786ad9..fbea7b3117c 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -1309,6 +1309,11 @@ def test_one_bad_model_info_value_cannot_stop_the_health_checks(model_info: dict assert hc_module.all_bill_media_on_probe([bad, control], {"bad"}, False) is False +def test_a_failed_memory_reading_does_not_break_health_check_logging(): + with patch("resource.getrusage", side_effect=OSError("no rusage")): # test-quality-ok: stdlib call, no seam + assert hc_module._rss_mb_for_log() == "unknown" + + def test_narrowing_by_an_id_that_matches_nothing_keeps_the_whole_list(): """Pinned because the disabled-dependency fix moved this filter into its own helper.""" deployments = [{"model_name": "a", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a-1"}}] From 473896fc39ec1be9c72352a097162c6c76c53c41 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 03:16:08 -0700 Subject: [PATCH 7/7] fix(health): say which media models were left out and stop repeating their old verdict GET /health now warns with the ids of billed media models it left out, even when other models were checked. /health/latest marks those models as skipped in place of an old saved result, and the dashboard shows that reason on load. The background loop logs the skipped ids once per change. A targeted request that also covers a model with the flag set no longer counts as all skipped when that model is excluded by the opt-out rule. --- litellm/proxy/health_check.py | 34 ++++++++--- .../shared_health_check_manager.py | 3 +- .../health_endpoints/_health_endpoints.py | 42 ++++++++++--- litellm/proxy/proxy_server.py | 10 +++ .../health_endpoints/test_health_endpoints.py | 55 +++++++++++++++++ .../proxy_server/test_background_health.py | 58 +++++++++++++++++- .../unit/proxy/test_health_check_functions.py | 61 +++++++++++++++++++ .../HealthCheckComponent.test.tsx | 17 ++++++ .../model_dashboard/HealthCheckComponent.tsx | 5 ++ 9 files changed, 265 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index dffce4f03c0..5c4716cd7a9 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -313,8 +313,9 @@ async def run_with_timeout(task, timeout): _BILLED_MEDIA_GENERATION_MODES: Final = frozenset({"image_generation", "image_edit", "video_generation"}) BILLED_MEDIA_NOT_CHECKED_WARNING: Final = ( - "Not checked, because this health check would generate billed media. " - "Set model_info.disable_background_health_check: false to check this model." + "Not checked, because this health check would generate media the provider may bill. " + "Set model_info.disable_background_health_check: false to check this model on every interval, " + "or use Test Connection for one check." ) _DEPLOYMENT_ADAPTER: Final = TypeAdapter(Mapping[str, object]) DEPLOYMENTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) @@ -397,20 +398,37 @@ def background_check_deployments(model_list: Sequence[Mapping[str, object]]) -> return tuple(deployment for deployment in model_list if not skips_background_health_check(deployment)) +def _probe_targets( + deployments: Sequence[Mapping[str, object]], + ids: AbstractSet[object] | None, + health_check_skip_disabled_background_models: bool, +) -> tuple[Mapping[str, object], ...]: + return tuple( + d + for d in deployments + if (ids is None or _deployment_id(d) in ids) + and not (health_check_skip_disabled_background_models and _disable_background_flag(d)) + ) + + def all_bill_media_on_probe( deployments: Sequence[Mapping[str, object]], ids: AbstractSet[object], health_check_skip_disabled_background_models: bool, ) -> bool: - targets: Final = tuple( - d - for d in deployments - if _deployment_id(d) in ids - and not (health_check_skip_disabled_background_models and _disable_background_flag(d)) - ) + targets: Final = _probe_targets(deployments, ids, health_check_skip_disabled_background_models) return bool(targets) and all(bills_media_on_probe(d) for d in targets) +def billed_media_model_ids( + deployments: Sequence[Mapping[str, object]], + ids: AbstractSet[object] | None = None, + health_check_skip_disabled_background_models: bool = False, +) -> tuple[str, ...]: + targets: Final = _probe_targets(deployments, ids, health_check_skip_disabled_background_models) + return tuple(sorted(filter(None, (_deployment_id(d) for d in targets if bills_media_on_probe(d))))) + + def _health_check_eligible( model_list: Sequence[Mapping[str, object]], skip_disabled: bool ) -> tuple[Mapping[str, object], ...]: diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py index cf090d82990..862be43be30 100644 --- a/litellm/proxy/health_check_utils/shared_health_check_manager.py +++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py @@ -225,7 +225,8 @@ class SharedHealthCheckManager: model_list: List of models to check details: Whether to include detailed information max_concurrency: Optional limit on concurrent health check requests - health_check_skip_disabled_background_models: Remove models with disable_background_health_check: true + health_check_skip_disabled_background_models: Also remove models with disable_background_health_check: true. + A model whose check would generate billed media is removed either way, unless it set that flag to false Returns: Tuple of (healthy_endpoints, unhealthy_endpoints) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index f21cc19b5f6..6fc6d224618 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -86,6 +86,7 @@ from litellm.proxy.health_check import ( # noqa: F401 # legacy module exports _clean_endpoint_data, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _update_litellm_params_for_health_check, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export all_bill_media_on_probe, + billed_media_model_ids, clean_endpoint_data, deployments_targeted_by_name, health_check_filter_kwargs_from_general_settings, @@ -1693,11 +1694,9 @@ async def health_endpoint( else f"key not allowed to health-check model {model}" }, ) - if targeted_ids is not None and all_bill_media_on_probe( - DEPLOYMENTS_ADAPTER.validate_python(_llm_model_list), - MODEL_IDS_ADAPTER.validate_python(targeted_ids), - **_hc_filter, - ): + deployments: Final = DEPLOYMENTS_ADAPTER.validate_python(_llm_model_list) + probe_ids: Final = None if targeted_ids is None else MODEL_IDS_ADAPTER.validate_python(targeted_ids) + if probe_ids is not None and all_bill_media_on_probe(deployments, probe_ids, **_hc_filter): return { "healthy_endpoints": [], "unhealthy_endpoints": [], @@ -1705,6 +1704,12 @@ async def health_endpoint( "unhealthy_count": 0, "warnings": [BILLED_MEDIA_NOT_CHECKED_WARNING], } + not_checked_ids: Final = await asyncio.to_thread(billed_media_model_ids, deployments, probe_ids, **_hc_filter) + media_warning: Final = ( + {"warnings": [f"{BILLED_MEDIA_NOT_CHECKED_WARNING} Model ids: {', '.join(not_checked_ids)}"]} + if not_checked_ids + else {} + ) if use_background_health_checks: # The cached background result covers every model. When the # caller targets a specific model/model_id we have to narrow the @@ -1743,13 +1748,18 @@ async def health_endpoint( "and were excluded from this response. Ask a proxy admin " "to populate model_info.id for these models." ] - return _post_process(filtered) + return _post_process({**filtered, **media_warning}) if targeted_ids is not None: # Admin caller targeting a specific model: filter the cache # so the response (and the targeted-503 check) reflects only # that deployment, not the global aggregate. - return _post_process(_filter_health_check_results_by_model_ids(health_check_results, targeted_ids)) - return _post_process(health_check_results) + return _post_process( + { + **_filter_health_check_results_by_model_ids(health_check_results, targeted_ids), + **media_warning, + } + ) + return _post_process({**health_check_results, **media_warning}) else: router_result: Final = await _perform_health_check_and_save( model_list=_llm_model_list, @@ -1766,7 +1776,7 @@ async def health_endpoint( team_id=user_api_key_dict.team_id, **_hc_filter, ) - return _post_process(router_result) + return _post_process({**router_result, **media_warning}) except Exception as e: verbose_proxy_logger.error("litellm.proxy.proxy_server.py::health_endpoint(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) @@ -1813,6 +1823,13 @@ async def health_check_history_endpoint( ) +def _latest_check_row(check: LatestHealthCheckRow, skipped_ids: frozenset[str]) -> Mapping[str, object]: + row: Final = convert_health_check_to_dict(check) + if check.model_id not in skipped_ids: + return row + return {**row, "status": "skipped", "error_message": BILLED_MEDIA_NOT_CHECKED_WARNING} + + @router.get("/health/latest", tags=["health"], dependencies=[Depends(user_api_key_auth)]) async def latest_health_checks_endpoint( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -1822,14 +1839,19 @@ async def latest_health_checks_endpoint( Returns the most recent health check result for each model. """ + from litellm.proxy.proxy_server import llm_model_list + prisma_client: Final = _check_prisma_client() try: latest_checks: Final = await prisma_client.get_all_latest_health_checks() + skipped_ids: Final = frozenset( + await asyncio.to_thread(billed_media_model_ids, DEPLOYMENTS_ADAPTER.validate_python(llm_model_list or ())) + ) # Convert to dict format for JSON response using helper function checks_data: Final = { - (check.model_id if check.model_id else check.model_name): convert_health_check_to_dict(check) + (check.model_id if check.model_id else check.model_name): _latest_check_row(check, skipped_ids) for check in latest_checks } diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 64b893341c7..dcfb7f33e5a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -654,6 +654,7 @@ from litellm.proxy.guardrails.init_guardrails import ( from litellm.proxy.health_check import ( DEPLOYMENTS_ADAPTER, background_check_deployments, + billed_media_model_ids, filter_deployments_to_model_groups, health_check_filter_kwargs_from_general_settings, parse_background_health_check_model_groups, @@ -4800,6 +4801,7 @@ async def _run_background_health_check(): ) verbose_proxy_logger.info("Initialized shared health check manager") + logged_skipped_media_ids: tuple[str, ...] = () # rebind-ok: the last set logged, compared each cycle while True: background_health_check_cycle_seq += 1 cycle_id = f"bg-{background_health_check_cycle_seq}" @@ -4816,6 +4818,14 @@ async def _run_background_health_check(): filter_deployments_to_model_groups(_llm_model_list, scoped_model_groups) ) _llm_model_list = list(await asyncio.to_thread(background_check_deployments, group_deployments)) + skipped_media_ids = await asyncio.to_thread(billed_media_model_ids, group_deployments) + if skipped_media_ids and skipped_media_ids != logged_skipped_media_ids: + verbose_proxy_logger.warning( + "background health checks skip deployments whose check would generate billed media; " + "set model_info.disable_background_health_check: false to check them; model_ids=%s", + skipped_media_ids, + ) + logged_skipped_media_ids = skipped_media_ids if scoped_model_groups is not None and not _llm_model_list: verbose_proxy_logger.warning( "background_health_check_model_groups %s; groups=%s", diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 0454987ea65..303692f680e 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -2721,6 +2721,61 @@ async def test_targeted_health_answers_without_probing_billed_media(use_backgrou assert "warnings" not in results[checked] +@pytest.mark.asyncio +@pytest.mark.parametrize("use_background_health_checks", [True, False]) +async def test_health_names_the_media_models_it_left_out(use_background_health_checks): + from fastapi import Response + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.health_check import BILLED_MEDIA_NOT_CHECKED_WARNING + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + def image(model_name: str, model_id: str) -> dict: + return { + "model_name": model_name, + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": model_id, "mode": "image_generation"}, + } + + chat = {"model_name": "mixed", "litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat-1"}} + model_list = [image("mixed", "image-in-mixed"), image("image", "image-2"), chat] + checked = { + "healthy_endpoints": [{"model": "openai/gpt-4o-mini", "model_id": "chat-1"}], + "unhealthy_endpoints": [], + "healthy_count": 1, + "unhealthy_count": 0, + } + + async def fake_perform(**kwargs): + return dict(checked) + + admin = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token") + with ( + _proxy_health_globals( + model_list, + litellm.Router(model_list=copy.deepcopy(model_list)), + use_background_health_checks=use_background_health_checks, + health_check_results=checked, + ), + patch( # test-quality-ok: the live probe is the outside call; the answer built around it is under test + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", + side_effect=fake_perform, + ), + ): + everything = await health_endpoint(response=Response(), user_api_key_dict=admin, model=None, model_id=None) + mixed_response = Response() + mixed = await health_endpoint(response=mixed_response, user_api_key_dict=admin, model="mixed", model_id=None) + one_chat = await health_endpoint(response=Response(), user_api_key_dict=admin, model=None, model_id="chat-1") + + assert everything["healthy_count"] == 1 + assert everything["warnings"] == [f"{BILLED_MEDIA_NOT_CHECKED_WARNING} Model ids: image-2, image-in-mixed"] + assert mixed_response.status_code == 200 + assert mixed["healthy_count"] == 1 + assert mixed["warnings"] == [f"{BILLED_MEDIA_NOT_CHECKED_WARNING} Model ids: image-in-mixed"] + assert "warnings" not in one_chat + assert "warnings" not in checked + + @pytest.mark.asyncio @pytest.mark.parametrize( ("model_info", "expected_status"), diff --git a/tests/unit/proxy/proxy_server/test_background_health.py b/tests/unit/proxy/proxy_server/test_background_health.py index ee26155f62c..b099eadfa1e 100644 --- a/tests/unit/proxy/proxy_server/test_background_health.py +++ b/tests/unit/proxy/proxy_server/test_background_health.py @@ -802,7 +802,63 @@ async def test_run_background_health_check_names_why_listed_model_groups_probe_n with pytest.raises(asyncio.CancelledError): await _run_background_health_check() - assert [call.args[1:] for call in warn.call_args_list] == [(expected_reason, [listed_group])] + assert [ + call.args[1:] for call in warn.call_args_list if call.args[0].startswith("background_health_check_model_groups") + ] == [(expected_reason, [listed_group])] + + +@pytest.mark.asyncio +async def test_run_background_health_check_names_skipped_media_models_once_per_change(monkeypatch): + monkeypatch.setattr(proxy_server, "health_check_interval", 60) + monkeypatch.setattr(proxy_server, "health_check_concurrency", 1) + monkeypatch.setattr(proxy_server, "health_check_details", True) + monkeypatch.setattr(proxy_server, "use_shared_health_check", False) + monkeypatch.setattr(proxy_server, "redis_usage_cache", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "background_health_check_loop_active", False) + monkeypatch.setattr( + proxy_server, + "llm_router", + SimpleNamespace( + settings=lambda: RouterSettings(background_health_check_model_groups=frozenset({"image", "chat"})) + ), + ) + + def image(model_id): + return { + "model_name": "image", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": model_id, "mode": "image_generation"}, + } + + chat = {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat-1"}} + monkeypatch.setattr(proxy_server, "llm_model_list", [image("image-1"), chat]) + monkeypatch.setattr(proxy_server, "health_check_results", {"healthy_endpoints": [], "unhealthy_endpoints": []}) + monkeypatch.setattr( + proxy_server, + "_run_direct_health_check_with_instrumentation", + AsyncMock(return_value=([], [], {})), + ) + monkeypatch.setattr(proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "health_check_filter_kwargs_from_general_settings", lambda _gs: {}) + warn = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn) + sleeps = iter((None, [image("image-1"), image("image-2"), chat], None)) + + async def _next_cycle(_seconds): + step = next(sleeps, asyncio.CancelledError()) + if isinstance(step, BaseException): + raise step + if step is not None: + monkeypatch.setattr(proxy_server, "llm_model_list", step) + + monkeypatch.setattr(proxy_server.asyncio, "sleep", _next_cycle) + + with pytest.raises(asyncio.CancelledError): + await _run_background_health_check() + + assert [call.args[1:] for call in warn.call_args_list] == [(("image-1",),), (("image-1", "image-2"),)] @pytest.mark.asyncio diff --git a/tests/unit/proxy/test_health_check_functions.py b/tests/unit/proxy/test_health_check_functions.py index 79eec44958f..970ce1ca50b 100644 --- a/tests/unit/proxy/test_health_check_functions.py +++ b/tests/unit/proxy/test_health_check_functions.py @@ -8,6 +8,7 @@ import pytest from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.db.health_check_latest import LatestHealthCheckRow +from litellm.proxy.health_check import BILLED_MEDIA_NOT_CHECKED_WARNING from litellm.proxy.health_endpoints._health_endpoints import ( _aggregate_health_check_results, _build_model_param_to_info_mapping, @@ -666,6 +667,66 @@ async def test_save_background_health_checks_compares_raw_checked_at_against_utc assert [call.kwargs["model_id"] for call in mock_prisma.save_health_check_result.await_args_list] == ["stale-id"] +def _saved_row(model_id: str, status: str, error_message: str | None) -> LatestHealthCheckRow: + return LatestHealthCheckRow( + health_check_id=f"hc-{model_id}", + model_name=model_id, + model_id=model_id, + status=status, + healthy_count=1 if status == "healthy" else 0, + unhealthy_count=0 if status == "healthy" else 1, + error_message=error_message, + response_time_ms=12.5, + details=None, + checked_by="pod-1", + checked_at=datetime(2026, 8, 25), + created_at=datetime(2026, 8, 25, tzinfo=timezone.utc), + updated_at=datetime(2026, 8, 25, tzinfo=timezone.utc), + ) + + +@pytest.mark.asyncio +async def test_latest_health_checks_stops_repeating_an_old_verdict_for_a_skipped_media_model(monkeypatch): + prisma = MagicMock() + prisma.get_all_latest_health_checks = AsyncMock( + return_value=( + _saved_row("image-was-healthy", "healthy", None), + _saved_row("image-was-dead", "unhealthy", "401 from the provider"), + _saved_row("image-opted-in", "unhealthy", "401 from the provider"), + _saved_row("chat", "unhealthy", "401 from the provider"), + ) + ) + image = {"litellm_params": {"model": "gemini/gemini-3-pro-image"}} + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma) + monkeypatch.setattr( + "litellm.proxy.proxy_server.llm_model_list", + [ + {**image, "model_info": {"id": "image-was-healthy", "mode": "image_generation"}}, + {**image, "model_info": {"id": "image-was-dead", "mode": "image_generation"}}, + { + **image, + "model_info": { + "id": "image-opted-in", + "mode": "image_generation", + "disable_background_health_check": False, + }, + }, + {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat"}}, + ], + ) + + response = await latest_health_checks_endpoint(user_api_key_dict=UserAPIKeyAuth()) + + rows = response["latest_health_checks"] + assert {model_id: (row["status"], row["error_message"]) for model_id, row in rows.items()} == { + "image-was-healthy": ("skipped", BILLED_MEDIA_NOT_CHECKED_WARNING), + "image-was-dead": ("skipped", BILLED_MEDIA_NOT_CHECKED_WARNING), + "image-opted-in": ("unhealthy", "401 from the provider"), + "chat": ("unhealthy", "401 from the provider"), + } + assert rows["image-was-healthy"]["checked_at"] == "2026-08-25T00:00:00+00:00" + + @pytest.mark.asyncio async def test_latest_health_checks_endpoint_serialises_raw_rows(monkeypatch): row = LatestHealthCheckRow( diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx index d6fde177be2..e96db13b633 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx @@ -135,6 +135,23 @@ describe("HealthCheckComponent", () => { } }); + it("shows the skip reason on load when the proxy marks a saved result as skipped", async () => { + const warning = "Not checked, because this health check would generate media the provider may bill."; + mockLatestHealthChecksCall.mockResolvedValue({ + latest_health_checks: { + "image-1": { status: "skipped", checked_at: "2026-10-01T00:00:00Z", error_message: warning }, + }, + }); + await renderHealthCheck({ modelData: { data: [makeModel("image-1", "image")] }, allModelsOnProxy: ["image-1"] }); + await act(async () => { + await new Promise((r) => setTimeout(r, 50)); + }); + + expect(screen.getByText("skipped")).toBeInTheDocument(); + expect(screen.getByText(warning)).toBeInTheDocument(); + expect(screen.getByText(warning)).not.toHaveClass("text-destructive"); + }); + it("should page through results with the shared pagination footer", async () => { const onPageIndexChange = vi.fn(); await renderHealthCheck({ diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx index e4d5191078c..572997f6e39 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx @@ -239,6 +239,11 @@ const HealthCheckComponent: React.FC = ({ const fullError = checkData.error_message || undefined; + if (checkData.status === "skipped" && fullError) { + healthStatusMap[modelId] = skippedStatus(fullError, toCheckedAtLabel(checkData.checked_at, "None")); + return; + } + healthStatusMap[modelId] = { status: checkData.status || "unknown", lastCheck: toCheckedAtLabel(checkData.checked_at, "None"),