diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 2c7d00552b4..fd12b8e06e7 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -50,11 +50,17 @@ def native_health_check_mode( return mantle_health_check_mode(model) -def _cost_map_mode(model: str) -> str | None: +def cost_map_entry(model: str) -> Mapping[str, object]: import litellm + from litellm.litellm_core_utils.health_check_utils import STR_KEY_MAPPING + + return STR_KEY_MAPPING.validate_python(litellm.model_cost.get(model, {})) + + +def _cost_map_mode(model: str) -> str | None: from litellm.litellm_core_utils.health_check_utils import OPTIONAL_STR - return OPTIONAL_STR.validate_python(litellm.model_cost.get(model, {}).get("mode")) + return OPTIONAL_STR.validate_python(cost_map_entry(model).get("mode")) def default_health_check_mode(requested_model: str, model: str, custom_llm_provider: str) -> str: diff --git a/litellm/litellm_core_utils/health_check_utils.py b/litellm/litellm_core_utils/health_check_utils.py index 74b6f6dea5d..840c4ed4010 100644 --- a/litellm/litellm_core_utils/health_check_utils.py +++ b/litellm/litellm_core_utils/health_check_utils.py @@ -13,6 +13,7 @@ from litellm.types.extraction import ExtractionCallParams DECISIONS_CALL_PARAMS: Final[TypeAdapter[DecisionsCallParams]] = TypeAdapter(DecisionsCallParams) EXTRACTION_CALL_PARAMS: Final[TypeAdapter[ExtractionCallParams]] = TypeAdapter(ExtractionCallParams) OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) +STR_KEY_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) def _filter_model_params(model_params: dict) -> dict: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e53b5d4ad52..32ed2f3b67f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2754,7 +2754,8 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): "deployments only within the listed groups; every other group, including newly " "added deployments, is skipped and keeps its configured routing strategy. " "When unset, all deployments participate (opt out per deployment via " - "model_info.disable_background_health_check)." + "model_info.disable_background_health_check). Set or unset, deployments whose check would " + "generate billed media take part only with model_info.disable_background_health_check: false." ), ) model_list_healthy_only: bool | None = Field( diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index de2ed4fd4ba..5c4716cd7a9 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -26,7 +26,11 @@ from litellm.constants import ( DEFAULT_HEALTH_CHECK_PROMPT, HEALTH_CHECK_TIMEOUT_SECONDS, ) -from litellm.litellm_core_utils.health_check_helpers import native_health_check_mode +from litellm.litellm_core_utils.health_check_helpers import ( + cost_map_entry, + default_health_check_mode, + native_health_check_mode, +) from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin, is_model_capability_field from litellm.router_strategy.auto_router.auto_router_model_naming import ( StrategyRouterDependency, @@ -193,7 +197,7 @@ def _get_process_rss_mb() -> float | None: if sys.platform == "darwin": return float(ru_maxrss) / (1024 * 1024) return float(ru_maxrss) / 1024 - except Exception: + except (ImportError, OSError): return None @@ -230,9 +234,9 @@ def health_check_filter_kwargs_from_general_settings( """ Build kwargs for ``perform_health_check`` from ``general_settings``. - When ``health_check_skip_disabled_background_models`` is true, deployments with - ``model_info.disable_background_health_check`` are omitted from health runs - (including on-demand ``GET /health``), matching the background loop behavior. + When ``health_check_skip_disabled_background_models`` is true, deployments the + background loop skips are omitted from health runs (including on-demand + ``GET /health``). """ g: Final = general_settings or {} return { @@ -307,9 +311,122 @@ async def run_with_timeout(task, timeout): return {"error": "Timeout exceeded", "exception": timeout_exception} -def _skips_health_checks(deployment: Mapping[str, object]) -> bool: - info: Final = deployment.get("model_info") - return bool(info.get("disable_background_health_check", False)) if isinstance(info, Mapping) else False +_BILLED_MEDIA_GENERATION_MODES: Final = frozenset({"image_generation", "image_edit", "video_generation"}) +BILLED_MEDIA_NOT_CHECKED_WARNING: Final = ( + "Not checked, because this health check would generate media the provider may bill. " + "Set model_info.disable_background_health_check: false to check this model on every interval, " + "or use Test Connection for one check." +) +_DEPLOYMENT_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +DEPLOYMENTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +MODEL_IDS_ADAPTER: Final = TypeAdapter(frozenset[object]) +_STR: Final = TypeAdapter(str) + + +def _str_key_mapping(value: object) -> Mapping[str, object]: + return _DEPLOYMENT_ADAPTER.validate_python(value) if isinstance(value, Mapping) else MappingProxyType({}) + + +def _fallback_probe_mode(probe_params: Mapping[str, object]) -> str | None: + """The mode `litellm.ahealth_check` picks for these probe params when the caller passes none.""" + provider: Final = probe_params.get("custom_llm_provider") + api_base: Final = probe_params.get("api_base") + api_key: Final = probe_params.get("api_key") + try: + model: Final = _STR.validate_python(probe_params.get("model")) + resolved_model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model, + custom_llm_provider=provider if isinstance(provider, str) else None, + api_base=api_base if isinstance(api_base, str) else None, + api_key=api_key if isinstance(api_key, str) else None, + ) + except (ValidationError, litellm.BadRequestError): + return None + return default_health_check_mode( + requested_model=model, model=resolved_model, custom_llm_provider=custom_llm_provider + ) + + +def _bills_per_generated_clip(entry: Mapping[str, object]) -> bool: + return bool(entry.get("output_cost_per_image")) and not entry.get("output_cost_per_token") + + +def _priced_per_generated_clip(probe_params: Mapping[str, object]) -> bool: + model: Final = _STR.validate_python(probe_params.get("model")) + provider: Final = probe_params.get("custom_llm_provider") + names: Final = (model, f"{provider}/{model}") if isinstance(provider, str) else (model,) + return any(_bills_per_generated_clip(cost_map_entry(name)) for name in names) + + +def _probe_generates_billed_media(model_info: Mapping[str, object], litellm_params: Mapping[str, object]) -> bool: + mode: Final = resolve_health_check_mode(model_info, litellm_params) + if mode in _BILLED_MEDIA_GENERATION_MODES: + return True + if not isinstance(litellm_params.get("model"), str): + return False + probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python( + update_litellm_params_for_health_check(dict(model_info), dict(litellm_params)) + ) + return (mode is None and _fallback_probe_mode(probe_params) in _BILLED_MEDIA_GENERATION_MODES) or ( + _priced_per_generated_clip(probe_params) + ) + + +def _disable_background_flag(deployment: Mapping[str, object]) -> object: + model_info: Final = deployment.get("model_info") + return model_info.get("disable_background_health_check") if isinstance(model_info, Mapping) else None + + +def bills_media_on_probe(deployment: Mapping[str, object]) -> bool: + explicit: Final = _disable_background_flag(deployment) + if explicit is not None and not explicit: + return False + try: + return _probe_generates_billed_media( + _str_key_mapping(deployment.get("model_info")), _str_key_mapping(deployment.get("litellm_params")) + ) + except Exception: + logger.debug("Could not work out the health check for %s", _deployment_id(deployment)) + return False + + +def skips_background_health_check(deployment: Mapping[str, object]) -> bool: + return bool(_disable_background_flag(deployment)) or bills_media_on_probe(deployment) + + +def background_check_deployments(model_list: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + return tuple(deployment for deployment in model_list if not skips_background_health_check(deployment)) + + +def _probe_targets( + deployments: Sequence[Mapping[str, object]], + ids: AbstractSet[object] | None, + health_check_skip_disabled_background_models: bool, +) -> tuple[Mapping[str, object], ...]: + return tuple( + d + for d in deployments + if (ids is None or _deployment_id(d) in ids) + and not (health_check_skip_disabled_background_models and _disable_background_flag(d)) + ) + + +def all_bill_media_on_probe( + deployments: Sequence[Mapping[str, object]], + ids: AbstractSet[object], + health_check_skip_disabled_background_models: bool, +) -> bool: + targets: Final = _probe_targets(deployments, ids, health_check_skip_disabled_background_models) + return bool(targets) and all(bills_media_on_probe(d) for d in targets) + + +def billed_media_model_ids( + deployments: Sequence[Mapping[str, object]], + ids: AbstractSet[object] | None = None, + health_check_skip_disabled_background_models: bool = False, +) -> tuple[str, ...]: + targets: Final = _probe_targets(deployments, ids, health_check_skip_disabled_background_models) + return tuple(sorted(filter(None, (_deployment_id(d) for d in targets if bills_media_on_probe(d))))) def _health_check_eligible( @@ -317,11 +434,12 @@ def _health_check_eligible( ) -> tuple[Mapping[str, object], ...]: """Deployments this run is allowed to contact. - The one eligibility gate, applied to the requested set and to the pool a router's - dependencies are drawn from alike, so an opted-out deployment cannot re-enter through a - router that depends on it. + The one eligibility gate, applied to the requested set and to each round of a router's + dependencies alike, so an opted-out deployment cannot re-enter through a router that + depends on it. """ - return tuple(x for x in model_list if not (skip_disabled and _skips_health_checks(x))) + skipped: Final = skips_background_health_check if skip_disabled else bills_media_on_probe + return tuple(x for x in model_list if not skipped(x)) def _deployment_model(deployment: Mapping[str, object]) -> str | None: @@ -471,6 +589,7 @@ def _dependency_deployments_to_probe( checked: Sequence[Mapping[str, object]], universe: Sequence[Mapping[str, object]], router: "Router", + skip_disabled: bool = False, ) -> tuple[Mapping[str, object], ...]: """Deployments backing the checked routers' dependencies that are not already checked. @@ -478,7 +597,7 @@ def _dependency_deployments_to_probe( `/health?model_id=` call the dashboard makes per deployment that needs them, since a router's verdict is a statement about models the request never named. Drawn from `universe`, the caller's access-filtered list, so no deployment is probed that the caller - was not already granted. Expansion follows routers through routers, one hop per round, + was not already granted, and only those `_health_check_eligible` passes. Expansion follows routers through routers, one hop per round, because a child router's own models must be probed for the parent to fail; stopping when a round adds nothing is what makes a router cycle terminate. """ @@ -498,9 +617,9 @@ def _dependency_deployments_to_probe( ) if not fresh_ids: break - frontier = _deployments_by_id(universe, fresh_ids) + frontier = _health_check_eligible(_deployments_by_id(universe, fresh_ids), skip_disabled) reached = reached | fresh_ids - return _deployments_by_id(universe, reached - checked_ids) + return _health_check_eligible(_deployments_by_id(universe, reached - checked_ids), skip_disabled) def _strategy_router_verdicts( @@ -972,11 +1091,13 @@ async def perform_health_check( another team's deployment carries, else the deployments whose ``litellm_params.model`` is that string. + Deployments whose check would generate billed media are omitted unless they set + ``model_info.disable_background_health_check: false``. + When ``health_check_skip_disabled_background_models`` is True (via ``general_settings.health_check_skip_disabled_background_models``), deployments - with ``model_info.disable_background_health_check: true`` are omitted from - this run (including targeted ``/health`` queries), consistent with the - background health loop. + the background health loop skips are omitted from this run (including targeted + ``/health`` queries). Returns: (bool): True if the health check passes, False otherwise. @@ -1001,7 +1122,9 @@ async def perform_health_check( cycle_start_time: Final = time.monotonic() requested_model_count: Final = len(model_list) skip_disabled: Final = health_check_skip_disabled_background_models - narrowed: Final = _health_check_eligible(_narrow_to_target(model_list, model, model_id, team_id), skip_disabled) + narrowed: Final = await asyncio.to_thread( + _health_check_eligible, _narrow_to_target(model_list, model, model_id, team_id), skip_disabled + ) if not narrowed: if instrumentation_enabled: logger.debug( @@ -1016,9 +1139,7 @@ async def perform_health_check( deduped_model_count: Final = len(requested) dependency_probes: Final = ( - _dependency_deployments_to_probe(requested, _health_check_eligible(model_list, skip_disabled), router) - if router is not None - else () + _dependency_deployments_to_probe(requested, model_list, router, skip_disabled) if router is not None else () ) checked: Final = requested + list(dependency_probes) diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py index cf090d82990..862be43be30 100644 --- a/litellm/proxy/health_check_utils/shared_health_check_manager.py +++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py @@ -225,7 +225,8 @@ class SharedHealthCheckManager: model_list: List of models to check details: Whether to include detailed information max_concurrency: Optional limit on concurrent health check requests - health_check_skip_disabled_background_models: Remove models with disable_background_health_check: true + health_check_skip_disabled_background_models: Also remove models with disable_background_health_check: true. + A model whose check would generate billed media is removed either way, unless it set that flag to false Returns: Tuple of (healthy_endpoints, unhealthy_endpoints) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 333687b07cd..6fc6d224618 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -80,8 +80,13 @@ from litellm.proxy.db.health_check_latest import ( from litellm.proxy.db.proxy_worker_heartbeat import count_live_proxy_workers from litellm.proxy.health_check import ( # noqa: F401 # legacy module exports ADMIN_ONLY_HEALTH_DISPLAY_PARAMS, + BILLED_MEDIA_NOT_CHECKED_WARNING, + DEPLOYMENTS_ADAPTER, + MODEL_IDS_ADAPTER, _clean_endpoint_data, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _update_litellm_params_for_health_check, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + all_bill_media_on_probe, + billed_media_model_ids, clean_endpoint_data, deployments_targeted_by_name, health_check_filter_kwargs_from_general_settings, @@ -1577,8 +1582,11 @@ async def health_endpoint( ``` else, the health checks will be run on models when /health is called. - To skip deployments that set ``model_info.disable_background_health_check: true`` - on ``GET /health`` as well as in the background loop, set + Deployments whose check would generate billed media (image_generation, image_edit, video_generation, + and models priced per generated clip) are not checked here or by the background loop unless they set + ``model_info.disable_background_health_check: false``. A ``model`` or ``model_id`` that targets only + such deployments answers 200 with zero counts and a ``warnings`` entry. To also skip deployments that + set ``disable_background_health_check: true`` on ``GET /health``, set ``general_settings.health_check_skip_disabled_background_models: true``. """ import time @@ -1686,6 +1694,22 @@ async def health_endpoint( else f"key not allowed to health-check model {model}" }, ) + deployments: Final = DEPLOYMENTS_ADAPTER.validate_python(_llm_model_list) + probe_ids: Final = None if targeted_ids is None else MODEL_IDS_ADAPTER.validate_python(targeted_ids) + if probe_ids is not None and all_bill_media_on_probe(deployments, probe_ids, **_hc_filter): + return { + "healthy_endpoints": [], + "unhealthy_endpoints": [], + "healthy_count": 0, + "unhealthy_count": 0, + "warnings": [BILLED_MEDIA_NOT_CHECKED_WARNING], + } + not_checked_ids: Final = await asyncio.to_thread(billed_media_model_ids, deployments, probe_ids, **_hc_filter) + media_warning: Final = ( + {"warnings": [f"{BILLED_MEDIA_NOT_CHECKED_WARNING} Model ids: {', '.join(not_checked_ids)}"]} + if not_checked_ids + else {} + ) if use_background_health_checks: # The cached background result covers every model. When the # caller targets a specific model/model_id we have to narrow the @@ -1724,13 +1748,18 @@ async def health_endpoint( "and were excluded from this response. Ask a proxy admin " "to populate model_info.id for these models." ] - return _post_process(filtered) + return _post_process({**filtered, **media_warning}) if targeted_ids is not None: # Admin caller targeting a specific model: filter the cache # so the response (and the targeted-503 check) reflects only # that deployment, not the global aggregate. - return _post_process(_filter_health_check_results_by_model_ids(health_check_results, targeted_ids)) - return _post_process(health_check_results) + return _post_process( + { + **_filter_health_check_results_by_model_ids(health_check_results, targeted_ids), + **media_warning, + } + ) + return _post_process({**health_check_results, **media_warning}) else: router_result: Final = await _perform_health_check_and_save( model_list=_llm_model_list, @@ -1747,7 +1776,7 @@ async def health_endpoint( team_id=user_api_key_dict.team_id, **_hc_filter, ) - return _post_process(router_result) + return _post_process({**router_result, **media_warning}) except Exception as e: verbose_proxy_logger.error("litellm.proxy.proxy_server.py::health_endpoint(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) @@ -1794,6 +1823,13 @@ async def health_check_history_endpoint( ) +def _latest_check_row(check: LatestHealthCheckRow, skipped_ids: frozenset[str]) -> Mapping[str, object]: + row: Final = convert_health_check_to_dict(check) + if check.model_id not in skipped_ids: + return row + return {**row, "status": "skipped", "error_message": BILLED_MEDIA_NOT_CHECKED_WARNING} + + @router.get("/health/latest", tags=["health"], dependencies=[Depends(user_api_key_auth)]) async def latest_health_checks_endpoint( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -1803,14 +1839,19 @@ async def latest_health_checks_endpoint( Returns the most recent health check result for each model. """ + from litellm.proxy.proxy_server import llm_model_list + prisma_client: Final = _check_prisma_client() try: latest_checks: Final = await prisma_client.get_all_latest_health_checks() + skipped_ids: Final = frozenset( + await asyncio.to_thread(billed_media_model_ids, DEPLOYMENTS_ADAPTER.validate_python(llm_model_list or ())) + ) # Convert to dict format for JSON response using helper function checks_data: Final = { - (check.model_id if check.model_id else check.model_name): convert_health_check_to_dict(check) + (check.model_id if check.model_id else check.model_name): _latest_check_row(check, skipped_ids) for check in latest_checks } diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ffd2dc305e7..7bb53919820 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -689,6 +689,9 @@ from litellm.proxy.guardrails.init_guardrails import ( initialize_guardrails, ) from litellm.proxy.health_check import ( + DEPLOYMENTS_ADAPTER, + background_check_deployments, + billed_media_model_ids, filter_deployments_to_model_groups, health_check_filter_kwargs_from_general_settings, parse_background_health_check_model_groups, @@ -4936,6 +4939,7 @@ async def _run_background_health_check(): ) verbose_proxy_logger.info("Initialized shared health check manager") + logged_skipped_media_ids: tuple[str, ...] = () # rebind-ok: the last set logged, compared each cycle while True: background_health_check_cycle_seq += 1 cycle_id = f"bg-{background_health_check_cycle_seq}" @@ -4945,17 +4949,25 @@ async def _run_background_health_check(): _llm_model_list = copy.deepcopy(llm_model_list) or [] model_count_total = len(_llm_model_list) - # filter out models that have disabled background health checks - _llm_model_list = [ - m for m in _llm_model_list if not m.get("model_info", {}).get("disable_background_health_check", False) - ] scoped_model_groups = ( llm_router.settings().background_health_check_model_groups if llm_router is not None else None ) - _llm_model_list = list(filter_deployments_to_model_groups(_llm_model_list, scoped_model_groups)) + group_deployments = DEPLOYMENTS_ADAPTER.validate_python( + filter_deployments_to_model_groups(_llm_model_list, scoped_model_groups) + ) + _llm_model_list = list(await asyncio.to_thread(background_check_deployments, group_deployments)) + skipped_media_ids = await asyncio.to_thread(billed_media_model_ids, group_deployments) + if skipped_media_ids and skipped_media_ids != logged_skipped_media_ids: + verbose_proxy_logger.warning( + "background health checks skip deployments whose check would generate billed media; " + "set model_info.disable_background_health_check: false to check them; model_ids=%s", + skipped_media_ids, + ) + logged_skipped_media_ids = skipped_media_ids if scoped_model_groups is not None and not _llm_model_list: verbose_proxy_logger.warning( - "background_health_check_model_groups matched no deployments; groups=%s", + "background_health_check_model_groups %s; groups=%s", + "matched only deployments the background loop skips" if group_deployments else "matched no deployments", sorted(scoped_model_groups), ) model_count_enabled = len(_llm_model_list) diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 2258eb95fb8..303692f680e 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -2641,6 +2641,181 @@ async def test_health_endpoint_omits_model_id_warning_when_no_deployment_matches assert "warnings" not in result +@pytest.mark.asyncio +@pytest.mark.parametrize("use_background_health_checks", [True, False]) +async def test_targeted_health_answers_without_probing_billed_media(use_background_health_checks): + from fastapi import Response + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.health_check import BILLED_MEDIA_NOT_CHECKED_WARNING + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + model_list = [ + { + "model_name": "image", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": "image-1", "mode": "image_generation"}, + }, + { + "model_name": "image-opted-out", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": "image-off", "mode": "image_generation", "disable_background_health_check": True}, + }, + { + "model_name": "image-opted-in", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": "image-on", "mode": "image_generation", "disable_background_health_check": False}, + }, + {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat-1"}}, + ] + cached = { + "healthy_endpoints": [ + {"model": "openai/gpt-4o-mini", "model_id": "chat-1"}, + {"model": "gemini/gemini-3-pro-image", "model_id": "image-on"}, + ], + "unhealthy_endpoints": [], + "healthy_count": 2, + "unhealthy_count": 0, + } + live_model_ids: list[str | None] = [] + + async def fake_perform(**kwargs): + live_model_ids.append(kwargs["model_id"]) + return { + "healthy_endpoints": [{"model": "probed", "model_id": kwargs["model_id"]}], + "unhealthy_endpoints": [], + "healthy_count": 1, + "unhealthy_count": 0, + } + + admin = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token") + responses = {model_id: Response() for model_id in ("image-1", "image-off", "image-on", "chat-1")} + with ( + _proxy_health_globals( + model_list, + litellm.Router(model_list=copy.deepcopy(model_list)), + use_background_health_checks=use_background_health_checks, + health_check_results=cached, + ), + patch( # test-quality-ok: whether the live probe ran is the assertion; no injection seam + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", + side_effect=fake_perform, + ), + ): + results = { + model_id: await health_endpoint( + response=response, user_api_key_dict=admin, model=None, model_id=model_id + ) + for model_id, response in responses.items() + } + + assert live_model_ids == ([] if use_background_health_checks else ["image-on", "chat-1"]) + for skipped in ("image-1", "image-off"): + assert responses[skipped].status_code == 200 + assert results[skipped]["healthy_endpoints"] == results[skipped]["unhealthy_endpoints"] == [] + assert results[skipped]["healthy_count"] == results[skipped]["unhealthy_count"] == 0 + assert results[skipped]["warnings"] == [BILLED_MEDIA_NOT_CHECKED_WARNING] + for checked in ("image-on", "chat-1"): + assert responses[checked].status_code == 200 + assert results[checked]["healthy_count"] == 1 + assert "warnings" not in results[checked] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_background_health_checks", [True, False]) +async def test_health_names_the_media_models_it_left_out(use_background_health_checks): + from fastapi import Response + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.health_check import BILLED_MEDIA_NOT_CHECKED_WARNING + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + def image(model_name: str, model_id: str) -> dict: + return { + "model_name": model_name, + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": model_id, "mode": "image_generation"}, + } + + chat = {"model_name": "mixed", "litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat-1"}} + model_list = [image("mixed", "image-in-mixed"), image("image", "image-2"), chat] + checked = { + "healthy_endpoints": [{"model": "openai/gpt-4o-mini", "model_id": "chat-1"}], + "unhealthy_endpoints": [], + "healthy_count": 1, + "unhealthy_count": 0, + } + + async def fake_perform(**kwargs): + return dict(checked) + + admin = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token") + with ( + _proxy_health_globals( + model_list, + litellm.Router(model_list=copy.deepcopy(model_list)), + use_background_health_checks=use_background_health_checks, + health_check_results=checked, + ), + patch( # test-quality-ok: the live probe is the outside call; the answer built around it is under test + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", + side_effect=fake_perform, + ), + ): + everything = await health_endpoint(response=Response(), user_api_key_dict=admin, model=None, model_id=None) + mixed_response = Response() + mixed = await health_endpoint(response=mixed_response, user_api_key_dict=admin, model="mixed", model_id=None) + one_chat = await health_endpoint(response=Response(), user_api_key_dict=admin, model=None, model_id="chat-1") + + assert everything["healthy_count"] == 1 + assert everything["warnings"] == [f"{BILLED_MEDIA_NOT_CHECKED_WARNING} Model ids: image-2, image-in-mixed"] + assert mixed_response.status_code == 200 + assert mixed["healthy_count"] == 1 + assert mixed["warnings"] == [f"{BILLED_MEDIA_NOT_CHECKED_WARNING} Model ids: image-in-mixed"] + assert "warnings" not in one_chat + assert "warnings" not in checked + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_info", "expected_status"), + [ + ({"id": "image-1", "mode": "image_generation", "disable_background_health_check": True}, 503), + ({"id": "image-1", "mode": "image_generation"}, 200), + ], +) +async def test_skip_disabled_setting_keeps_the_503_for_an_opted_out_media_model(model_info, expected_status): + from fastapi import Response + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + model_list = [ + {"model_name": "image", "litellm_params": {"model": "gemini/gemini-3-pro-image"}, "model_info": model_info} + ] + response = Response() + with ( + _proxy_health_globals( + model_list, + litellm.Router(model_list=copy.deepcopy(model_list)), + use_background_health_checks=True, + health_check_results={"healthy_endpoints": [], "unhealthy_endpoints": []}, + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.general_settings", + {"health_check_skip_disabled_background_models": True}, + ), + ): + await health_endpoint( + response=response, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token"), + model=None, + model_id="image-1", + ) + + assert response.status_code == expected_status + + @pytest.mark.asyncio async def test_health_endpoint_filters_background_cache_by_user_access(): """ diff --git a/tests/unit/proxy/proxy_server/test_background_health.py b/tests/unit/proxy/proxy_server/test_background_health.py index 4c1c3d4ece9..b099eadfa1e 100644 --- a/tests/unit/proxy/proxy_server/test_background_health.py +++ b/tests/unit/proxy/proxy_server/test_background_health.py @@ -740,3 +740,220 @@ async def test_run_background_health_check_probes_only_listed_model_groups(monke await _run_background_health_check() assert probed["ids"] == ["listed-1", "listed-2"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "listed_group, expected_reason", + [ + ("image-group", "matched only deployments the background loop skips"), + ("no-such-group", "matched no deployments"), + ], +) +async def test_run_background_health_check_names_why_listed_model_groups_probe_nothing( + monkeypatch, listed_group, expected_reason +): + monkeypatch.setattr(proxy_server, "health_check_interval", 60) + monkeypatch.setattr(proxy_server, "health_check_concurrency", 1) + monkeypatch.setattr(proxy_server, "health_check_details", True) + monkeypatch.setattr(proxy_server, "use_shared_health_check", False) + monkeypatch.setattr(proxy_server, "redis_usage_cache", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "background_health_check_loop_active", False) + monkeypatch.setattr( + proxy_server, + "llm_router", + SimpleNamespace( + settings=lambda: RouterSettings(background_health_check_model_groups=frozenset({listed_group})) + ), + ) + monkeypatch.setattr( + proxy_server, + "llm_model_list", + [ + { + "model_name": "image-group", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": "image", "mode": "image_generation"}, + }, + ], + ) + monkeypatch.setattr( + proxy_server, + "health_check_results", + {"healthy_endpoints": [], "unhealthy_endpoints": []}, + ) + monkeypatch.setattr( + proxy_server, + "_run_direct_health_check_with_instrumentation", + AsyncMock(return_value=([], [], {})), + ) + monkeypatch.setattr(proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "health_check_filter_kwargs_from_general_settings", lambda _gs: {}) + warn = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn) + + async def _stop_sleep(_seconds): + raise asyncio.CancelledError() + + monkeypatch.setattr(proxy_server.asyncio, "sleep", _stop_sleep) + + with pytest.raises(asyncio.CancelledError): + await _run_background_health_check() + + assert [ + call.args[1:] for call in warn.call_args_list if call.args[0].startswith("background_health_check_model_groups") + ] == [(expected_reason, [listed_group])] + + +@pytest.mark.asyncio +async def test_run_background_health_check_names_skipped_media_models_once_per_change(monkeypatch): + monkeypatch.setattr(proxy_server, "health_check_interval", 60) + monkeypatch.setattr(proxy_server, "health_check_concurrency", 1) + monkeypatch.setattr(proxy_server, "health_check_details", True) + monkeypatch.setattr(proxy_server, "use_shared_health_check", False) + monkeypatch.setattr(proxy_server, "redis_usage_cache", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "background_health_check_loop_active", False) + monkeypatch.setattr( + proxy_server, + "llm_router", + SimpleNamespace( + settings=lambda: RouterSettings(background_health_check_model_groups=frozenset({"image", "chat"})) + ), + ) + + def image(model_id): + return { + "model_name": "image", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": model_id, "mode": "image_generation"}, + } + + chat = {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat-1"}} + monkeypatch.setattr(proxy_server, "llm_model_list", [image("image-1"), chat]) + monkeypatch.setattr(proxy_server, "health_check_results", {"healthy_endpoints": [], "unhealthy_endpoints": []}) + monkeypatch.setattr( + proxy_server, + "_run_direct_health_check_with_instrumentation", + AsyncMock(return_value=([], [], {})), + ) + monkeypatch.setattr(proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "health_check_filter_kwargs_from_general_settings", lambda _gs: {}) + warn = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn) + sleeps = iter((None, [image("image-1"), image("image-2"), chat], None)) + + async def _next_cycle(_seconds): + step = next(sleeps, asyncio.CancelledError()) + if isinstance(step, BaseException): + raise step + if step is not None: + monkeypatch.setattr(proxy_server, "llm_model_list", step) + + monkeypatch.setattr(proxy_server.asyncio, "sleep", _next_cycle) + + with pytest.raises(asyncio.CancelledError): + await _run_background_health_check() + + assert [call.args[1:] for call in warn.call_args_list] == [(("image-1",),), (("image-1", "image-2"),)] + + +@pytest.mark.asyncio +async def test_run_background_health_check_skips_billed_media_generation_unless_opted_in(monkeypatch): + monkeypatch.setattr(proxy_server, "health_check_interval", 60) + monkeypatch.setattr(proxy_server, "health_check_concurrency", 1) + monkeypatch.setattr(proxy_server, "health_check_details", True) + monkeypatch.setattr(proxy_server, "use_shared_health_check", False) + monkeypatch.setattr(proxy_server, "redis_usage_cache", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "background_health_check_loop_active", False) + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr( + proxy_server, + "llm_model_list", + [ + {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat"}}, + { + "model_name": "image", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": {"id": "image", "mode": "image_generation"}, + }, + { + "model_name": "image-edit", + "litellm_params": {"model": "openai/gpt-image-1"}, + "model_info": {"id": "image-edit", "mode": "image_edit"}, + }, + { + "model_name": "video", + "litellm_params": {"model": "openai/sora-2"}, + "model_info": {"id": "video", "mode": "video_generation"}, + }, + { + "model_name": "speech", + "litellm_params": {"model": "openai/tts-1"}, + "model_info": {"id": "speech", "mode": "audio_speech"}, + }, + { + "model_name": "embedding", + "litellm_params": {"model": "openai/text-embedding-3-small"}, + "model_info": {"id": "embedding", "mode": "embedding"}, + }, + {"model_name": "fine-tune", "litellm_params": {"model": "openai/my-fine-tune"}, "model_info": {"id": "fine-tune"}}, + { + "model_name": "bad-max-tokens", + "litellm_params": {"model": "openai/my-fine-tune"}, + "model_info": {"id": "bad-max-tokens", "health_check_max_tokens": "abc"}, + }, + { + "model_name": "azure-ai-image", + "litellm_params": {"model": "azure_ai/gpt-image-1", "api_base": "https://example.services.ai.azure.com"}, + "model_info": {"id": "azure-ai-image"}, + }, + { + "model_name": "image-opted-in", + "litellm_params": {"model": "gemini/gemini-3-pro-image"}, + "model_info": { + "id": "image-opted-in", + "mode": "image_generation", + "disable_background_health_check": False, + }, + }, + ], + ) + monkeypatch.setattr( + proxy_server, + "health_check_results", + {"healthy_endpoints": [], "unhealthy_endpoints": []}, + ) + + probed = {} + + async def _fake_direct(model_list, *_a, **_kw): + probed["ids"] = [m["model_info"]["id"] for m in model_list] + return ([], [], {}) + + monkeypatch.setattr( + proxy_server, + "_run_direct_health_check_with_instrumentation", + _fake_direct, + ) + monkeypatch.setattr(proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None) + monkeypatch.setattr( + proxy_server, + "health_check_filter_kwargs_from_general_settings", + lambda _gs: {}, + ) + + async def _stop_sleep(_seconds): + raise asyncio.CancelledError() + + monkeypatch.setattr(proxy_server.asyncio, "sleep", _stop_sleep) + + with pytest.raises(asyncio.CancelledError): + await _run_background_health_check() + + assert probed["ids"] == ["chat", "speech", "embedding", "fine-tune", "bad-max-tokens", "image-opted-in"] diff --git a/tests/unit/proxy/test_health_check_functions.py b/tests/unit/proxy/test_health_check_functions.py index 79eec44958f..970ce1ca50b 100644 --- a/tests/unit/proxy/test_health_check_functions.py +++ b/tests/unit/proxy/test_health_check_functions.py @@ -8,6 +8,7 @@ import pytest from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.db.health_check_latest import LatestHealthCheckRow +from litellm.proxy.health_check import BILLED_MEDIA_NOT_CHECKED_WARNING from litellm.proxy.health_endpoints._health_endpoints import ( _aggregate_health_check_results, _build_model_param_to_info_mapping, @@ -666,6 +667,66 @@ async def test_save_background_health_checks_compares_raw_checked_at_against_utc assert [call.kwargs["model_id"] for call in mock_prisma.save_health_check_result.await_args_list] == ["stale-id"] +def _saved_row(model_id: str, status: str, error_message: str | None) -> LatestHealthCheckRow: + return LatestHealthCheckRow( + health_check_id=f"hc-{model_id}", + model_name=model_id, + model_id=model_id, + status=status, + healthy_count=1 if status == "healthy" else 0, + unhealthy_count=0 if status == "healthy" else 1, + error_message=error_message, + response_time_ms=12.5, + details=None, + checked_by="pod-1", + checked_at=datetime(2026, 8, 25), + created_at=datetime(2026, 8, 25, tzinfo=timezone.utc), + updated_at=datetime(2026, 8, 25, tzinfo=timezone.utc), + ) + + +@pytest.mark.asyncio +async def test_latest_health_checks_stops_repeating_an_old_verdict_for_a_skipped_media_model(monkeypatch): + prisma = MagicMock() + prisma.get_all_latest_health_checks = AsyncMock( + return_value=( + _saved_row("image-was-healthy", "healthy", None), + _saved_row("image-was-dead", "unhealthy", "401 from the provider"), + _saved_row("image-opted-in", "unhealthy", "401 from the provider"), + _saved_row("chat", "unhealthy", "401 from the provider"), + ) + ) + image = {"litellm_params": {"model": "gemini/gemini-3-pro-image"}} + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma) + monkeypatch.setattr( + "litellm.proxy.proxy_server.llm_model_list", + [ + {**image, "model_info": {"id": "image-was-healthy", "mode": "image_generation"}}, + {**image, "model_info": {"id": "image-was-dead", "mode": "image_generation"}}, + { + **image, + "model_info": { + "id": "image-opted-in", + "mode": "image_generation", + "disable_background_health_check": False, + }, + }, + {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat"}}, + ], + ) + + response = await latest_health_checks_endpoint(user_api_key_dict=UserAPIKeyAuth()) + + rows = response["latest_health_checks"] + assert {model_id: (row["status"], row["error_message"]) for model_id, row in rows.items()} == { + "image-was-healthy": ("skipped", BILLED_MEDIA_NOT_CHECKED_WARNING), + "image-was-dead": ("skipped", BILLED_MEDIA_NOT_CHECKED_WARNING), + "image-opted-in": ("unhealthy", "401 from the provider"), + "chat": ("unhealthy", "401 from the provider"), + } + assert rows["image-was-healthy"]["checked_at"] == "2026-08-25T00:00:00+00:00" + + @pytest.mark.asyncio async def test_latest_health_checks_endpoint_serialises_raw_rows(monkeypatch): row = LatestHealthCheckRow( diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index 26d9fe2fb3d..fbea7b3117c 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -1223,11 +1223,95 @@ def test_a_dependency_that_opted_out_of_health_checks_is_never_probed(): ) marker = [d for d in router.model_list if d["model_info"]["id"] == "router-1"] - eligible = hc_module._health_check_eligible(router.model_list, skip_disabled=True) - probes = hc_module._dependency_deployments_to_probe(marker, eligible, router) + probes = hc_module._dependency_deployments_to_probe(marker, router.model_list, router, skip_disabled=True) assert probes == () - assert [d["model_info"]["id"] for d in eligible] == ["router-1"] + assert [d["model_info"]["id"] for d in hc_module._dependency_deployments_to_probe(marker, router.model_list, router)] == [ + "dead-1" + ] + + +def test_health_checks_skip_billed_media_by_the_mode_the_probe_would_run(): + """Modes and prices as model_prices_and_context_window.json listed them on 2026-10-30: gemini/gemini-3-pro-image + and gpt-image-1 are image_generation, openai/text-embedding-3-small is embedding, gemini/lyria-3-clip-preview and + vertex_ai/lyria-002 bill per clip with no token price, and gemini/gemini-3.1-pro-preview also has a token price.""" + + def deployment(model_id: str, model: str, **model_info: object) -> dict[str, object]: + return { + "model_name": model_id, + "litellm_params": {"model": model, "api_key": "sk-x", "api_base": "https://example.services.ai.azure.com"}, + "model_info": {"id": model_id, **model_info}, + } + + billed = ( + deployment("cost-map-image", "gemini/gemini-3-pro-image"), + deployment("azure-ai-image", "azure_ai/gpt-image-1"), + deployment("wildcard-probing-an-image", "openai/*", health_check_model="openai/gpt-image-1"), + deployment("clip-priced-chat", "gemini/lyria-3-clip-preview"), + deployment("clip-priced-speech", "vertex_ai/lyria-002", mode="audio_speech"), + deployment("image-with-bad-max-tokens", "openai/x", mode="image_generation", health_check_max_tokens="abc"), + deployment("opted-out-image", "gemini/gemini-3-pro-image", disable_background_health_check=True), + ) + probed = ( + deployment("chat", "openai/gpt-4o-mini"), + deployment("embedding", "openai/text-embedding-3-small"), + deployment("token-priced-chat-with-image-price", "gemini/gemini-3.1-pro-preview"), + deployment("wildcard", "openai/*"), + deployment("unknown", "openai/my-fine-tune"), + deployment("no-provider-prefix", "my-fine-tune"), + deployment("bad-max-tokens", "openai/my-fine-tune", health_check_max_tokens="abc"), + deployment("image-opted-in", "gemini/gemini-3-pro-image", disable_background_health_check=False), + ) + opted_out_chat = deployment("opted-out-chat", "openai/gpt-4o-mini", disable_background_health_check=True) + + assert hc_module._health_check_eligible([*billed, *probed, opted_out_chat], skip_disabled=True) == probed + assert hc_module._health_check_eligible([*billed, *probed, opted_out_chat], skip_disabled=False) == ( + *probed, + opted_out_chat, + ) + + +@pytest.mark.parametrize( + ("litellm_params", "billed"), + [ + ({"model": "lyria-002", "custom_llm_provider": "vertex_ai"}, True), + ({"model": "lyria-3-clip-preview", "custom_llm_provider": "gemini"}, True), + ({"model": "gemini-3.1-pro-preview", "custom_llm_provider": "gemini"}, False), + ({"model": "my-fine-tune", "custom_llm_provider": "vertex_ai"}, False), + ], +) +def test_per_clip_price_is_found_when_the_provider_is_set_apart_from_the_model( + litellm_params: dict[str, str], billed: bool +): + """Prices as model_prices_and_context_window.json listed them on 2026-10-31: the Lyria entries bill per clip with + no token price under their provider prefix alone, and gemini/gemini-3.1-pro-preview also has a token price.""" + deployment = {"litellm_params": litellm_params, "model_info": {"id": "clip", "mode": "audio_speech"}} + + assert hc_module.bills_media_on_probe(deployment) is billed + + +@pytest.mark.parametrize( + "model_info", + [ + {"health_check_max_tokens": float("inf")}, + {"health_check_max_tokens": "many"}, + {"health_check_model": 123}, + {1: "not a string key"}, + ], +) +def test_one_bad_model_info_value_cannot_stop_the_health_checks(model_info: dict[object, object]): + bad = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "bad", **model_info}} + control = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "control"}} + + assert hc_module.background_check_deployments([bad, control]) == (bad, control) + assert hc_module._health_check_eligible([bad, control], skip_disabled=True) == (bad, control) + assert hc_module._health_check_eligible([bad, control], skip_disabled=False) == (bad, control) + assert hc_module.all_bill_media_on_probe([bad, control], {"bad"}, False) is False + + +def test_a_failed_memory_reading_does_not_break_health_check_logging(): + with patch("resource.getrusage", side_effect=OSError("no rusage")): # test-quality-ok: stdlib call, no seam + assert hc_module._rss_mb_for_log() == "unknown" def test_narrowing_by_an_id_that_matches_nothing_keeps_the_whole_list(): diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx index 1b0194b6478..e96db13b633 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx @@ -105,6 +105,53 @@ describe("HealthCheckComponent", () => { expect(mockIndividualModelHealthCheckCall).not.toHaveBeenCalledWith("token", "gpt-4"); }); + it("shows a model the proxy declined to check as skipped, never healthy, even with an old saved result", async () => { + const warning = "Not checked, because this health check would generate billed media."; + const declinedResponse = { + healthy_count: 0, + unhealthy_count: 0, + healthy_endpoints: [], + unhealthy_endpoints: [], + warnings: [warning], + }; + mockIndividualModelHealthCheckCall.mockResolvedValue(declinedResponse); + mockLatestHealthChecksCall.mockResolvedValue({ + latest_health_checks: { "image-1": { status: "healthy", checked_at: "2026-10-01T00:00:00Z" } }, + }); + await renderHealthCheck({ modelData: { data: [makeModel("image-1", "image")] }, allModelsOnProxy: ["image-1"] }); + + for (const runButton of ["run-health-check-btn", "run-health-checks"]) { + await act(async () => { + screen.getByTestId(runButton).click(); + }); + await act(async () => { + await new Promise((r) => setTimeout(r, 50)); + }); + + expect(screen.getByText("skipped")).toBeInTheDocument(); + expect(screen.queryByText("healthy")).not.toBeInTheDocument(); + expect(screen.getByText(warning)).toBeInTheDocument(); + expect(screen.getByText(warning)).not.toHaveClass("text-destructive"); + } + }); + + it("shows the skip reason on load when the proxy marks a saved result as skipped", async () => { + const warning = "Not checked, because this health check would generate media the provider may bill."; + mockLatestHealthChecksCall.mockResolvedValue({ + latest_health_checks: { + "image-1": { status: "skipped", checked_at: "2026-10-01T00:00:00Z", error_message: warning }, + }, + }); + await renderHealthCheck({ modelData: { data: [makeModel("image-1", "image")] }, allModelsOnProxy: ["image-1"] }); + await act(async () => { + await new Promise((r) => setTimeout(r, 50)); + }); + + expect(screen.getByText("skipped")).toBeInTheDocument(); + expect(screen.getByText(warning)).toBeInTheDocument(); + expect(screen.getByText(warning)).not.toHaveClass("text-destructive"); + }); + it("should page through results with the shared pagination footer", async () => { const onPageIndexChange = vi.fn(); await renderHealthCheck({ diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx index 3c274d2e59b..89374439946 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx @@ -20,6 +20,22 @@ import { individualModelHealthCheckCall, latestHealthChecksCall } from "../netwo import { HealthChecksTable } from "./HealthChecksTable"; import type { HealthCheckData, HealthStatus } from "./HealthChecksTableColumns"; +const isSkippedResponse = (response: { + healthy_count: number; + unhealthy_count: number; + warnings?: string[]; +}): response is { healthy_count: 0; unhealthy_count: 0; warnings: string[] } => + response.healthy_count === 0 && response.unhealthy_count === 0 && Boolean(response.warnings?.length); + +const skippedStatus = (warning: string, lastCheck: string, previous?: HealthStatus): HealthStatus => ({ + status: "skipped", + lastCheck, + lastSuccess: previous?.lastSuccess || "None", + loading: false, + error: warning, + fullError: warning, +}); + interface LatestHealthCheck { status?: string; checked_at?: string | null; @@ -227,6 +243,11 @@ const HealthCheckComponent: React.FC = ({ const fullError = checkData.error_message || undefined; + if (checkData.status === "skipped" && fullError) { + healthStatusMap[modelId] = skippedStatus(fullError, toCheckedAtLabel(checkData.checked_at, "None")); + return; + } + healthStatusMap[modelId] = { status: checkData.status || "unknown", lastCheck: toCheckedAtLabel(checkData.checked_at, "None"), @@ -265,6 +286,14 @@ const HealthCheckComponent: React.FC = ({ const response = await individualModelHealthCheckCall(accessToken, modelId); const currentTime = new Date().toLocaleString(); + if (isSkippedResponse(response)) { + setModelHealthStatuses((prev) => ({ + ...prev, + [modelId]: skippedStatus(response.warnings[0], currentTime, prev[modelId]), + })); + return; + } + if (response.unhealthy_count > 0 && response.unhealthy_endpoints && response.unhealthy_endpoints.length > 0) { const rawError = response.unhealthy_endpoints[0]?.error || i18n.t("modelDashboard.healthCheckComponent.healthCheckFailed"); @@ -362,6 +391,14 @@ const HealthCheckComponent: React.FC = ({ const response = await individualModelHealthCheckCall(accessToken, modelId); const currentTime = new Date().toLocaleString(); + if (isSkippedResponse(response)) { + setModelHealthStatuses((prev) => ({ + ...prev, + [modelId]: skippedStatus(response.warnings[0], currentTime, prev[modelId]), + })); + return; + } + if (response.unhealthy_count > 0 && response.unhealthy_endpoints && response.unhealthy_endpoints.length > 0) { const rawError = response.unhealthy_endpoints[0]?.error || t("modelDashboard.healthCheckComponent.healthCheckFailed"); @@ -422,6 +459,9 @@ const HealthCheckComponent: React.FC = ({ setModelHealthStatuses((prev) => { const currentStatus = prev[modelId]; + if (currentStatus?.status === "skipped") { + return prev; + } return { ...prev, [modelId]: { diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx index 85539dc1baa..820a3cc3b08 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx @@ -43,10 +43,11 @@ const HEALTH_STATUS_TONES: Record = { unhealthy: "error", checking: "info", none: "neutral", + skipped: "neutral", }; // healthy > checking > unknown > unhealthy, matching the legacy health table ordering. -const HEALTH_STATUS_ORDER: Record = { healthy: 0, checking: 1, unknown: 2, unhealthy: 3 }; +const HEALTH_STATUS_ORDER: Record = { healthy: 0, checking: 1, unknown: 2, skipped: 2, unhealthy: 3 }; const NEVER_CHECKED = "Never checked"; const CHECK_IN_PROGRESS = "Check in progress..."; @@ -384,6 +385,15 @@ export const getHealthChecksTableColumns = ( } const cleanedError = healthStatus.error; + + if (healthStatus.status === "skipped") { + return ( + + {cleanedError} + + ); + } + const fullError = healthStatus.fullError || healthStatus.error; const displayName = getDisplayModelName(model) || model.model_name; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2272b4bea43..e7feedb8132 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7702,8 +7702,11 @@ export interface paths { * ``` * else, the health checks will be run on models when /health is called. * - * To skip deployments that set ``model_info.disable_background_health_check: true`` - * on ``GET /health`` as well as in the background loop, set + * Deployments whose check would generate billed media (image_generation, image_edit, video_generation, + * and models priced per generated clip) are not checked here or by the background loop unless they set + * ``model_info.disable_background_health_check: false``. A ``model`` or ``model_id`` that targets only + * such deployments answers 200 with zero counts and a ``warnings`` entry. To also skip deployments that + * set ``disable_background_health_check: true`` on ``GET /health``, set * ``general_settings.health_check_skip_disabled_background_models: true``. */ get: operations["health_endpoint_health_get"]; @@ -30818,7 +30821,7 @@ export interface components { apply_user_budget_to_team_keys?: boolean | null; /** * Background Health Check Model Groups - * @description Opt-in allowlist of model group names for background health checks and health-check routing. When set, the background loop probes only deployments whose model_name is listed, and enable_health_check_routing filters unhealthy deployments only within the listed groups; every other group, including newly added deployments, is skipped and keeps its configured routing strategy. When unset, all deployments participate (opt out per deployment via model_info.disable_background_health_check). + * @description Opt-in allowlist of model group names for background health checks and health-check routing. When set, the background loop probes only deployments whose model_name is listed, and enable_health_check_routing filters unhealthy deployments only within the listed groups; every other group, including newly added deployments, is skipped and keeps its configured routing strategy. When unset, all deployments participate (opt out per deployment via model_info.disable_background_health_check). Set or unset, deployments whose check would generate billed media take part only with model_info.disable_background_health_check: false. */ background_health_check_model_groups?: string[] | null; /**