mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
Merge 473896fc39 into 08f2a98250
This commit is contained in:
Shayan Danish
GitHub
commit
24eca873fb
15 changed files with 866 additions and 46 deletions
|
|
@ -50,11 +50,17 @@ def native_health_check_mode(
|
|||
return mantle_health_check_mode(model)
|
||||
|
||||
|
||||
def _cost_map_mode(model: str) -> str | None:
|
||||
def cost_map_entry(model: str) -> Mapping[str, object]:
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.health_check_utils import STR_KEY_MAPPING
|
||||
|
||||
return STR_KEY_MAPPING.validate_python(litellm.model_cost.get(model, {}))
|
||||
|
||||
|
||||
def _cost_map_mode(model: str) -> str | None:
|
||||
from litellm.litellm_core_utils.health_check_utils import OPTIONAL_STR
|
||||
|
||||
return OPTIONAL_STR.validate_python(litellm.model_cost.get(model, {}).get("mode"))
|
||||
return OPTIONAL_STR.validate_python(cost_map_entry(model).get("mode"))
|
||||
|
||||
|
||||
def default_health_check_mode(requested_model: str, model: str, custom_llm_provider: str) -> str:
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from litellm.types.extraction import ExtractionCallParams
|
|||
DECISIONS_CALL_PARAMS: Final[TypeAdapter[DecisionsCallParams]] = TypeAdapter(DecisionsCallParams)
|
||||
EXTRACTION_CALL_PARAMS: Final[TypeAdapter[ExtractionCallParams]] = TypeAdapter(ExtractionCallParams)
|
||||
OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
|
||||
STR_KEY_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _filter_model_params(model_params: dict) -> dict:
|
||||
|
|
|
|||
|
|
@ -2754,7 +2754,8 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
"deployments only within the listed groups; every other group, including newly "
|
||||
"added deployments, is skipped and keeps its configured routing strategy. "
|
||||
"When unset, all deployments participate (opt out per deployment via "
|
||||
"model_info.disable_background_health_check)."
|
||||
"model_info.disable_background_health_check). Set or unset, deployments whose check would "
|
||||
"generate billed media take part only with model_info.disable_background_health_check: false."
|
||||
),
|
||||
)
|
||||
model_list_healthy_only: bool | None = Field(
|
||||
|
|
|
|||
|
|
@ -26,7 +26,11 @@ from litellm.constants import (
|
|||
DEFAULT_HEALTH_CHECK_PROMPT,
|
||||
HEALTH_CHECK_TIMEOUT_SECONDS,
|
||||
)
|
||||
from litellm.litellm_core_utils.health_check_helpers import native_health_check_mode
|
||||
from litellm.litellm_core_utils.health_check_helpers import (
|
||||
cost_map_entry,
|
||||
default_health_check_mode,
|
||||
native_health_check_mode,
|
||||
)
|
||||
from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin, is_model_capability_field
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
StrategyRouterDependency,
|
||||
|
|
@ -193,7 +197,7 @@ def _get_process_rss_mb() -> float | None:
|
|||
if sys.platform == "darwin":
|
||||
return float(ru_maxrss) / (1024 * 1024)
|
||||
return float(ru_maxrss) / 1024
|
||||
except Exception:
|
||||
except (ImportError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -230,9 +234,9 @@ def health_check_filter_kwargs_from_general_settings(
|
|||
"""
|
||||
Build kwargs for ``perform_health_check`` from ``general_settings``.
|
||||
|
||||
When ``health_check_skip_disabled_background_models`` is true, deployments with
|
||||
``model_info.disable_background_health_check`` are omitted from health runs
|
||||
(including on-demand ``GET /health``), matching the background loop behavior.
|
||||
When ``health_check_skip_disabled_background_models`` is true, deployments the
|
||||
background loop skips are omitted from health runs (including on-demand
|
||||
``GET /health``).
|
||||
"""
|
||||
g: Final = general_settings or {}
|
||||
return {
|
||||
|
|
@ -307,9 +311,122 @@ async def run_with_timeout(task, timeout):
|
|||
return {"error": "Timeout exceeded", "exception": timeout_exception}
|
||||
|
||||
|
||||
def _skips_health_checks(deployment: Mapping[str, object]) -> bool:
|
||||
info: Final = deployment.get("model_info")
|
||||
return bool(info.get("disable_background_health_check", False)) if isinstance(info, Mapping) else False
|
||||
_BILLED_MEDIA_GENERATION_MODES: Final = frozenset({"image_generation", "image_edit", "video_generation"})
|
||||
BILLED_MEDIA_NOT_CHECKED_WARNING: Final = (
|
||||
"Not checked, because this health check would generate media the provider may bill. "
|
||||
"Set model_info.disable_background_health_check: false to check this model on every interval, "
|
||||
"or use Test Connection for one check."
|
||||
)
|
||||
_DEPLOYMENT_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
DEPLOYMENTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
MODEL_IDS_ADAPTER: Final = TypeAdapter(frozenset[object])
|
||||
_STR: Final = TypeAdapter(str)
|
||||
|
||||
|
||||
def _str_key_mapping(value: object) -> Mapping[str, object]:
|
||||
return _DEPLOYMENT_ADAPTER.validate_python(value) if isinstance(value, Mapping) else MappingProxyType({})
|
||||
|
||||
|
||||
def _fallback_probe_mode(probe_params: Mapping[str, object]) -> str | None:
|
||||
"""The mode `litellm.ahealth_check` picks for these probe params when the caller passes none."""
|
||||
provider: Final = probe_params.get("custom_llm_provider")
|
||||
api_base: Final = probe_params.get("api_base")
|
||||
api_key: Final = probe_params.get("api_key")
|
||||
try:
|
||||
model: Final = _STR.validate_python(probe_params.get("model"))
|
||||
resolved_model, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=provider if isinstance(provider, str) else None,
|
||||
api_base=api_base if isinstance(api_base, str) else None,
|
||||
api_key=api_key if isinstance(api_key, str) else None,
|
||||
)
|
||||
except (ValidationError, litellm.BadRequestError):
|
||||
return None
|
||||
return default_health_check_mode(
|
||||
requested_model=model, model=resolved_model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
|
||||
def _bills_per_generated_clip(entry: Mapping[str, object]) -> bool:
|
||||
return bool(entry.get("output_cost_per_image")) and not entry.get("output_cost_per_token")
|
||||
|
||||
|
||||
def _priced_per_generated_clip(probe_params: Mapping[str, object]) -> bool:
|
||||
model: Final = _STR.validate_python(probe_params.get("model"))
|
||||
provider: Final = probe_params.get("custom_llm_provider")
|
||||
names: Final = (model, f"{provider}/{model}") if isinstance(provider, str) else (model,)
|
||||
return any(_bills_per_generated_clip(cost_map_entry(name)) for name in names)
|
||||
|
||||
|
||||
def _probe_generates_billed_media(model_info: Mapping[str, object], litellm_params: Mapping[str, object]) -> bool:
|
||||
mode: Final = resolve_health_check_mode(model_info, litellm_params)
|
||||
if mode in _BILLED_MEDIA_GENERATION_MODES:
|
||||
return True
|
||||
if not isinstance(litellm_params.get("model"), str):
|
||||
return False
|
||||
probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python(
|
||||
update_litellm_params_for_health_check(dict(model_info), dict(litellm_params))
|
||||
)
|
||||
return (mode is None and _fallback_probe_mode(probe_params) in _BILLED_MEDIA_GENERATION_MODES) or (
|
||||
_priced_per_generated_clip(probe_params)
|
||||
)
|
||||
|
||||
|
||||
def _disable_background_flag(deployment: Mapping[str, object]) -> object:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
return model_info.get("disable_background_health_check") if isinstance(model_info, Mapping) else None
|
||||
|
||||
|
||||
def bills_media_on_probe(deployment: Mapping[str, object]) -> bool:
|
||||
explicit: Final = _disable_background_flag(deployment)
|
||||
if explicit is not None and not explicit:
|
||||
return False
|
||||
try:
|
||||
return _probe_generates_billed_media(
|
||||
_str_key_mapping(deployment.get("model_info")), _str_key_mapping(deployment.get("litellm_params"))
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Could not work out the health check for %s", _deployment_id(deployment))
|
||||
return False
|
||||
|
||||
|
||||
def skips_background_health_check(deployment: Mapping[str, object]) -> bool:
|
||||
return bool(_disable_background_flag(deployment)) or bills_media_on_probe(deployment)
|
||||
|
||||
|
||||
def background_check_deployments(model_list: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(deployment for deployment in model_list if not skips_background_health_check(deployment))
|
||||
|
||||
|
||||
def _probe_targets(
|
||||
deployments: Sequence[Mapping[str, object]],
|
||||
ids: AbstractSet[object] | None,
|
||||
health_check_skip_disabled_background_models: bool,
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(
|
||||
d
|
||||
for d in deployments
|
||||
if (ids is None or _deployment_id(d) in ids)
|
||||
and not (health_check_skip_disabled_background_models and _disable_background_flag(d))
|
||||
)
|
||||
|
||||
|
||||
def all_bill_media_on_probe(
|
||||
deployments: Sequence[Mapping[str, object]],
|
||||
ids: AbstractSet[object],
|
||||
health_check_skip_disabled_background_models: bool,
|
||||
) -> bool:
|
||||
targets: Final = _probe_targets(deployments, ids, health_check_skip_disabled_background_models)
|
||||
return bool(targets) and all(bills_media_on_probe(d) for d in targets)
|
||||
|
||||
|
||||
def billed_media_model_ids(
|
||||
deployments: Sequence[Mapping[str, object]],
|
||||
ids: AbstractSet[object] | None = None,
|
||||
health_check_skip_disabled_background_models: bool = False,
|
||||
) -> tuple[str, ...]:
|
||||
targets: Final = _probe_targets(deployments, ids, health_check_skip_disabled_background_models)
|
||||
return tuple(sorted(filter(None, (_deployment_id(d) for d in targets if bills_media_on_probe(d)))))
|
||||
|
||||
|
||||
def _health_check_eligible(
|
||||
|
|
@ -317,11 +434,12 @@ def _health_check_eligible(
|
|||
) -> tuple[Mapping[str, object], ...]:
|
||||
"""Deployments this run is allowed to contact.
|
||||
|
||||
The one eligibility gate, applied to the requested set and to the pool a router's
|
||||
dependencies are drawn from alike, so an opted-out deployment cannot re-enter through a
|
||||
router that depends on it.
|
||||
The one eligibility gate, applied to the requested set and to each round of a router's
|
||||
dependencies alike, so an opted-out deployment cannot re-enter through a router that
|
||||
depends on it.
|
||||
"""
|
||||
return tuple(x for x in model_list if not (skip_disabled and _skips_health_checks(x)))
|
||||
skipped: Final = skips_background_health_check if skip_disabled else bills_media_on_probe
|
||||
return tuple(x for x in model_list if not skipped(x))
|
||||
|
||||
|
||||
def _deployment_model(deployment: Mapping[str, object]) -> str | None:
|
||||
|
|
@ -471,6 +589,7 @@ def _dependency_deployments_to_probe(
|
|||
checked: Sequence[Mapping[str, object]],
|
||||
universe: Sequence[Mapping[str, object]],
|
||||
router: "Router",
|
||||
skip_disabled: bool = False,
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
"""Deployments backing the checked routers' dependencies that are not already checked.
|
||||
|
||||
|
|
@ -478,7 +597,7 @@ def _dependency_deployments_to_probe(
|
|||
`/health?model_id=<router>` call the dashboard makes per deployment that needs them,
|
||||
since a router's verdict is a statement about models the request never named. Drawn from
|
||||
`universe`, the caller's access-filtered list, so no deployment is probed that the caller
|
||||
was not already granted. Expansion follows routers through routers, one hop per round,
|
||||
was not already granted, and only those `_health_check_eligible` passes. Expansion follows routers through routers, one hop per round,
|
||||
because a child router's own models must be probed for the parent to fail; stopping when
|
||||
a round adds nothing is what makes a router cycle terminate.
|
||||
"""
|
||||
|
|
@ -498,9 +617,9 @@ def _dependency_deployments_to_probe(
|
|||
)
|
||||
if not fresh_ids:
|
||||
break
|
||||
frontier = _deployments_by_id(universe, fresh_ids)
|
||||
frontier = _health_check_eligible(_deployments_by_id(universe, fresh_ids), skip_disabled)
|
||||
reached = reached | fresh_ids
|
||||
return _deployments_by_id(universe, reached - checked_ids)
|
||||
return _health_check_eligible(_deployments_by_id(universe, reached - checked_ids), skip_disabled)
|
||||
|
||||
|
||||
def _strategy_router_verdicts(
|
||||
|
|
@ -972,11 +1091,13 @@ async def perform_health_check(
|
|||
another team's deployment carries, else the deployments whose ``litellm_params.model``
|
||||
is that string.
|
||||
|
||||
Deployments whose check would generate billed media are omitted unless they set
|
||||
``model_info.disable_background_health_check: false``.
|
||||
|
||||
When ``health_check_skip_disabled_background_models`` is True (via
|
||||
``general_settings.health_check_skip_disabled_background_models``), deployments
|
||||
with ``model_info.disable_background_health_check: true`` are omitted from
|
||||
this run (including targeted ``/health`` queries), consistent with the
|
||||
background health loop.
|
||||
the background health loop skips are omitted from this run (including targeted
|
||||
``/health`` queries).
|
||||
|
||||
Returns:
|
||||
(bool): True if the health check passes, False otherwise.
|
||||
|
|
@ -1001,7 +1122,9 @@ async def perform_health_check(
|
|||
cycle_start_time: Final = time.monotonic()
|
||||
requested_model_count: Final = len(model_list)
|
||||
skip_disabled: Final = health_check_skip_disabled_background_models
|
||||
narrowed: Final = _health_check_eligible(_narrow_to_target(model_list, model, model_id, team_id), skip_disabled)
|
||||
narrowed: Final = await asyncio.to_thread(
|
||||
_health_check_eligible, _narrow_to_target(model_list, model, model_id, team_id), skip_disabled
|
||||
)
|
||||
if not narrowed:
|
||||
if instrumentation_enabled:
|
||||
logger.debug(
|
||||
|
|
@ -1016,9 +1139,7 @@ async def perform_health_check(
|
|||
deduped_model_count: Final = len(requested)
|
||||
|
||||
dependency_probes: Final = (
|
||||
_dependency_deployments_to_probe(requested, _health_check_eligible(model_list, skip_disabled), router)
|
||||
if router is not None
|
||||
else ()
|
||||
_dependency_deployments_to_probe(requested, model_list, router, skip_disabled) if router is not None else ()
|
||||
)
|
||||
checked: Final = requested + list(dependency_probes)
|
||||
|
||||
|
|
|
|||
|
|
@ -225,7 +225,8 @@ class SharedHealthCheckManager:
|
|||
model_list: List of models to check
|
||||
details: Whether to include detailed information
|
||||
max_concurrency: Optional limit on concurrent health check requests
|
||||
health_check_skip_disabled_background_models: Remove models with disable_background_health_check: true
|
||||
health_check_skip_disabled_background_models: Also remove models with disable_background_health_check: true.
|
||||
A model whose check would generate billed media is removed either way, unless it set that flag to false
|
||||
|
||||
Returns:
|
||||
Tuple of (healthy_endpoints, unhealthy_endpoints)
|
||||
|
|
|
|||
|
|
@ -80,8 +80,13 @@ from litellm.proxy.db.health_check_latest import (
|
|||
from litellm.proxy.db.proxy_worker_heartbeat import count_live_proxy_workers
|
||||
from litellm.proxy.health_check import ( # noqa: F401 # legacy module exports
|
||||
ADMIN_ONLY_HEALTH_DISPLAY_PARAMS,
|
||||
BILLED_MEDIA_NOT_CHECKED_WARNING,
|
||||
DEPLOYMENTS_ADAPTER,
|
||||
MODEL_IDS_ADAPTER,
|
||||
_clean_endpoint_data, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
_update_litellm_params_for_health_check, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
all_bill_media_on_probe,
|
||||
billed_media_model_ids,
|
||||
clean_endpoint_data,
|
||||
deployments_targeted_by_name,
|
||||
health_check_filter_kwargs_from_general_settings,
|
||||
|
|
@ -1577,8 +1582,11 @@ async def health_endpoint(
|
|||
```
|
||||
else, the health checks will be run on models when /health is called.
|
||||
|
||||
To skip deployments that set ``model_info.disable_background_health_check: true``
|
||||
on ``GET /health`` as well as in the background loop, set
|
||||
Deployments whose check would generate billed media (image_generation, image_edit, video_generation,
|
||||
and models priced per generated clip) are not checked here or by the background loop unless they set
|
||||
``model_info.disable_background_health_check: false``. A ``model`` or ``model_id`` that targets only
|
||||
such deployments answers 200 with zero counts and a ``warnings`` entry. To also skip deployments that
|
||||
set ``disable_background_health_check: true`` on ``GET /health``, set
|
||||
``general_settings.health_check_skip_disabled_background_models: true``.
|
||||
"""
|
||||
import time
|
||||
|
|
@ -1686,6 +1694,22 @@ async def health_endpoint(
|
|||
else f"key not allowed to health-check model {model}"
|
||||
},
|
||||
)
|
||||
deployments: Final = DEPLOYMENTS_ADAPTER.validate_python(_llm_model_list)
|
||||
probe_ids: Final = None if targeted_ids is None else MODEL_IDS_ADAPTER.validate_python(targeted_ids)
|
||||
if probe_ids is not None and all_bill_media_on_probe(deployments, probe_ids, **_hc_filter):
|
||||
return {
|
||||
"healthy_endpoints": [],
|
||||
"unhealthy_endpoints": [],
|
||||
"healthy_count": 0,
|
||||
"unhealthy_count": 0,
|
||||
"warnings": [BILLED_MEDIA_NOT_CHECKED_WARNING],
|
||||
}
|
||||
not_checked_ids: Final = await asyncio.to_thread(billed_media_model_ids, deployments, probe_ids, **_hc_filter)
|
||||
media_warning: Final = (
|
||||
{"warnings": [f"{BILLED_MEDIA_NOT_CHECKED_WARNING} Model ids: {', '.join(not_checked_ids)}"]}
|
||||
if not_checked_ids
|
||||
else {}
|
||||
)
|
||||
if use_background_health_checks:
|
||||
# The cached background result covers every model. When the
|
||||
# caller targets a specific model/model_id we have to narrow the
|
||||
|
|
@ -1724,13 +1748,18 @@ async def health_endpoint(
|
|||
"and were excluded from this response. Ask a proxy admin "
|
||||
"to populate model_info.id for these models."
|
||||
]
|
||||
return _post_process(filtered)
|
||||
return _post_process({**filtered, **media_warning})
|
||||
if targeted_ids is not None:
|
||||
# Admin caller targeting a specific model: filter the cache
|
||||
# so the response (and the targeted-503 check) reflects only
|
||||
# that deployment, not the global aggregate.
|
||||
return _post_process(_filter_health_check_results_by_model_ids(health_check_results, targeted_ids))
|
||||
return _post_process(health_check_results)
|
||||
return _post_process(
|
||||
{
|
||||
**_filter_health_check_results_by_model_ids(health_check_results, targeted_ids),
|
||||
**media_warning,
|
||||
}
|
||||
)
|
||||
return _post_process({**health_check_results, **media_warning})
|
||||
else:
|
||||
router_result: Final = await _perform_health_check_and_save(
|
||||
model_list=_llm_model_list,
|
||||
|
|
@ -1747,7 +1776,7 @@ async def health_endpoint(
|
|||
team_id=user_api_key_dict.team_id,
|
||||
**_hc_filter,
|
||||
)
|
||||
return _post_process(router_result)
|
||||
return _post_process({**router_result, **media_warning})
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error("litellm.proxy.proxy_server.py::health_endpoint(): Exception occured - %s", e)
|
||||
verbose_proxy_logger.debug(traceback.format_exc())
|
||||
|
|
@ -1794,6 +1823,13 @@ async def health_check_history_endpoint(
|
|||
)
|
||||
|
||||
|
||||
def _latest_check_row(check: LatestHealthCheckRow, skipped_ids: frozenset[str]) -> Mapping[str, object]:
|
||||
row: Final = convert_health_check_to_dict(check)
|
||||
if check.model_id not in skipped_ids:
|
||||
return row
|
||||
return {**row, "status": "skipped", "error_message": BILLED_MEDIA_NOT_CHECKED_WARNING}
|
||||
|
||||
|
||||
@router.get("/health/latest", tags=["health"], dependencies=[Depends(user_api_key_auth)])
|
||||
async def latest_health_checks_endpoint(
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
|
|
@ -1803,14 +1839,19 @@ async def latest_health_checks_endpoint(
|
|||
|
||||
Returns the most recent health check result for each model.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import llm_model_list
|
||||
|
||||
prisma_client: Final = _check_prisma_client()
|
||||
|
||||
try:
|
||||
latest_checks: Final = await prisma_client.get_all_latest_health_checks()
|
||||
skipped_ids: Final = frozenset(
|
||||
await asyncio.to_thread(billed_media_model_ids, DEPLOYMENTS_ADAPTER.validate_python(llm_model_list or ()))
|
||||
)
|
||||
|
||||
# Convert to dict format for JSON response using helper function
|
||||
checks_data: Final = {
|
||||
(check.model_id if check.model_id else check.model_name): convert_health_check_to_dict(check)
|
||||
(check.model_id if check.model_id else check.model_name): _latest_check_row(check, skipped_ids)
|
||||
for check in latest_checks
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -689,6 +689,9 @@ from litellm.proxy.guardrails.init_guardrails import (
|
|||
initialize_guardrails,
|
||||
)
|
||||
from litellm.proxy.health_check import (
|
||||
DEPLOYMENTS_ADAPTER,
|
||||
background_check_deployments,
|
||||
billed_media_model_ids,
|
||||
filter_deployments_to_model_groups,
|
||||
health_check_filter_kwargs_from_general_settings,
|
||||
parse_background_health_check_model_groups,
|
||||
|
|
@ -4936,6 +4939,7 @@ async def _run_background_health_check():
|
|||
)
|
||||
verbose_proxy_logger.info("Initialized shared health check manager")
|
||||
|
||||
logged_skipped_media_ids: tuple[str, ...] = () # rebind-ok: the last set logged, compared each cycle
|
||||
while True:
|
||||
background_health_check_cycle_seq += 1
|
||||
cycle_id = f"bg-{background_health_check_cycle_seq}"
|
||||
|
|
@ -4945,17 +4949,25 @@ async def _run_background_health_check():
|
|||
_llm_model_list = copy.deepcopy(llm_model_list) or []
|
||||
model_count_total = len(_llm_model_list)
|
||||
|
||||
# filter out models that have disabled background health checks
|
||||
_llm_model_list = [
|
||||
m for m in _llm_model_list if not m.get("model_info", {}).get("disable_background_health_check", False)
|
||||
]
|
||||
scoped_model_groups = (
|
||||
llm_router.settings().background_health_check_model_groups if llm_router is not None else None
|
||||
)
|
||||
_llm_model_list = list(filter_deployments_to_model_groups(_llm_model_list, scoped_model_groups))
|
||||
group_deployments = DEPLOYMENTS_ADAPTER.validate_python(
|
||||
filter_deployments_to_model_groups(_llm_model_list, scoped_model_groups)
|
||||
)
|
||||
_llm_model_list = list(await asyncio.to_thread(background_check_deployments, group_deployments))
|
||||
skipped_media_ids = await asyncio.to_thread(billed_media_model_ids, group_deployments)
|
||||
if skipped_media_ids and skipped_media_ids != logged_skipped_media_ids:
|
||||
verbose_proxy_logger.warning(
|
||||
"background health checks skip deployments whose check would generate billed media; "
|
||||
"set model_info.disable_background_health_check: false to check them; model_ids=%s",
|
||||
skipped_media_ids,
|
||||
)
|
||||
logged_skipped_media_ids = skipped_media_ids
|
||||
if scoped_model_groups is not None and not _llm_model_list:
|
||||
verbose_proxy_logger.warning(
|
||||
"background_health_check_model_groups matched no deployments; groups=%s",
|
||||
"background_health_check_model_groups %s; groups=%s",
|
||||
"matched only deployments the background loop skips" if group_deployments else "matched no deployments",
|
||||
sorted(scoped_model_groups),
|
||||
)
|
||||
model_count_enabled = len(_llm_model_list)
|
||||
|
|
|
|||
|
|
@ -2641,6 +2641,181 @@ async def test_health_endpoint_omits_model_id_warning_when_no_deployment_matches
|
|||
assert "warnings" not in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_background_health_checks", [True, False])
|
||||
async def test_targeted_health_answers_without_probing_billed_media(use_background_health_checks):
|
||||
from fastapi import Response
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.health_check import BILLED_MEDIA_NOT_CHECKED_WARNING
|
||||
from litellm.proxy.health_endpoints._health_endpoints import health_endpoint
|
||||
|
||||
model_list = [
|
||||
{
|
||||
"model_name": "image",
|
||||
"litellm_params": {"model": "gemini/gemini-3-pro-image"},
|
||||
"model_info": {"id": "image-1", "mode": "image_generation"},
|
||||
},
|
||||
{
|
||||
"model_name": "image-opted-out",
|
||||
"litellm_params": {"model": "gemini/gemini-3-pro-image"},
|
||||
"model_info": {"id": "image-off", "mode": "image_generation", "disable_background_health_check": True},
|
||||
},
|
||||
{
|
||||
"model_name": "image-opted-in",
|
||||
"litellm_params": {"model": "gemini/gemini-3-pro-image"},
|
||||
"model_info": {"id": "image-on", "mode": "image_generation", "disable_background_health_check": False},
|
||||
},
|
||||
{"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat-1"}},
|
||||
]
|
||||
cached = {
|
||||
"healthy_endpoints": [
|
||||
{"model": "openai/gpt-4o-mini", "model_id": "chat-1"},
|
||||
{"model": "gemini/gemini-3-pro-image", "model_id": "image-on"},
|
||||
],
|
||||
"unhealthy_endpoints": [],
|
||||
"healthy_count": 2,
|
||||
"unhealthy_count": 0,
|
||||
}
|
||||
live_model_ids: list[str | None] = []
|
||||
|
||||
async def fake_perform(**kwargs):
|
||||
live_model_ids.append(kwargs["model_id"])
|
||||
return {
|
||||
"healthy_endpoints": [{"model": "probed", "model_id": kwargs["model_id"]}],
|
||||
"unhealthy_endpoints": [],
|
||||
"healthy_count": 1,
|
||||
"unhealthy_count": 0,
|
||||
}
|
||||
|
||||
admin = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token")
|
||||
responses = {model_id: Response() for model_id in ("image-1", "image-off", "image-on", "chat-1")}
|
||||
with (
|
||||
_proxy_health_globals(
|
||||
model_list,
|
||||
litellm.Router(model_list=copy.deepcopy(model_list)),
|
||||
use_background_health_checks=use_background_health_checks,
|
||||
health_check_results=cached,
|
||||
),
|
||||
patch( # test-quality-ok: whether the live probe ran is the assertion; no injection seam
|
||||
"litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save",
|
||||
side_effect=fake_perform,
|
||||
),
|
||||
):
|
||||
results = {
|
||||
model_id: await health_endpoint(
|
||||
response=response, user_api_key_dict=admin, model=None, model_id=model_id
|
||||
)
|
||||
for model_id, response in responses.items()
|
||||
}
|
||||
|
||||
assert live_model_ids == ([] if use_background_health_checks else ["image-on", "chat-1"])
|
||||
for skipped in ("image-1", "image-off"):
|
||||
assert responses[skipped].status_code == 200
|
||||
assert results[skipped]["healthy_endpoints"] == results[skipped]["unhealthy_endpoints"] == []
|
||||
assert results[skipped]["healthy_count"] == results[skipped]["unhealthy_count"] == 0
|
||||
assert results[skipped]["warnings"] == [BILLED_MEDIA_NOT_CHECKED_WARNING]
|
||||
for checked in ("image-on", "chat-1"):
|
||||
assert responses[checked].status_code == 200
|
||||
assert results[checked]["healthy_count"] == 1
|
||||
assert "warnings" not in results[checked]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_background_health_checks", [True, False])
|
||||
async def test_health_names_the_media_models_it_left_out(use_background_health_checks):
|
||||
from fastapi import Response
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.health_check import BILLED_MEDIA_NOT_CHECKED_WARNING
|
||||
from litellm.proxy.health_endpoints._health_endpoints import health_endpoint
|
||||
|
||||
def image(model_name: str, model_id: str) -> dict:
|
||||
return {
|
||||
"model_name": model_name,
|
||||
"litellm_params": {"model": "gemini/gemini-3-pro-image"},
|
||||
"model_info": {"id": model_id, "mode": "image_generation"},
|
||||
}
|
||||
|
||||
chat = {"model_name": "mixed", "litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat-1"}}
|
||||
model_list = [image("mixed", "image-in-mixed"), image("image", "image-2"), chat]
|
||||
checked = {
|
||||
"healthy_endpoints": [{"model": "openai/gpt-4o-mini", "model_id": "chat-1"}],
|
||||
"unhealthy_endpoints": [],
|
||||
"healthy_count": 1,
|
||||
"unhealthy_count": 0,
|
||||
}
|
||||
|
||||
async def fake_perform(**kwargs):
|
||||
return dict(checked)
|
||||
|
||||
admin = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token")
|
||||
with (
|
||||
_proxy_health_globals(
|
||||
model_list,
|
||||
litellm.Router(model_list=copy.deepcopy(model_list)),
|
||||
use_background_health_checks=use_background_health_checks,
|
||||
health_check_results=checked,
|
||||
),
|
||||
patch( # test-quality-ok: the live probe is the outside call; the answer built around it is under test
|
||||
"litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save",
|
||||
side_effect=fake_perform,
|
||||
),
|
||||
):
|
||||
everything = await health_endpoint(response=Response(), user_api_key_dict=admin, model=None, model_id=None)
|
||||
mixed_response = Response()
|
||||
mixed = await health_endpoint(response=mixed_response, user_api_key_dict=admin, model="mixed", model_id=None)
|
||||
one_chat = await health_endpoint(response=Response(), user_api_key_dict=admin, model=None, model_id="chat-1")
|
||||
|
||||
assert everything["healthy_count"] == 1
|
||||
assert everything["warnings"] == [f"{BILLED_MEDIA_NOT_CHECKED_WARNING} Model ids: image-2, image-in-mixed"]
|
||||
assert mixed_response.status_code == 200
|
||||
assert mixed["healthy_count"] == 1
|
||||
assert mixed["warnings"] == [f"{BILLED_MEDIA_NOT_CHECKED_WARNING} Model ids: image-in-mixed"]
|
||||
assert "warnings" not in one_chat
|
||||
assert "warnings" not in checked
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("model_info", "expected_status"),
|
||||
[
|
||||
({"id": "image-1", "mode": "image_generation", "disable_background_health_check": True}, 503),
|
||||
({"id": "image-1", "mode": "image_generation"}, 200),
|
||||
],
|
||||
)
|
||||
async def test_skip_disabled_setting_keeps_the_503_for_an_opted_out_media_model(model_info, expected_status):
|
||||
from fastapi import Response
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.health_endpoints._health_endpoints import health_endpoint
|
||||
|
||||
model_list = [
|
||||
{"model_name": "image", "litellm_params": {"model": "gemini/gemini-3-pro-image"}, "model_info": model_info}
|
||||
]
|
||||
response = Response()
|
||||
with (
|
||||
_proxy_health_globals(
|
||||
model_list,
|
||||
litellm.Router(model_list=copy.deepcopy(model_list)),
|
||||
use_background_health_checks=True,
|
||||
health_check_results={"healthy_endpoints": [], "unhealthy_endpoints": []},
|
||||
),
|
||||
patch( # test-quality-ok: proxy module global, no injection seam
|
||||
"litellm.proxy.proxy_server.general_settings",
|
||||
{"health_check_skip_disabled_background_models": True},
|
||||
),
|
||||
):
|
||||
await health_endpoint(
|
||||
response=response,
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token"),
|
||||
model=None,
|
||||
model_id="image-1",
|
||||
)
|
||||
|
||||
assert response.status_code == expected_status
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_endpoint_filters_background_cache_by_user_access():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -740,3 +740,220 @@ async def test_run_background_health_check_probes_only_listed_model_groups(monke
|
|||
await _run_background_health_check()
|
||||
|
||||
assert probed["ids"] == ["listed-1", "listed-2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"listed_group, expected_reason",
|
||||
[
|
||||
("image-group", "matched only deployments the background loop skips"),
|
||||
("no-such-group", "matched no deployments"),
|
||||
],
|
||||
)
|
||||
async def test_run_background_health_check_names_why_listed_model_groups_probe_nothing(
|
||||
monkeypatch, listed_group, expected_reason
|
||||
):
|
||||
monkeypatch.setattr(proxy_server, "health_check_interval", 60)
|
||||
monkeypatch.setattr(proxy_server, "health_check_concurrency", 1)
|
||||
monkeypatch.setattr(proxy_server, "health_check_details", True)
|
||||
monkeypatch.setattr(proxy_server, "use_shared_health_check", False)
|
||||
monkeypatch.setattr(proxy_server, "redis_usage_cache", None)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", None)
|
||||
monkeypatch.setattr(proxy_server, "background_health_check_loop_active", False)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
SimpleNamespace(
|
||||
settings=lambda: RouterSettings(background_health_check_model_groups=frozenset({listed_group}))
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_model_list",
|
||||
[
|
||||
{
|
||||
"model_name": "image-group",
|
||||
"litellm_params": {"model": "gemini/gemini-3-pro-image"},
|
||||
"model_info": {"id": "image", "mode": "image_generation"},
|
||||
},
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"health_check_results",
|
||||
{"healthy_endpoints": [], "unhealthy_endpoints": []},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"_run_direct_health_check_with_instrumentation",
|
||||
AsyncMock(return_value=([], [], {})),
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(proxy_server, "health_check_filter_kwargs_from_general_settings", lambda _gs: {})
|
||||
warn = MagicMock()
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn)
|
||||
|
||||
async def _stop_sleep(_seconds):
|
||||
raise asyncio.CancelledError()
|
||||
|
||||
monkeypatch.setattr(proxy_server.asyncio, "sleep", _stop_sleep)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await _run_background_health_check()
|
||||
|
||||
assert [
|
||||
call.args[1:] for call in warn.call_args_list if call.args[0].startswith("background_health_check_model_groups")
|
||||
] == [(expected_reason, [listed_group])]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_background_health_check_names_skipped_media_models_once_per_change(monkeypatch):
|
||||
monkeypatch.setattr(proxy_server, "health_check_interval", 60)
|
||||
monkeypatch.setattr(proxy_server, "health_check_concurrency", 1)
|
||||
monkeypatch.setattr(proxy_server, "health_check_details", True)
|
||||
monkeypatch.setattr(proxy_server, "use_shared_health_check", False)
|
||||
monkeypatch.setattr(proxy_server, "redis_usage_cache", None)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", None)
|
||||
monkeypatch.setattr(proxy_server, "background_health_check_loop_active", False)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
SimpleNamespace(
|
||||
settings=lambda: RouterSettings(background_health_check_model_groups=frozenset({"image", "chat"}))
|
||||
),
|
||||
)
|
||||
|
||||
def image(model_id):
|
||||
return {
|
||||
"model_name": "image",
|
||||
"litellm_params": {"model": "gemini/gemini-3-pro-image"},
|
||||
"model_info": {"id": model_id, "mode": "image_generation"},
|
||||
}
|
||||
|
||||
chat = {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat-1"}}
|
||||
monkeypatch.setattr(proxy_server, "llm_model_list", [image("image-1"), chat])
|
||||
monkeypatch.setattr(proxy_server, "health_check_results", {"healthy_endpoints": [], "unhealthy_endpoints": []})
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"_run_direct_health_check_with_instrumentation",
|
||||
AsyncMock(return_value=([], [], {})),
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(proxy_server, "health_check_filter_kwargs_from_general_settings", lambda _gs: {})
|
||||
warn = MagicMock()
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn)
|
||||
sleeps = iter((None, [image("image-1"), image("image-2"), chat], None))
|
||||
|
||||
async def _next_cycle(_seconds):
|
||||
step = next(sleeps, asyncio.CancelledError())
|
||||
if isinstance(step, BaseException):
|
||||
raise step
|
||||
if step is not None:
|
||||
monkeypatch.setattr(proxy_server, "llm_model_list", step)
|
||||
|
||||
monkeypatch.setattr(proxy_server.asyncio, "sleep", _next_cycle)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await _run_background_health_check()
|
||||
|
||||
assert [call.args[1:] for call in warn.call_args_list] == [(("image-1",),), (("image-1", "image-2"),)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_background_health_check_skips_billed_media_generation_unless_opted_in(monkeypatch):
|
||||
monkeypatch.setattr(proxy_server, "health_check_interval", 60)
|
||||
monkeypatch.setattr(proxy_server, "health_check_concurrency", 1)
|
||||
monkeypatch.setattr(proxy_server, "health_check_details", True)
|
||||
monkeypatch.setattr(proxy_server, "use_shared_health_check", False)
|
||||
monkeypatch.setattr(proxy_server, "redis_usage_cache", None)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", None)
|
||||
monkeypatch.setattr(proxy_server, "background_health_check_loop_active", False)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", None)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_model_list",
|
||||
[
|
||||
{"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat"}},
|
||||
{
|
||||
"model_name": "image",
|
||||
"litellm_params": {"model": "gemini/gemini-3-pro-image"},
|
||||
"model_info": {"id": "image", "mode": "image_generation"},
|
||||
},
|
||||
{
|
||||
"model_name": "image-edit",
|
||||
"litellm_params": {"model": "openai/gpt-image-1"},
|
||||
"model_info": {"id": "image-edit", "mode": "image_edit"},
|
||||
},
|
||||
{
|
||||
"model_name": "video",
|
||||
"litellm_params": {"model": "openai/sora-2"},
|
||||
"model_info": {"id": "video", "mode": "video_generation"},
|
||||
},
|
||||
{
|
||||
"model_name": "speech",
|
||||
"litellm_params": {"model": "openai/tts-1"},
|
||||
"model_info": {"id": "speech", "mode": "audio_speech"},
|
||||
},
|
||||
{
|
||||
"model_name": "embedding",
|
||||
"litellm_params": {"model": "openai/text-embedding-3-small"},
|
||||
"model_info": {"id": "embedding", "mode": "embedding"},
|
||||
},
|
||||
{"model_name": "fine-tune", "litellm_params": {"model": "openai/my-fine-tune"}, "model_info": {"id": "fine-tune"}},
|
||||
{
|
||||
"model_name": "bad-max-tokens",
|
||||
"litellm_params": {"model": "openai/my-fine-tune"},
|
||||
"model_info": {"id": "bad-max-tokens", "health_check_max_tokens": "abc"},
|
||||
},
|
||||
{
|
||||
"model_name": "azure-ai-image",
|
||||
"litellm_params": {"model": "azure_ai/gpt-image-1", "api_base": "https://example.services.ai.azure.com"},
|
||||
"model_info": {"id": "azure-ai-image"},
|
||||
},
|
||||
{
|
||||
"model_name": "image-opted-in",
|
||||
"litellm_params": {"model": "gemini/gemini-3-pro-image"},
|
||||
"model_info": {
|
||||
"id": "image-opted-in",
|
||||
"mode": "image_generation",
|
||||
"disable_background_health_check": False,
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"health_check_results",
|
||||
{"healthy_endpoints": [], "unhealthy_endpoints": []},
|
||||
)
|
||||
|
||||
probed = {}
|
||||
|
||||
async def _fake_direct(model_list, *_a, **_kw):
|
||||
probed["ids"] = [m["model_info"]["id"] for m in model_list]
|
||||
return ([], [], {})
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"_run_direct_health_check_with_instrumentation",
|
||||
_fake_direct,
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"health_check_filter_kwargs_from_general_settings",
|
||||
lambda _gs: {},
|
||||
)
|
||||
|
||||
async def _stop_sleep(_seconds):
|
||||
raise asyncio.CancelledError()
|
||||
|
||||
monkeypatch.setattr(proxy_server.asyncio, "sleep", _stop_sleep)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await _run_background_health_check()
|
||||
|
||||
assert probed["ids"] == ["chat", "speech", "embedding", "fine-tune", "bad-max-tokens", "image-opted-in"]
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import pytest
|
|||
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.db.health_check_latest import LatestHealthCheckRow
|
||||
from litellm.proxy.health_check import BILLED_MEDIA_NOT_CHECKED_WARNING
|
||||
from litellm.proxy.health_endpoints._health_endpoints import (
|
||||
_aggregate_health_check_results,
|
||||
_build_model_param_to_info_mapping,
|
||||
|
|
@ -666,6 +667,66 @@ async def test_save_background_health_checks_compares_raw_checked_at_against_utc
|
|||
assert [call.kwargs["model_id"] for call in mock_prisma.save_health_check_result.await_args_list] == ["stale-id"]
|
||||
|
||||
|
||||
def _saved_row(model_id: str, status: str, error_message: str | None) -> LatestHealthCheckRow:
|
||||
return LatestHealthCheckRow(
|
||||
health_check_id=f"hc-{model_id}",
|
||||
model_name=model_id,
|
||||
model_id=model_id,
|
||||
status=status,
|
||||
healthy_count=1 if status == "healthy" else 0,
|
||||
unhealthy_count=0 if status == "healthy" else 1,
|
||||
error_message=error_message,
|
||||
response_time_ms=12.5,
|
||||
details=None,
|
||||
checked_by="pod-1",
|
||||
checked_at=datetime(2026, 8, 25),
|
||||
created_at=datetime(2026, 8, 25, tzinfo=timezone.utc),
|
||||
updated_at=datetime(2026, 8, 25, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_latest_health_checks_stops_repeating_an_old_verdict_for_a_skipped_media_model(monkeypatch):
|
||||
prisma = MagicMock()
|
||||
prisma.get_all_latest_health_checks = AsyncMock(
|
||||
return_value=(
|
||||
_saved_row("image-was-healthy", "healthy", None),
|
||||
_saved_row("image-was-dead", "unhealthy", "401 from the provider"),
|
||||
_saved_row("image-opted-in", "unhealthy", "401 from the provider"),
|
||||
_saved_row("chat", "unhealthy", "401 from the provider"),
|
||||
)
|
||||
)
|
||||
image = {"litellm_params": {"model": "gemini/gemini-3-pro-image"}}
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.proxy_server.llm_model_list",
|
||||
[
|
||||
{**image, "model_info": {"id": "image-was-healthy", "mode": "image_generation"}},
|
||||
{**image, "model_info": {"id": "image-was-dead", "mode": "image_generation"}},
|
||||
{
|
||||
**image,
|
||||
"model_info": {
|
||||
"id": "image-opted-in",
|
||||
"mode": "image_generation",
|
||||
"disable_background_health_check": False,
|
||||
},
|
||||
},
|
||||
{"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "chat"}},
|
||||
],
|
||||
)
|
||||
|
||||
response = await latest_health_checks_endpoint(user_api_key_dict=UserAPIKeyAuth())
|
||||
|
||||
rows = response["latest_health_checks"]
|
||||
assert {model_id: (row["status"], row["error_message"]) for model_id, row in rows.items()} == {
|
||||
"image-was-healthy": ("skipped", BILLED_MEDIA_NOT_CHECKED_WARNING),
|
||||
"image-was-dead": ("skipped", BILLED_MEDIA_NOT_CHECKED_WARNING),
|
||||
"image-opted-in": ("unhealthy", "401 from the provider"),
|
||||
"chat": ("unhealthy", "401 from the provider"),
|
||||
}
|
||||
assert rows["image-was-healthy"]["checked_at"] == "2026-08-25T00:00:00+00:00"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_latest_health_checks_endpoint_serialises_raw_rows(monkeypatch):
|
||||
row = LatestHealthCheckRow(
|
||||
|
|
|
|||
|
|
@ -1223,11 +1223,95 @@ def test_a_dependency_that_opted_out_of_health_checks_is_never_probed():
|
|||
)
|
||||
marker = [d for d in router.model_list if d["model_info"]["id"] == "router-1"]
|
||||
|
||||
eligible = hc_module._health_check_eligible(router.model_list, skip_disabled=True)
|
||||
probes = hc_module._dependency_deployments_to_probe(marker, eligible, router)
|
||||
probes = hc_module._dependency_deployments_to_probe(marker, router.model_list, router, skip_disabled=True)
|
||||
|
||||
assert probes == ()
|
||||
assert [d["model_info"]["id"] for d in eligible] == ["router-1"]
|
||||
assert [d["model_info"]["id"] for d in hc_module._dependency_deployments_to_probe(marker, router.model_list, router)] == [
|
||||
"dead-1"
|
||||
]
|
||||
|
||||
|
||||
def test_health_checks_skip_billed_media_by_the_mode_the_probe_would_run():
|
||||
"""Modes and prices as model_prices_and_context_window.json listed them on 2026-10-30: gemini/gemini-3-pro-image
|
||||
and gpt-image-1 are image_generation, openai/text-embedding-3-small is embedding, gemini/lyria-3-clip-preview and
|
||||
vertex_ai/lyria-002 bill per clip with no token price, and gemini/gemini-3.1-pro-preview also has a token price."""
|
||||
|
||||
def deployment(model_id: str, model: str, **model_info: object) -> dict[str, object]:
|
||||
return {
|
||||
"model_name": model_id,
|
||||
"litellm_params": {"model": model, "api_key": "sk-x", "api_base": "https://example.services.ai.azure.com"},
|
||||
"model_info": {"id": model_id, **model_info},
|
||||
}
|
||||
|
||||
billed = (
|
||||
deployment("cost-map-image", "gemini/gemini-3-pro-image"),
|
||||
deployment("azure-ai-image", "azure_ai/gpt-image-1"),
|
||||
deployment("wildcard-probing-an-image", "openai/*", health_check_model="openai/gpt-image-1"),
|
||||
deployment("clip-priced-chat", "gemini/lyria-3-clip-preview"),
|
||||
deployment("clip-priced-speech", "vertex_ai/lyria-002", mode="audio_speech"),
|
||||
deployment("image-with-bad-max-tokens", "openai/x", mode="image_generation", health_check_max_tokens="abc"),
|
||||
deployment("opted-out-image", "gemini/gemini-3-pro-image", disable_background_health_check=True),
|
||||
)
|
||||
probed = (
|
||||
deployment("chat", "openai/gpt-4o-mini"),
|
||||
deployment("embedding", "openai/text-embedding-3-small"),
|
||||
deployment("token-priced-chat-with-image-price", "gemini/gemini-3.1-pro-preview"),
|
||||
deployment("wildcard", "openai/*"),
|
||||
deployment("unknown", "openai/my-fine-tune"),
|
||||
deployment("no-provider-prefix", "my-fine-tune"),
|
||||
deployment("bad-max-tokens", "openai/my-fine-tune", health_check_max_tokens="abc"),
|
||||
deployment("image-opted-in", "gemini/gemini-3-pro-image", disable_background_health_check=False),
|
||||
)
|
||||
opted_out_chat = deployment("opted-out-chat", "openai/gpt-4o-mini", disable_background_health_check=True)
|
||||
|
||||
assert hc_module._health_check_eligible([*billed, *probed, opted_out_chat], skip_disabled=True) == probed
|
||||
assert hc_module._health_check_eligible([*billed, *probed, opted_out_chat], skip_disabled=False) == (
|
||||
*probed,
|
||||
opted_out_chat,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("litellm_params", "billed"),
|
||||
[
|
||||
({"model": "lyria-002", "custom_llm_provider": "vertex_ai"}, True),
|
||||
({"model": "lyria-3-clip-preview", "custom_llm_provider": "gemini"}, True),
|
||||
({"model": "gemini-3.1-pro-preview", "custom_llm_provider": "gemini"}, False),
|
||||
({"model": "my-fine-tune", "custom_llm_provider": "vertex_ai"}, False),
|
||||
],
|
||||
)
|
||||
def test_per_clip_price_is_found_when_the_provider_is_set_apart_from_the_model(
|
||||
litellm_params: dict[str, str], billed: bool
|
||||
):
|
||||
"""Prices as model_prices_and_context_window.json listed them on 2026-10-31: the Lyria entries bill per clip with
|
||||
no token price under their provider prefix alone, and gemini/gemini-3.1-pro-preview also has a token price."""
|
||||
deployment = {"litellm_params": litellm_params, "model_info": {"id": "clip", "mode": "audio_speech"}}
|
||||
|
||||
assert hc_module.bills_media_on_probe(deployment) is billed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_info",
|
||||
[
|
||||
{"health_check_max_tokens": float("inf")},
|
||||
{"health_check_max_tokens": "many"},
|
||||
{"health_check_model": 123},
|
||||
{1: "not a string key"},
|
||||
],
|
||||
)
|
||||
def test_one_bad_model_info_value_cannot_stop_the_health_checks(model_info: dict[object, object]):
|
||||
bad = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "bad", **model_info}}
|
||||
control = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "control"}}
|
||||
|
||||
assert hc_module.background_check_deployments([bad, control]) == (bad, control)
|
||||
assert hc_module._health_check_eligible([bad, control], skip_disabled=True) == (bad, control)
|
||||
assert hc_module._health_check_eligible([bad, control], skip_disabled=False) == (bad, control)
|
||||
assert hc_module.all_bill_media_on_probe([bad, control], {"bad"}, False) is False
|
||||
|
||||
|
||||
def test_a_failed_memory_reading_does_not_break_health_check_logging():
|
||||
with patch("resource.getrusage", side_effect=OSError("no rusage")): # test-quality-ok: stdlib call, no seam
|
||||
assert hc_module._rss_mb_for_log() == "unknown"
|
||||
|
||||
|
||||
def test_narrowing_by_an_id_that_matches_nothing_keeps_the_whole_list():
|
||||
|
|
|
|||
|
|
@ -105,6 +105,53 @@ describe("HealthCheckComponent", () => {
|
|||
expect(mockIndividualModelHealthCheckCall).not.toHaveBeenCalledWith("token", "gpt-4");
|
||||
});
|
||||
|
||||
it("shows a model the proxy declined to check as skipped, never healthy, even with an old saved result", async () => {
|
||||
const warning = "Not checked, because this health check would generate billed media.";
|
||||
const declinedResponse = {
|
||||
healthy_count: 0,
|
||||
unhealthy_count: 0,
|
||||
healthy_endpoints: [],
|
||||
unhealthy_endpoints: [],
|
||||
warnings: [warning],
|
||||
};
|
||||
mockIndividualModelHealthCheckCall.mockResolvedValue(declinedResponse);
|
||||
mockLatestHealthChecksCall.mockResolvedValue({
|
||||
latest_health_checks: { "image-1": { status: "healthy", checked_at: "2026-10-01T00:00:00Z" } },
|
||||
});
|
||||
await renderHealthCheck({ modelData: { data: [makeModel("image-1", "image")] }, allModelsOnProxy: ["image-1"] });
|
||||
|
||||
for (const runButton of ["run-health-check-btn", "run-health-checks"]) {
|
||||
await act(async () => {
|
||||
screen.getByTestId(runButton).click();
|
||||
});
|
||||
await act(async () => {
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
});
|
||||
|
||||
expect(screen.getByText("skipped")).toBeInTheDocument();
|
||||
expect(screen.queryByText("healthy")).not.toBeInTheDocument();
|
||||
expect(screen.getByText(warning)).toBeInTheDocument();
|
||||
expect(screen.getByText(warning)).not.toHaveClass("text-destructive");
|
||||
}
|
||||
});
|
||||
|
||||
it("shows the skip reason on load when the proxy marks a saved result as skipped", async () => {
|
||||
const warning = "Not checked, because this health check would generate media the provider may bill.";
|
||||
mockLatestHealthChecksCall.mockResolvedValue({
|
||||
latest_health_checks: {
|
||||
"image-1": { status: "skipped", checked_at: "2026-10-01T00:00:00Z", error_message: warning },
|
||||
},
|
||||
});
|
||||
await renderHealthCheck({ modelData: { data: [makeModel("image-1", "image")] }, allModelsOnProxy: ["image-1"] });
|
||||
await act(async () => {
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
});
|
||||
|
||||
expect(screen.getByText("skipped")).toBeInTheDocument();
|
||||
expect(screen.getByText(warning)).toBeInTheDocument();
|
||||
expect(screen.getByText(warning)).not.toHaveClass("text-destructive");
|
||||
});
|
||||
|
||||
it("should page through results with the shared pagination footer", async () => {
|
||||
const onPageIndexChange = vi.fn();
|
||||
await renderHealthCheck({
|
||||
|
|
|
|||
|
|
@ -20,6 +20,22 @@ import { individualModelHealthCheckCall, latestHealthChecksCall } from "../netwo
|
|||
import { HealthChecksTable } from "./HealthChecksTable";
|
||||
import type { HealthCheckData, HealthStatus } from "./HealthChecksTableColumns";
|
||||
|
||||
const isSkippedResponse = (response: {
|
||||
healthy_count: number;
|
||||
unhealthy_count: number;
|
||||
warnings?: string[];
|
||||
}): response is { healthy_count: 0; unhealthy_count: 0; warnings: string[] } =>
|
||||
response.healthy_count === 0 && response.unhealthy_count === 0 && Boolean(response.warnings?.length);
|
||||
|
||||
const skippedStatus = (warning: string, lastCheck: string, previous?: HealthStatus): HealthStatus => ({
|
||||
status: "skipped",
|
||||
lastCheck,
|
||||
lastSuccess: previous?.lastSuccess || "None",
|
||||
loading: false,
|
||||
error: warning,
|
||||
fullError: warning,
|
||||
});
|
||||
|
||||
interface LatestHealthCheck {
|
||||
status?: string;
|
||||
checked_at?: string | null;
|
||||
|
|
@ -227,6 +243,11 @@ const HealthCheckComponent: React.FC<HealthCheckComponentProps> = ({
|
|||
|
||||
const fullError = checkData.error_message || undefined;
|
||||
|
||||
if (checkData.status === "skipped" && fullError) {
|
||||
healthStatusMap[modelId] = skippedStatus(fullError, toCheckedAtLabel(checkData.checked_at, "None"));
|
||||
return;
|
||||
}
|
||||
|
||||
healthStatusMap[modelId] = {
|
||||
status: checkData.status || "unknown",
|
||||
lastCheck: toCheckedAtLabel(checkData.checked_at, "None"),
|
||||
|
|
@ -265,6 +286,14 @@ const HealthCheckComponent: React.FC<HealthCheckComponentProps> = ({
|
|||
const response = await individualModelHealthCheckCall(accessToken, modelId);
|
||||
const currentTime = new Date().toLocaleString();
|
||||
|
||||
if (isSkippedResponse(response)) {
|
||||
setModelHealthStatuses((prev) => ({
|
||||
...prev,
|
||||
[modelId]: skippedStatus(response.warnings[0], currentTime, prev[modelId]),
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.unhealthy_count > 0 && response.unhealthy_endpoints && response.unhealthy_endpoints.length > 0) {
|
||||
const rawError =
|
||||
response.unhealthy_endpoints[0]?.error || i18n.t("modelDashboard.healthCheckComponent.healthCheckFailed");
|
||||
|
|
@ -362,6 +391,14 @@ const HealthCheckComponent: React.FC<HealthCheckComponentProps> = ({
|
|||
const response = await individualModelHealthCheckCall(accessToken, modelId);
|
||||
|
||||
const currentTime = new Date().toLocaleString();
|
||||
if (isSkippedResponse(response)) {
|
||||
setModelHealthStatuses((prev) => ({
|
||||
...prev,
|
||||
[modelId]: skippedStatus(response.warnings[0], currentTime, prev[modelId]),
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.unhealthy_count > 0 && response.unhealthy_endpoints && response.unhealthy_endpoints.length > 0) {
|
||||
const rawError =
|
||||
response.unhealthy_endpoints[0]?.error || t("modelDashboard.healthCheckComponent.healthCheckFailed");
|
||||
|
|
@ -422,6 +459,9 @@ const HealthCheckComponent: React.FC<HealthCheckComponentProps> = ({
|
|||
|
||||
setModelHealthStatuses((prev) => {
|
||||
const currentStatus = prev[modelId];
|
||||
if (currentStatus?.status === "skipped") {
|
||||
return prev;
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[modelId]: {
|
||||
|
|
|
|||
|
|
@ -43,10 +43,11 @@ const HEALTH_STATUS_TONES: Record<string, StatusTone> = {
|
|||
unhealthy: "error",
|
||||
checking: "info",
|
||||
none: "neutral",
|
||||
skipped: "neutral",
|
||||
};
|
||||
|
||||
// healthy > checking > unknown > unhealthy, matching the legacy health table ordering.
|
||||
const HEALTH_STATUS_ORDER: Record<string, number> = { healthy: 0, checking: 1, unknown: 2, unhealthy: 3 };
|
||||
const HEALTH_STATUS_ORDER: Record<string, number> = { healthy: 0, checking: 1, unknown: 2, skipped: 2, unhealthy: 3 };
|
||||
|
||||
const NEVER_CHECKED = "Never checked";
|
||||
const CHECK_IN_PROGRESS = "Check in progress...";
|
||||
|
|
@ -384,6 +385,15 @@ export const getHealthChecksTableColumns = (
|
|||
}
|
||||
|
||||
const cleanedError = healthStatus.error;
|
||||
|
||||
if (healthStatus.status === "skipped") {
|
||||
return (
|
||||
<span className="block max-w-50 truncate text-sm text-muted-foreground" title={cleanedError}>
|
||||
{cleanedError}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
const fullError = healthStatus.fullError || healthStatus.error;
|
||||
const displayName = getDisplayModelName(model) || model.model_name;
|
||||
|
||||
|
|
|
|||
9
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
9
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -7702,8 +7702,11 @@ export interface paths {
|
|||
* ```
|
||||
* else, the health checks will be run on models when /health is called.
|
||||
*
|
||||
* To skip deployments that set ``model_info.disable_background_health_check: true``
|
||||
* on ``GET /health`` as well as in the background loop, set
|
||||
* Deployments whose check would generate billed media (image_generation, image_edit, video_generation,
|
||||
* and models priced per generated clip) are not checked here or by the background loop unless they set
|
||||
* ``model_info.disable_background_health_check: false``. A ``model`` or ``model_id`` that targets only
|
||||
* such deployments answers 200 with zero counts and a ``warnings`` entry. To also skip deployments that
|
||||
* set ``disable_background_health_check: true`` on ``GET /health``, set
|
||||
* ``general_settings.health_check_skip_disabled_background_models: true``.
|
||||
*/
|
||||
get: operations["health_endpoint_health_get"];
|
||||
|
|
@ -30818,7 +30821,7 @@ export interface components {
|
|||
apply_user_budget_to_team_keys?: boolean | null;
|
||||
/**
|
||||
* Background Health Check Model Groups
|
||||
* @description Opt-in allowlist of model group names for background health checks and health-check routing. When set, the background loop probes only deployments whose model_name is listed, and enable_health_check_routing filters unhealthy deployments only within the listed groups; every other group, including newly added deployments, is skipped and keeps its configured routing strategy. When unset, all deployments participate (opt out per deployment via model_info.disable_background_health_check).
|
||||
* @description Opt-in allowlist of model group names for background health checks and health-check routing. When set, the background loop probes only deployments whose model_name is listed, and enable_health_check_routing filters unhealthy deployments only within the listed groups; every other group, including newly added deployments, is skipped and keeps its configured routing strategy. When unset, all deployments participate (opt out per deployment via model_info.disable_background_health_check). Set or unset, deployments whose check would generate billed media take part only with model_info.disable_background_health_check: false.
|
||||
*/
|
||||
background_health_check_model_groups?: string[] | null;
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue