mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(health): keep bad values from stopping checks and restore the skip flag 503
A bad model_info value raised inside the skip rule and ended the background loop, so the rule now fails open and the deployment fails its own check as it does on the base. A media deployment with disable_background_health_check true keeps its 503 when health_check_skip_disabled_background_models is on. The per-clip price lookup also reads the provider set apart from the model, and the dashboard shows the skip message as neutral text
This commit is contained in:
parent
a6ec0be5df
commit
fd1d1c9011
6 changed files with 131 additions and 18 deletions
|
|
@ -197,7 +197,7 @@ def _get_process_rss_mb() -> float | None:
|
|||
if sys.platform == "darwin":
|
||||
return float(ru_maxrss) / (1024 * 1024)
|
||||
return float(ru_maxrss) / 1024
|
||||
except Exception:
|
||||
except (ImportError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -346,47 +346,70 @@ def _fallback_probe_mode(probe_params: Mapping[str, object]) -> str | None:
|
|||
)
|
||||
|
||||
|
||||
def _priced_per_generated_clip(model: object) -> bool:
|
||||
entry: Final = cost_map_entry(model) if isinstance(model, str) else MappingProxyType({})
|
||||
def _bills_per_generated_clip(entry: Mapping[str, object]) -> bool:
|
||||
return bool(entry.get("output_cost_per_image")) and not entry.get("output_cost_per_token")
|
||||
|
||||
|
||||
def _priced_per_generated_clip(probe_params: Mapping[str, object]) -> bool:
|
||||
model: Final = probe_params.get("model")
|
||||
provider: Final = probe_params.get("custom_llm_provider")
|
||||
if not isinstance(model, str):
|
||||
return False
|
||||
names: Final = (model, f"{provider}/{model}") if isinstance(provider, str) else (model,)
|
||||
return any(_bills_per_generated_clip(cost_map_entry(name)) for name in names)
|
||||
|
||||
|
||||
def _probe_generates_billed_media(model_info: Mapping[str, object], litellm_params: Mapping[str, object]) -> bool:
|
||||
mode: Final = resolve_health_check_mode(model_info, litellm_params)
|
||||
if mode in _BILLED_MEDIA_GENERATION_MODES:
|
||||
return True
|
||||
if not isinstance(litellm_params.get("model"), str):
|
||||
return False
|
||||
try:
|
||||
probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python(
|
||||
update_litellm_params_for_health_check(dict(model_info), dict(litellm_params))
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python(
|
||||
update_litellm_params_for_health_check(dict(model_info), dict(litellm_params))
|
||||
)
|
||||
return (mode is None and _fallback_probe_mode(probe_params) in _BILLED_MEDIA_GENERATION_MODES) or (
|
||||
_priced_per_generated_clip(probe_params.get("model"))
|
||||
_priced_per_generated_clip(probe_params)
|
||||
)
|
||||
|
||||
|
||||
def _disable_background_flag(deployment: Mapping[str, object]) -> object:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
return model_info.get("disable_background_health_check") if isinstance(model_info, Mapping) else None
|
||||
|
||||
|
||||
def bills_media_on_probe(deployment: Mapping[str, object]) -> bool:
|
||||
model_info: Final = _str_key_mapping(deployment.get("model_info"))
|
||||
explicit: Final = model_info.get("disable_background_health_check")
|
||||
explicit: Final = _disable_background_flag(deployment)
|
||||
if explicit is not None and not explicit:
|
||||
return False
|
||||
return _probe_generates_billed_media(model_info, _str_key_mapping(deployment.get("litellm_params")))
|
||||
try:
|
||||
return _probe_generates_billed_media(
|
||||
_str_key_mapping(deployment.get("model_info")), _str_key_mapping(deployment.get("litellm_params"))
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Could not work out the health check for %s", _deployment_id(deployment))
|
||||
return False
|
||||
|
||||
|
||||
def skips_background_health_check(deployment: Mapping[str, object]) -> bool:
|
||||
disabled: Final = _str_key_mapping(deployment.get("model_info")).get("disable_background_health_check")
|
||||
return bool(disabled) or bills_media_on_probe(deployment)
|
||||
return bool(_disable_background_flag(deployment)) or bills_media_on_probe(deployment)
|
||||
|
||||
|
||||
def background_check_deployments(model_list: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(deployment for deployment in model_list if not skips_background_health_check(deployment))
|
||||
|
||||
|
||||
def all_bill_media_on_probe(deployments: Sequence[Mapping[str, object]], ids: AbstractSet[object]) -> bool:
|
||||
targets: Final = tuple(d for d in deployments if _deployment_id(d) in ids)
|
||||
def all_bill_media_on_probe(
|
||||
deployments: Sequence[Mapping[str, object]],
|
||||
ids: AbstractSet[object],
|
||||
health_check_skip_disabled_background_models: bool,
|
||||
) -> bool:
|
||||
targets: Final = tuple(
|
||||
d
|
||||
for d in deployments
|
||||
if _deployment_id(d) in ids
|
||||
and not (health_check_skip_disabled_background_models and _disable_background_flag(d))
|
||||
)
|
||||
return bool(targets) and all(bills_media_on_probe(d) for d in targets)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1694,7 +1694,9 @@ async def health_endpoint(
|
|||
},
|
||||
)
|
||||
if targeted_ids is not None and all_bill_media_on_probe(
|
||||
DEPLOYMENTS_ADAPTER.validate_python(_llm_model_list), MODEL_IDS_ADAPTER.validate_python(targeted_ids)
|
||||
DEPLOYMENTS_ADAPTER.validate_python(_llm_model_list),
|
||||
MODEL_IDS_ADAPTER.validate_python(targeted_ids),
|
||||
**_hc_filter,
|
||||
):
|
||||
return {
|
||||
"healthy_endpoints": [],
|
||||
|
|
|
|||
|
|
@ -2721,6 +2721,46 @@ async def test_targeted_health_answers_without_probing_billed_media(use_backgrou
|
|||
assert "warnings" not in results[checked]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("model_info", "expected_status"),
|
||||
[
|
||||
({"id": "image-1", "mode": "image_generation", "disable_background_health_check": True}, 503),
|
||||
({"id": "image-1", "mode": "image_generation"}, 200),
|
||||
],
|
||||
)
|
||||
async def test_skip_disabled_setting_keeps_the_503_for_an_opted_out_media_model(model_info, expected_status):
|
||||
from fastapi import Response
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.health_endpoints._health_endpoints import health_endpoint
|
||||
|
||||
model_list = [
|
||||
{"model_name": "image", "litellm_params": {"model": "gemini/gemini-3-pro-image"}, "model_info": model_info}
|
||||
]
|
||||
response = Response()
|
||||
with (
|
||||
_proxy_health_globals(
|
||||
model_list,
|
||||
litellm.Router(model_list=copy.deepcopy(model_list)),
|
||||
use_background_health_checks=True,
|
||||
health_check_results={"healthy_endpoints": [], "unhealthy_endpoints": []},
|
||||
),
|
||||
patch( # test-quality-ok: proxy module global, no injection seam
|
||||
"litellm.proxy.proxy_server.general_settings",
|
||||
{"health_check_skip_disabled_background_models": True},
|
||||
),
|
||||
):
|
||||
await health_endpoint(
|
||||
response=response,
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token"),
|
||||
model=None,
|
||||
model_id="image-1",
|
||||
)
|
||||
|
||||
assert response.status_code == expected_status
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_endpoint_filters_background_cache_by_user_access():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1271,6 +1271,44 @@ def test_health_checks_skip_billed_media_by_the_mode_the_probe_would_run():
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("litellm_params", "billed"),
|
||||
[
|
||||
({"model": "lyria-002", "custom_llm_provider": "vertex_ai"}, True),
|
||||
({"model": "lyria-3-clip-preview", "custom_llm_provider": "gemini"}, True),
|
||||
({"model": "gemini-3.1-pro-preview", "custom_llm_provider": "gemini"}, False),
|
||||
({"model": "my-fine-tune", "custom_llm_provider": "vertex_ai"}, False),
|
||||
],
|
||||
)
|
||||
def test_per_clip_price_is_found_when_the_provider_is_set_apart_from_the_model(
|
||||
litellm_params: dict[str, str], billed: bool
|
||||
):
|
||||
"""Prices as model_prices_and_context_window.json listed them on 2026-10-31: the Lyria entries bill per clip with
|
||||
no token price under their provider prefix alone, and gemini/gemini-3.1-pro-preview also has a token price."""
|
||||
deployment = {"litellm_params": litellm_params, "model_info": {"id": "clip", "mode": "audio_speech"}}
|
||||
|
||||
assert hc_module.bills_media_on_probe(deployment) is billed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_info",
|
||||
[
|
||||
{"health_check_max_tokens": float("inf")},
|
||||
{"health_check_max_tokens": "many"},
|
||||
{"health_check_model": 123},
|
||||
{1: "not a string key"},
|
||||
],
|
||||
)
|
||||
def test_one_bad_model_info_value_cannot_stop_the_health_checks(model_info: dict[object, object]):
|
||||
bad = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "bad", **model_info}}
|
||||
control = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "control"}}
|
||||
|
||||
assert hc_module.background_check_deployments([bad, control]) == (bad, control)
|
||||
assert hc_module._health_check_eligible([bad, control], skip_disabled=True) == (bad, control)
|
||||
assert hc_module._health_check_eligible([bad, control], skip_disabled=False) == (bad, control)
|
||||
assert hc_module.all_bill_media_on_probe([bad, control], {"bad"}, False) is False
|
||||
|
||||
|
||||
def test_narrowing_by_an_id_that_matches_nothing_keeps_the_whole_list():
|
||||
"""Pinned because the disabled-dependency fix moved this filter into its own helper."""
|
||||
deployments = [{"model_name": "a", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a-1"}}]
|
||||
|
|
|
|||
|
|
@ -131,6 +131,7 @@ describe("HealthCheckComponent", () => {
|
|||
expect(screen.getByText("skipped")).toBeInTheDocument();
|
||||
expect(screen.queryByText("healthy")).not.toBeInTheDocument();
|
||||
expect(screen.getByText(warning)).toBeInTheDocument();
|
||||
expect(screen.getByText(warning)).not.toHaveClass("text-destructive");
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -340,6 +340,15 @@ export const getHealthChecksTableColumns = ({
|
|||
}
|
||||
|
||||
const cleanedError = healthStatus.error;
|
||||
|
||||
if (healthStatus.status === "skipped") {
|
||||
return (
|
||||
<span className="block max-w-50 truncate text-sm text-muted-foreground" title={cleanedError}>
|
||||
{cleanedError}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
const fullError = healthStatus.fullError || healthStatus.error;
|
||||
const displayName = getDisplayModelName(model) || model.model_name;
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue