diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index e42b9159a35..6b51fea02c9 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -135,7 +135,7 @@ _AUDIO_OUTPUT_PROBE_PROVIDERS: Final[frozenset[str]] = frozenset({"openai", "azu def _health_check_requires_audio_output( - model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None + model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None, deployment_model: object ) -> bool: if (mode or "chat") != "chat": return False @@ -152,16 +152,16 @@ def _health_check_requires_audio_output( if provider not in _AUDIO_OUTPUT_PROBE_PROVIDERS: return False base_model: Final = model_info.get("base_model") or probe_params.get("base_model") - probes_own_model: Final = model_info.get("health_check_model") is None return litellm.supports_audio_output( - model=base_model if isinstance(base_model, str) and probes_own_model else model, custom_llm_provider=provider + model=base_model if isinstance(base_model, str) and model == deployment_model else model, + custom_llm_provider=provider, ) def _health_check_audio_output_params( - model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None + model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None, deployment_model: object ) -> Mapping[str, object]: - if not _health_check_requires_audio_output(model_info, probe_params, mode): + if not _health_check_requires_audio_output(model_info, probe_params, mode, deployment_model): return MappingProxyType({}) voice: Final = model_info.get("health_check_voice") return MappingProxyType( @@ -826,7 +826,9 @@ _PROBE_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) -def update_litellm_params_for_health_check(model_info: dict, litellm_params: dict) -> dict: +def update_litellm_params_for_health_check( + model_info: dict, litellm_params: dict, deployment_model: str | None = None +) -> dict: """ Update the litellm params for health check. @@ -842,11 +844,13 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic - updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes - updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models - for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID, and pins `custom_llm_provider` to `bedrock` (only when the deployment hasn't already set one, so an explicit `bedrock_converse` survives) so the bare model id still resolves to the provider (e.g. cross-region ids like `us.cohere.embed-v4:0`) - - asks OpenAI and Azure chat models that support audio output (`base_model` from `model_info`, then - `litellm_params`, first, unless `health_check_model` replaces the probed model) for `modalities` - text + audio and a wav `audio` in the `health_check_voice` (default "alloy"), since those models reject - text-only requests. Keys already set through `health_check_params` are kept as they are + - asks OpenAI and Azure chat models that support audio output for `modalities` text + audio and a wav + `audio` in the `health_check_voice` (default "alloy"), since those models reject text-only requests. + `base_model` (from `model_info`, then `litellm_params`) decides it only while the probe still targets + `deployment_model` (default: the incoming `litellm_params` model), so a `health_check_model` or a `model` + in `health_check_params` is judged on its own. Keys already set through `health_check_params` are kept """ + base_model_owner: Final = deployment_model or _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model") mode: Final = resolve_health_check_mode( model_info, litellm_params, # any-ok: untyped router config dict @@ -922,7 +926,7 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params) audio_output_params: Final = _health_check_audio_output_params( - _MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode + _MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode, base_model_owner ) return { key: value diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index a44c50ce44a..be5b94ca350 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -203,21 +203,13 @@ def _config_base_for_health_check( return {key: value for key, value in config_params.items() if key not in _CONFIG_CONNECTION_FIELDS} -_STORED_MODEL_INFO_KEYS: Final = frozenset(("mode", "base_model")) -_STORED_MODEL_PARAM_KEYS: Final = frozenset(("base_model",)) - - -def _request_swaps_the_model(stored_params: Mapping[str, object], request_params: Mapping[str, object]) -> bool: - stored_model: Final = stored_params.get("model") - return stored_model is not None and request_params.get("model") not in (None, stored_model) - - -def _without_stored_model_keys( - fields: Mapping[str, object], stored_model_keys: frozenset[str], *, request_swaps_the_model: bool +def _model_info_for_mode_resolution( + model_info: Mapping[str, object], stored_params: Mapping[str, object], request_params: Mapping[str, object] ) -> Mapping[str, object]: - if not request_swaps_the_model: - return fields - return {key: value for key, value in fields.items() if key not in stored_model_keys} + stored_model: Final = stored_params.get("model") + if stored_model is None or request_params.get("model") in (None, stored_model): + return model_info + return {key: value for key, value in model_info.items() if key != "mode"} def _string_mode_or_bad_request(params_mode: object) -> str | None: @@ -2314,33 +2306,27 @@ async def test_model_connection( selected_by_id: Final = deployment_by_id is not None reject_server_owned_wif_fields(request_litellm_params) # Merge: config params (from proxy config) as base, request params override - request_swaps_the_model: Final = _request_swaps_the_model( - _OBJECT_MAPPING.validate_python(config_litellm_params), - _OBJECT_MAPPING.validate_python(request_litellm_params), - ) litellm_params = { - **_without_stored_model_keys( - _config_base_for_health_check( - config_litellm_params, - request_litellm_params, - allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True, - selected_by_id=selected_by_id, - ), - _STORED_MODEL_PARAM_KEYS, - request_swaps_the_model=request_swaps_the_model, + **_config_base_for_health_check( + config_litellm_params, + request_litellm_params, + allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True, + selected_by_id=selected_by_id, ), **request_litellm_params, } resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info - probe_model_info: Final = _without_stored_model_keys( + probe_model_info: Final = _model_info_for_mode_resolution( _OBJECT_MAPPING.validate_python(resolved_model_info or {}), - _STORED_MODEL_INFO_KEYS, - request_swaps_the_model=request_swaps_the_model, + stored_params=_OBJECT_MAPPING.validate_python(config_litellm_params), + request_params=_OBJECT_MAPPING.validate_python(request_litellm_params), ) + stored_model: Final = _OBJECT_MAPPING.validate_python(config_litellm_params).get("model") litellm_params = update_litellm_params_for_health_check( model_info=dict(probe_model_info), litellm_params=litellm_params, + deployment_model=stored_model if isinstance(stored_model, str) else None, ) ## Auth check, on the final probe params so health_check_params cannot retarget it afterwards diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 9ac3185dcb2..1055de06e38 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -394,7 +394,7 @@ async def test_test_model_connection_loads_config_from_router(): mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result) # Mock _update_litellm_params_for_health_check - def mock_update_params(model_info, litellm_params): + def mock_update_params(model_info, litellm_params, deployment_model=None): # Just return params with messages added params = litellm_params.copy() params["messages"] = [{"role": "user", "content": "test"}] @@ -545,7 +545,7 @@ async def test_test_model_connection_uses_model_info_id_to_disambiguate_duplicat mock_ahealth_check = AsyncMock(return_value=mock_health_check_result) mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result) - def mock_update_params(model_info, litellm_params): + def mock_update_params(model_info, litellm_params, deployment_model=None): params = litellm_params.copy() params["messages"] = [{"role": "user", "content": "test"}] return params @@ -656,7 +656,7 @@ async def test_test_model_connection_falls_back_to_deployments_zero_without_id() mock_ahealth_check = AsyncMock(return_value=mock_health_check_result) mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result) - def mock_update_params(model_info, litellm_params): + def mock_update_params(model_info, litellm_params, deployment_model=None): params = litellm_params.copy() params["messages"] = [{"role": "user", "content": "test"}] return params @@ -938,7 +938,9 @@ async def test_test_model_connection_stored_audio_base_model_follows_the_stored_ user_api_key_dict=UserAPIKeyAuth(user_id="test-user", token="test-token"), ) - assert ("modalities" in ahealth_check.call_args.kwargs["model_params"]) is expects_audio_probe + probe_params: Final = ahealth_check.call_args.kwargs["model_params"] + assert ("modalities" in probe_params) is expects_audio_probe + assert probe_params.get("base_model") == deployment_params.get("base_model") @pytest.mark.asyncio diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index e6450f8f520..b6ce1893202 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -851,6 +851,10 @@ def test_health_check_params_apply_to_non_chat_modes(): {"base_model": "azure/gpt-4o-mini", "health_check_model": "azure/gpt-audio-mini"}, {"model": "azure/my-text-deployment"}, ), + ( + {"base_model": "gpt-4o-mini", "health_check_params": {"model": "openai/gpt-audio-mini"}}, + {"model": "openai/my-text-model"}, + ), ], ) def test_openai_and_azure_audio_chat_probe_asks_for_audio_output( @@ -879,6 +883,14 @@ def test_openai_and_azure_audio_chat_probe_asks_for_audio_output( {"health_check_model": "azure/gpt-4o-mini"}, {"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-mini"}, ), + ( + {"base_model": "gpt-audio-mini", "health_check_params": {"model": "openai/gpt-5.4-mini"}}, + {"model": "openai/gpt-audio-mini"}, + ), + ( + {"health_check_params": {"model": "openai/gpt-5.4-mini"}}, + {"model": "openai/gpt-audio-mini", "base_model": "gpt-audio-mini"}, + ), ], ) def test_probe_without_openai_audio_chat_model_asks_for_no_audio_output(