From 89b8f346d2eddad44e65f7fdf125bcf4a443a383 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:50:47 -0700 Subject: [PATCH] fix(health): read litellm_params.base_model, and drop the stored base_model when Test Connect swaps the model --- litellm/proxy/health_check.py | 6 +-- .../health_endpoints/_health_endpoints.py | 44 +++++++++++++------ .../health_endpoints/test_health_endpoints.py | 39 ++++++++++++++++ .../proxy/test_health_check_max_tokens.py | 5 +++ 4 files changed, 77 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index f4253cfde99..e42b9159a35 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -151,7 +151,7 @@ def _health_check_requires_audio_output( return False if provider not in _AUDIO_OUTPUT_PROBE_PROVIDERS: return False - base_model: Final = model_info.get("base_model") + base_model: Final = model_info.get("base_model") or probe_params.get("base_model") probes_own_model: Final = model_info.get("health_check_model") is None return litellm.supports_audio_output( model=base_model if isinstance(base_model, str) and probes_own_model else model, custom_llm_provider=provider @@ -842,8 +842,8 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic - updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes - updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models - for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID, and pins `custom_llm_provider` to `bedrock` (only when the deployment hasn't already set one, so an explicit `bedrock_converse` survives) so the bare model id still resolves to the provider (e.g. cross-region ids like `us.cohere.embed-v4:0`) - - asks OpenAI and Azure chat models that support audio output (`model_info.base_model` first, unless - `health_check_model` replaces the probed model) for `modalities` + - asks OpenAI and Azure chat models that support audio output (`base_model` from `model_info`, then + `litellm_params`, first, unless `health_check_model` replaces the probed model) for `modalities` text + audio and a wav `audio` in the `health_check_voice` (default "alloy"), since those models reject text-only requests. Keys already set through `health_check_params` are kept as they are """ diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 684f1619512..a44c50ce44a 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -203,13 +203,21 @@ def _config_base_for_health_check( return {key: value for key, value in config_params.items() if key not in _CONFIG_CONNECTION_FIELDS} -def _model_info_for_mode_resolution( - model_info: Mapping[str, object], stored_params: Mapping[str, object], request_params: Mapping[str, object] -) -> Mapping[str, object]: +_STORED_MODEL_INFO_KEYS: Final = frozenset(("mode", "base_model")) +_STORED_MODEL_PARAM_KEYS: Final = frozenset(("base_model",)) + + +def _request_swaps_the_model(stored_params: Mapping[str, object], request_params: Mapping[str, object]) -> bool: stored_model: Final = stored_params.get("model") - if stored_model is None or request_params.get("model") in (None, stored_model): - return model_info - return {key: value for key, value in model_info.items() if key != "mode"} + return stored_model is not None and request_params.get("model") not in (None, stored_model) + + +def _without_stored_model_keys( + fields: Mapping[str, object], stored_model_keys: frozenset[str], *, request_swaps_the_model: bool +) -> Mapping[str, object]: + if not request_swaps_the_model: + return fields + return {key: value for key, value in fields.items() if key not in stored_model_keys} def _string_mode_or_bad_request(params_mode: object) -> str | None: @@ -2306,21 +2314,29 @@ async def test_model_connection( selected_by_id: Final = deployment_by_id is not None reject_server_owned_wif_fields(request_litellm_params) # Merge: config params (from proxy config) as base, request params override + request_swaps_the_model: Final = _request_swaps_the_model( + _OBJECT_MAPPING.validate_python(config_litellm_params), + _OBJECT_MAPPING.validate_python(request_litellm_params), + ) litellm_params = { - **_config_base_for_health_check( - config_litellm_params, - request_litellm_params, - allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True, - selected_by_id=selected_by_id, + **_without_stored_model_keys( + _config_base_for_health_check( + config_litellm_params, + request_litellm_params, + allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True, + selected_by_id=selected_by_id, + ), + _STORED_MODEL_PARAM_KEYS, + request_swaps_the_model=request_swaps_the_model, ), **request_litellm_params, } resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info - probe_model_info: Final = _model_info_for_mode_resolution( + probe_model_info: Final = _without_stored_model_keys( _OBJECT_MAPPING.validate_python(resolved_model_info or {}), - stored_params=_OBJECT_MAPPING.validate_python(config_litellm_params), - request_params=_OBJECT_MAPPING.validate_python(request_litellm_params), + _STORED_MODEL_INFO_KEYS, + request_swaps_the_model=request_swaps_the_model, ) litellm_params = update_litellm_params_for_health_check( model_info=dict(probe_model_info), diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 950d1742d49..9ac3185dcb2 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -902,6 +902,45 @@ async def test_test_model_connection_overridden_model_probe_params_follow_the_pr assert "max_tokens" not in ahealth_check.call_args.kwargs["model_params"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_params", "deployment_model_info"), + [ + ({"model": "azure/my-audio-deployment"}, {"id": "audio-id", "base_model": "azure/gpt-audio-1.5"}), + ({"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-1.5"}, {"id": "audio-id"}), + ], + ids=["model_info.base_model", "litellm_params.base_model"], +) +@pytest.mark.parametrize( + ("request_params", "expects_audio_probe"), + [({}, True), ({"model": "azure/my-text-deployment"}, False)], + ids=["own model", "swapped model"], +) +async def test_test_model_connection_stored_audio_base_model_follows_the_stored_model( + deployment_params: Mapping[str, str], + deployment_model_info: Mapping[str, str], + request_params: Mapping[str, str], + expects_audio_probe: bool, +): + deployment: Final = MappingProxyType( + { + "model_name": "custom-name-audio", + "litellm_params": {**deployment_params, "api_key": "fake-azure-key", "api_base": "https://x.openai.azure.com"}, + "model_info": deployment_model_info, + } + ) + with _test_connection_probe(deployment) as ahealth_check: + await health_test_model_connection( + request=MagicMock(), + mode=None, + litellm_params=dict(request_params), + model_info={"id": "audio-id"}, + user_api_key_dict=UserAPIKeyAuth(user_id="test-user", token="test-token"), + ) + + assert ("modalities" in ahealth_check.call_args.kwargs["model_params"]) is expects_audio_probe + + @pytest.mark.asyncio @pytest.mark.parametrize("params_mode", [123, ["chat"], {"mode": "chat"}, False], ids=["int", "list", "dict", "bool"]) async def test_test_model_connection_non_string_params_mode_is_a_bad_request(params_mode: object): diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index 062f5b87e4c..e6450f8f520 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -846,6 +846,7 @@ def test_health_check_params_apply_to_non_chat_modes(): ({}, {"model": "azure/gpt-audio-mini"}), ({}, {"model": "openai/gpt-audio-mini"}), ({"base_model": "azure/gpt-audio-mini"}, {"model": "azure/my-audio-deployment"}), + ({}, {"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-1.5"}), ( {"base_model": "azure/gpt-4o-mini", "health_check_model": "azure/gpt-audio-mini"}, {"model": "azure/my-text-deployment"}, @@ -874,6 +875,10 @@ def test_openai_and_azure_audio_chat_probe_asks_for_audio_output( {"base_model": "azure/gpt-audio-mini", "health_check_model": "azure/gpt-4o-mini"}, {"model": "azure/my-audio-deployment"}, ), + ( + {"health_check_model": "azure/gpt-4o-mini"}, + {"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-mini"}, + ), ], ) def test_probe_without_openai_audio_chat_model_asks_for_no_audio_output(