From 3f2027729ce65047b18c6cf716c534fd90bc5dca Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:27:27 -0700 Subject: [PATCH 1/4] fix(health): ask OpenAI and Azure audio chat models for audio output in the health check probe --- litellm/proxy/health_check.py | 53 ++++++++++++++++- .../proxy/test_health_check_max_tokens.py | 57 +++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 0d2f8dc9a0c..877279ec971 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -131,6 +131,46 @@ def _should_inject_health_check_max_tokens(model_info: Mapping[str, object], mod return (mode or "chat") in _MAX_TOKEN_SUPPORT_MODES +_AUDIO_OUTPUT_PROBE_PROVIDERS: Final[frozenset[str]] = frozenset({"openai", "azure"}) + + +def _health_check_requires_audio_output( + model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None +) -> bool: + if (mode or "chat") != "chat": + return False + model: Final = probe_params.get("model") + if not isinstance(model, str): + return False + provider_param: Final = probe_params.get("custom_llm_provider") + try: + _, provider, _, _ = litellm.get_llm_provider( + model=model, custom_llm_provider=provider_param if isinstance(provider_param, str) else None + ) + except litellm.exceptions.BadRequestError: + return False + if provider not in _AUDIO_OUTPUT_PROBE_PROVIDERS: + return False + base_model: Final = model_info.get("base_model") + return litellm.supports_audio_output( + model=base_model if isinstance(base_model, str) else model, custom_llm_provider=provider + ) + + +def _health_check_audio_output_params( + model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None +) -> Mapping[str, object]: + if not _health_check_requires_audio_output(model_info, probe_params, mode): + return MappingProxyType({}) + voice: Final = model_info.get("health_check_voice") + return MappingProxyType( + { + "modalities": ["text", "audio"], + "audio": {"voice": voice if isinstance(voice, str) else "alloy", "format": "wav"}, + } + ) + + # Health-check modes that forward `reasoning_effort` to the provider (chat-style calls). _HEALTH_CHECK_MODES_SUPPORTING_REASONING_EFFORT: Final = frozenset((None, "chat", "completion")) @@ -782,6 +822,7 @@ def _resolve_health_check_max_tokens(model_info: dict, litellm_params: dict) -> _PROBE_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) def update_litellm_params_for_health_check(model_info: dict, litellm_params: dict) -> dict: @@ -800,6 +841,9 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic - updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes - updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models - for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID, and pins `custom_llm_provider` to `bedrock` (only when the deployment hasn't already set one, so an explicit `bedrock_converse` survives) so the bare model id still resolves to the provider (e.g. cross-region ids like `us.cohere.embed-v4:0`) + - asks OpenAI and Azure chat models that support audio output (`model_info.base_model` first) for `modalities` + text + audio and a wav `audio` in the `health_check_voice` (default "alloy"), since those models reject + text-only requests. Keys already set through `health_check_params` are kept as they are """ mode: Final = resolve_health_check_mode( model_info, @@ -875,7 +919,14 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic ) probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params) - return {key: value for key, value in probe_params.items() if not is_model_capability_field(key)} + audio_output_params: Final = _health_check_audio_output_params( + _MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode + ) + return { + key: value + for key, value in {**audio_output_params, **probe_params}.items() + if not is_model_capability_field(key) + } _update_litellm_params_for_health_check: Final = update_litellm_params_for_health_check diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index 24117f45106..2e8a94c6cd5 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -840,6 +840,63 @@ def test_health_check_params_apply_to_non_chat_modes(): assert "max_tokens" not in updated +@pytest.mark.parametrize( + ("model_info", "litellm_params"), + [ + ({}, {"model": "azure/gpt-audio-mini"}), + ({}, {"model": "openai/gpt-audio-mini"}), + ({"base_model": "azure/gpt-audio-mini"}, {"model": "azure/my-audio-deployment"}), + ], +) +def test_openai_and_azure_audio_chat_probe_asks_for_audio_output(model_info, litellm_params): + updated: Final = update_litellm_params_for_health_check(model_info, litellm_params) + + assert updated["modalities"] == ["text", "audio"] + assert updated["audio"] == {"voice": "alloy", "format": "wav"} + assert updated["max_tokens"] == 16 + + +@pytest.mark.parametrize( + ("model_info", "litellm_params"), + [ + ({}, {"model": "azure/gpt-4o-mini"}), + ({}, {"model": "gemini/lyria-3-clip-preview"}), + ({}, {"model": "azure/my-audio-deployment"}), + ({}, {"model": "unmapped-model-name"}), + ({"mode": "responses"}, {"model": "openai/gpt-audio-mini"}), + ], +) +def test_probe_without_openai_audio_chat_model_asks_for_no_audio_output(model_info, litellm_params): + updated: Final = update_litellm_params_for_health_check(model_info, litellm_params) + + assert "modalities" not in updated + assert "audio" not in updated + + +@pytest.mark.parametrize( + "operator_params", + [ + {"modalities": ["audio", "text"], "audio": {"voice": "verse", "format": "mp3"}}, + {"audio": {"voice": "verse", "format": "mp3"}}, + ], +) +def test_audio_chat_probe_keeps_audio_params_set_in_health_check_params(operator_params): + updated: Final = update_litellm_params_for_health_check( + {"health_check_params": operator_params}, {"model": "azure/gpt-audio-mini"} + ) + + assert {key: updated[key] for key in operator_params} == operator_params + assert updated["modalities"] == operator_params.get("modalities", ["text", "audio"]) + + +def test_audio_chat_probe_uses_health_check_voice(): + updated: Final = update_litellm_params_for_health_check( + {"health_check_voice": "verse"}, {"model": "openai/gpt-audio-mini"} + ) + + assert updated["audio"] == {"voice": "verse", "format": "wav"} + + async def _pegasus_health_check_request_body( model_info: dict[str, object], monkeypatch: pytest.MonkeyPatch ) -> dict[str, object]: From 366b116574d81e5ecb6a596b2008fb88031c6715 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:33:35 -0700 Subject: [PATCH 2/4] fix(health): check the probed model when health_check_model is set, and mark Azure audio models as audio output A deployment with an audio base_model and a text health_check_model got the audio fields sent to its text probe model. The Azure gpt-audio and gpt-4o audio preview cost-map entries now set supports_audio_output, so a dated Azure name like azure/gpt-audio-1.5-2026-02-23 gets the audio probe too --- ...odel_prices_and_context_window_backup.json | 8 ++++ litellm/proxy/health_check.py | 6 ++- model_prices_and_context_window.json | 8 ++++ .../proxy/test_health_check_max_tokens.py | 39 +++++++++++++++++-- 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3d89af9d691..bc620899833 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6103,6 +6103,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6136,6 +6137,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6168,6 +6170,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6202,6 +6205,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6234,6 +6238,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6266,6 +6271,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6297,6 +6303,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6366,6 +6373,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 877279ec971..f4253cfde99 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -152,8 +152,9 @@ def _health_check_requires_audio_output( if provider not in _AUDIO_OUTPUT_PROBE_PROVIDERS: return False base_model: Final = model_info.get("base_model") + probes_own_model: Final = model_info.get("health_check_model") is None return litellm.supports_audio_output( - model=base_model if isinstance(base_model, str) else model, custom_llm_provider=provider + model=base_model if isinstance(base_model, str) and probes_own_model else model, custom_llm_provider=provider ) @@ -841,7 +842,8 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic - updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes - updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models - for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID, and pins `custom_llm_provider` to `bedrock` (only when the deployment hasn't already set one, so an explicit `bedrock_converse` survives) so the bare model id still resolves to the provider (e.g. cross-region ids like `us.cohere.embed-v4:0`) - - asks OpenAI and Azure chat models that support audio output (`model_info.base_model` first) for `modalities` + - asks OpenAI and Azure chat models that support audio output (`model_info.base_model` first, unless + `health_check_model` replaces the probed model) for `modalities` text + audio and a wav `audio` in the `health_check_voice` (default "alloy"), since those models reject text-only requests. Keys already set through `health_check_params` are kept as they are """ diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3d89af9d691..bc620899833 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6103,6 +6103,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6136,6 +6137,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6168,6 +6170,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6202,6 +6205,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6234,6 +6238,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6266,6 +6271,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6297,6 +6303,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6366,6 +6373,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index 2e8a94c6cd5..062f5b87e4c 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -846,9 +846,15 @@ def test_health_check_params_apply_to_non_chat_modes(): ({}, {"model": "azure/gpt-audio-mini"}), ({}, {"model": "openai/gpt-audio-mini"}), ({"base_model": "azure/gpt-audio-mini"}, {"model": "azure/my-audio-deployment"}), + ( + {"base_model": "azure/gpt-4o-mini", "health_check_model": "azure/gpt-audio-mini"}, + {"model": "azure/my-text-deployment"}, + ), ], ) -def test_openai_and_azure_audio_chat_probe_asks_for_audio_output(model_info, litellm_params): +def test_openai_and_azure_audio_chat_probe_asks_for_audio_output( + model_info: dict[str, str], litellm_params: dict[str, str] +): updated: Final = update_litellm_params_for_health_check(model_info, litellm_params) assert updated["modalities"] == ["text", "audio"] @@ -864,9 +870,15 @@ def test_openai_and_azure_audio_chat_probe_asks_for_audio_output(model_info, lit ({}, {"model": "azure/my-audio-deployment"}), ({}, {"model": "unmapped-model-name"}), ({"mode": "responses"}, {"model": "openai/gpt-audio-mini"}), + ( + {"base_model": "azure/gpt-audio-mini", "health_check_model": "azure/gpt-4o-mini"}, + {"model": "azure/my-audio-deployment"}, + ), ], ) -def test_probe_without_openai_audio_chat_model_asks_for_no_audio_output(model_info, litellm_params): +def test_probe_without_openai_audio_chat_model_asks_for_no_audio_output( + model_info: dict[str, str], litellm_params: dict[str, str] +): updated: Final = update_litellm_params_for_health_check(model_info, litellm_params) assert "modalities" not in updated @@ -880,7 +892,7 @@ def test_probe_without_openai_audio_chat_model_asks_for_no_audio_output(model_in {"audio": {"voice": "verse", "format": "mp3"}}, ], ) -def test_audio_chat_probe_keeps_audio_params_set_in_health_check_params(operator_params): +def test_audio_chat_probe_keeps_audio_params_set_in_health_check_params(operator_params: dict[str, object]): updated: Final = update_litellm_params_for_health_check( {"health_check_params": operator_params}, {"model": "azure/gpt-audio-mini"} ) @@ -889,6 +901,27 @@ def test_audio_chat_probe_keeps_audio_params_set_in_health_check_params(operator assert updated["modalities"] == operator_params.get("modalities", ["text", "audio"]) +_AUDIO_OUTPUT_CHAT_MODELS: Final = tuple( + sorted( + name + for name, info in litellm.model_cost.items() + if info.get("litellm_provider") in ("openai", "azure") + and info.get("mode") == "chat" + and "audio" in (info.get("supported_output_modalities") or []) + ) +) + + +@pytest.mark.parametrize("cost_map_model", _AUDIO_OUTPUT_CHAT_MODELS) +def test_every_audio_output_chat_model_in_the_cost_map_gets_the_audio_probe(cost_map_model: str): + provider: Final = litellm.model_cost[cost_map_model]["litellm_provider"] + updated: Final = update_litellm_params_for_health_check( + {"base_model": cost_map_model}, {"model": f"{provider}/my-audio-deployment"} + ) + + assert updated["modalities"] == ["text", "audio"] + + def test_audio_chat_probe_uses_health_check_voice(): updated: Final = update_litellm_params_for_health_check( {"health_check_voice": "verse"}, {"model": "openai/gpt-audio-mini"} From 89b8f346d2eddad44e65f7fdf125bcf4a443a383 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:50:47 -0700 Subject: [PATCH 3/4] fix(health): read litellm_params.base_model, and drop the stored base_model when Test Connect swaps the model --- litellm/proxy/health_check.py | 6 +-- .../health_endpoints/_health_endpoints.py | 44 +++++++++++++------ .../health_endpoints/test_health_endpoints.py | 39 ++++++++++++++++ .../proxy/test_health_check_max_tokens.py | 5 +++ 4 files changed, 77 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index f4253cfde99..e42b9159a35 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -151,7 +151,7 @@ def _health_check_requires_audio_output( return False if provider not in _AUDIO_OUTPUT_PROBE_PROVIDERS: return False - base_model: Final = model_info.get("base_model") + base_model: Final = model_info.get("base_model") or probe_params.get("base_model") probes_own_model: Final = model_info.get("health_check_model") is None return litellm.supports_audio_output( model=base_model if isinstance(base_model, str) and probes_own_model else model, custom_llm_provider=provider @@ -842,8 +842,8 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic - updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes - updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models - for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID, and pins `custom_llm_provider` to `bedrock` (only when the deployment hasn't already set one, so an explicit `bedrock_converse` survives) so the bare model id still resolves to the provider (e.g. cross-region ids like `us.cohere.embed-v4:0`) - - asks OpenAI and Azure chat models that support audio output (`model_info.base_model` first, unless - `health_check_model` replaces the probed model) for `modalities` + - asks OpenAI and Azure chat models that support audio output (`base_model` from `model_info`, then + `litellm_params`, first, unless `health_check_model` replaces the probed model) for `modalities` text + audio and a wav `audio` in the `health_check_voice` (default "alloy"), since those models reject text-only requests. Keys already set through `health_check_params` are kept as they are """ diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 684f1619512..a44c50ce44a 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -203,13 +203,21 @@ def _config_base_for_health_check( return {key: value for key, value in config_params.items() if key not in _CONFIG_CONNECTION_FIELDS} -def _model_info_for_mode_resolution( - model_info: Mapping[str, object], stored_params: Mapping[str, object], request_params: Mapping[str, object] -) -> Mapping[str, object]: +_STORED_MODEL_INFO_KEYS: Final = frozenset(("mode", "base_model")) +_STORED_MODEL_PARAM_KEYS: Final = frozenset(("base_model",)) + + +def _request_swaps_the_model(stored_params: Mapping[str, object], request_params: Mapping[str, object]) -> bool: stored_model: Final = stored_params.get("model") - if stored_model is None or request_params.get("model") in (None, stored_model): - return model_info - return {key: value for key, value in model_info.items() if key != "mode"} + return stored_model is not None and request_params.get("model") not in (None, stored_model) + + +def _without_stored_model_keys( + fields: Mapping[str, object], stored_model_keys: frozenset[str], *, request_swaps_the_model: bool +) -> Mapping[str, object]: + if not request_swaps_the_model: + return fields + return {key: value for key, value in fields.items() if key not in stored_model_keys} def _string_mode_or_bad_request(params_mode: object) -> str | None: @@ -2306,21 +2314,29 @@ async def test_model_connection( selected_by_id: Final = deployment_by_id is not None reject_server_owned_wif_fields(request_litellm_params) # Merge: config params (from proxy config) as base, request params override + request_swaps_the_model: Final = _request_swaps_the_model( + _OBJECT_MAPPING.validate_python(config_litellm_params), + _OBJECT_MAPPING.validate_python(request_litellm_params), + ) litellm_params = { - **_config_base_for_health_check( - config_litellm_params, - request_litellm_params, - allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True, - selected_by_id=selected_by_id, + **_without_stored_model_keys( + _config_base_for_health_check( + config_litellm_params, + request_litellm_params, + allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True, + selected_by_id=selected_by_id, + ), + _STORED_MODEL_PARAM_KEYS, + request_swaps_the_model=request_swaps_the_model, ), **request_litellm_params, } resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info - probe_model_info: Final = _model_info_for_mode_resolution( + probe_model_info: Final = _without_stored_model_keys( _OBJECT_MAPPING.validate_python(resolved_model_info or {}), - stored_params=_OBJECT_MAPPING.validate_python(config_litellm_params), - request_params=_OBJECT_MAPPING.validate_python(request_litellm_params), + _STORED_MODEL_INFO_KEYS, + request_swaps_the_model=request_swaps_the_model, ) litellm_params = update_litellm_params_for_health_check( model_info=dict(probe_model_info), diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 950d1742d49..9ac3185dcb2 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -902,6 +902,45 @@ async def test_test_model_connection_overridden_model_probe_params_follow_the_pr assert "max_tokens" not in ahealth_check.call_args.kwargs["model_params"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_params", "deployment_model_info"), + [ + ({"model": "azure/my-audio-deployment"}, {"id": "audio-id", "base_model": "azure/gpt-audio-1.5"}), + ({"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-1.5"}, {"id": "audio-id"}), + ], + ids=["model_info.base_model", "litellm_params.base_model"], +) +@pytest.mark.parametrize( + ("request_params", "expects_audio_probe"), + [({}, True), ({"model": "azure/my-text-deployment"}, False)], + ids=["own model", "swapped model"], +) +async def test_test_model_connection_stored_audio_base_model_follows_the_stored_model( + deployment_params: Mapping[str, str], + deployment_model_info: Mapping[str, str], + request_params: Mapping[str, str], + expects_audio_probe: bool, +): + deployment: Final = MappingProxyType( + { + "model_name": "custom-name-audio", + "litellm_params": {**deployment_params, "api_key": "fake-azure-key", "api_base": "https://x.openai.azure.com"}, + "model_info": deployment_model_info, + } + ) + with _test_connection_probe(deployment) as ahealth_check: + await health_test_model_connection( + request=MagicMock(), + mode=None, + litellm_params=dict(request_params), + model_info={"id": "audio-id"}, + user_api_key_dict=UserAPIKeyAuth(user_id="test-user", token="test-token"), + ) + + assert ("modalities" in ahealth_check.call_args.kwargs["model_params"]) is expects_audio_probe + + @pytest.mark.asyncio @pytest.mark.parametrize("params_mode", [123, ["chat"], {"mode": "chat"}, False], ids=["int", "list", "dict", "bool"]) async def test_test_model_connection_non_string_params_mode_is_a_bad_request(params_mode: object): diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index 062f5b87e4c..e6450f8f520 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -846,6 +846,7 @@ def test_health_check_params_apply_to_non_chat_modes(): ({}, {"model": "azure/gpt-audio-mini"}), ({}, {"model": "openai/gpt-audio-mini"}), ({"base_model": "azure/gpt-audio-mini"}, {"model": "azure/my-audio-deployment"}), + ({}, {"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-1.5"}), ( {"base_model": "azure/gpt-4o-mini", "health_check_model": "azure/gpt-audio-mini"}, {"model": "azure/my-text-deployment"}, @@ -874,6 +875,10 @@ def test_openai_and_azure_audio_chat_probe_asks_for_audio_output( {"base_model": "azure/gpt-audio-mini", "health_check_model": "azure/gpt-4o-mini"}, {"model": "azure/my-audio-deployment"}, ), + ( + {"health_check_model": "azure/gpt-4o-mini"}, + {"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-mini"}, + ), ], ) def test_probe_without_openai_audio_chat_model_asks_for_no_audio_output( From e707374599ff7f7a05f8dbdf6fb2f3e5859258f2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:40:21 -0700 Subject: [PATCH 4/4] fix(health): judge the audio probe on a swapped-in health_check_params model, and keep Test Connect's stored base_model --- litellm/proxy/health_check.py | 26 ++++++----- .../health_endpoints/_health_endpoints.py | 46 +++++++------------ .../health_endpoints/test_health_endpoints.py | 10 ++-- .../proxy/test_health_check_max_tokens.py | 12 +++++ 4 files changed, 49 insertions(+), 45 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index e42b9159a35..6b51fea02c9 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -135,7 +135,7 @@ _AUDIO_OUTPUT_PROBE_PROVIDERS: Final[frozenset[str]] = frozenset({"openai", "azu def _health_check_requires_audio_output( - model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None + model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None, deployment_model: object ) -> bool: if (mode or "chat") != "chat": return False @@ -152,16 +152,16 @@ def _health_check_requires_audio_output( if provider not in _AUDIO_OUTPUT_PROBE_PROVIDERS: return False base_model: Final = model_info.get("base_model") or probe_params.get("base_model") - probes_own_model: Final = model_info.get("health_check_model") is None return litellm.supports_audio_output( - model=base_model if isinstance(base_model, str) and probes_own_model else model, custom_llm_provider=provider + model=base_model if isinstance(base_model, str) and model == deployment_model else model, + custom_llm_provider=provider, ) def _health_check_audio_output_params( - model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None + model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None, deployment_model: object ) -> Mapping[str, object]: - if not _health_check_requires_audio_output(model_info, probe_params, mode): + if not _health_check_requires_audio_output(model_info, probe_params, mode, deployment_model): return MappingProxyType({}) voice: Final = model_info.get("health_check_voice") return MappingProxyType( @@ -826,7 +826,9 @@ _PROBE_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) -def update_litellm_params_for_health_check(model_info: dict, litellm_params: dict) -> dict: +def update_litellm_params_for_health_check( + model_info: dict, litellm_params: dict, deployment_model: str | None = None +) -> dict: """ Update the litellm params for health check. @@ -842,11 +844,13 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic - updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes - updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models - for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID, and pins `custom_llm_provider` to `bedrock` (only when the deployment hasn't already set one, so an explicit `bedrock_converse` survives) so the bare model id still resolves to the provider (e.g. cross-region ids like `us.cohere.embed-v4:0`) - - asks OpenAI and Azure chat models that support audio output (`base_model` from `model_info`, then - `litellm_params`, first, unless `health_check_model` replaces the probed model) for `modalities` - text + audio and a wav `audio` in the `health_check_voice` (default "alloy"), since those models reject - text-only requests. Keys already set through `health_check_params` are kept as they are + - asks OpenAI and Azure chat models that support audio output for `modalities` text + audio and a wav + `audio` in the `health_check_voice` (default "alloy"), since those models reject text-only requests. + `base_model` (from `model_info`, then `litellm_params`) decides it only while the probe still targets + `deployment_model` (default: the incoming `litellm_params` model), so a `health_check_model` or a `model` + in `health_check_params` is judged on its own. Keys already set through `health_check_params` are kept """ + base_model_owner: Final = deployment_model or _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model") mode: Final = resolve_health_check_mode( model_info, litellm_params, # any-ok: untyped router config dict @@ -922,7 +926,7 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params) audio_output_params: Final = _health_check_audio_output_params( - _MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode + _MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode, base_model_owner ) return { key: value diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index a44c50ce44a..be5b94ca350 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -203,21 +203,13 @@ def _config_base_for_health_check( return {key: value for key, value in config_params.items() if key not in _CONFIG_CONNECTION_FIELDS} -_STORED_MODEL_INFO_KEYS: Final = frozenset(("mode", "base_model")) -_STORED_MODEL_PARAM_KEYS: Final = frozenset(("base_model",)) - - -def _request_swaps_the_model(stored_params: Mapping[str, object], request_params: Mapping[str, object]) -> bool: - stored_model: Final = stored_params.get("model") - return stored_model is not None and request_params.get("model") not in (None, stored_model) - - -def _without_stored_model_keys( - fields: Mapping[str, object], stored_model_keys: frozenset[str], *, request_swaps_the_model: bool +def _model_info_for_mode_resolution( + model_info: Mapping[str, object], stored_params: Mapping[str, object], request_params: Mapping[str, object] ) -> Mapping[str, object]: - if not request_swaps_the_model: - return fields - return {key: value for key, value in fields.items() if key not in stored_model_keys} + stored_model: Final = stored_params.get("model") + if stored_model is None or request_params.get("model") in (None, stored_model): + return model_info + return {key: value for key, value in model_info.items() if key != "mode"} def _string_mode_or_bad_request(params_mode: object) -> str | None: @@ -2314,33 +2306,27 @@ async def test_model_connection( selected_by_id: Final = deployment_by_id is not None reject_server_owned_wif_fields(request_litellm_params) # Merge: config params (from proxy config) as base, request params override - request_swaps_the_model: Final = _request_swaps_the_model( - _OBJECT_MAPPING.validate_python(config_litellm_params), - _OBJECT_MAPPING.validate_python(request_litellm_params), - ) litellm_params = { - **_without_stored_model_keys( - _config_base_for_health_check( - config_litellm_params, - request_litellm_params, - allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True, - selected_by_id=selected_by_id, - ), - _STORED_MODEL_PARAM_KEYS, - request_swaps_the_model=request_swaps_the_model, + **_config_base_for_health_check( + config_litellm_params, + request_litellm_params, + allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True, + selected_by_id=selected_by_id, ), **request_litellm_params, } resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info - probe_model_info: Final = _without_stored_model_keys( + probe_model_info: Final = _model_info_for_mode_resolution( _OBJECT_MAPPING.validate_python(resolved_model_info or {}), - _STORED_MODEL_INFO_KEYS, - request_swaps_the_model=request_swaps_the_model, + stored_params=_OBJECT_MAPPING.validate_python(config_litellm_params), + request_params=_OBJECT_MAPPING.validate_python(request_litellm_params), ) + stored_model: Final = _OBJECT_MAPPING.validate_python(config_litellm_params).get("model") litellm_params = update_litellm_params_for_health_check( model_info=dict(probe_model_info), litellm_params=litellm_params, + deployment_model=stored_model if isinstance(stored_model, str) else None, ) ## Auth check, on the final probe params so health_check_params cannot retarget it afterwards diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 9ac3185dcb2..1055de06e38 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -394,7 +394,7 @@ async def test_test_model_connection_loads_config_from_router(): mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result) # Mock _update_litellm_params_for_health_check - def mock_update_params(model_info, litellm_params): + def mock_update_params(model_info, litellm_params, deployment_model=None): # Just return params with messages added params = litellm_params.copy() params["messages"] = [{"role": "user", "content": "test"}] @@ -545,7 +545,7 @@ async def test_test_model_connection_uses_model_info_id_to_disambiguate_duplicat mock_ahealth_check = AsyncMock(return_value=mock_health_check_result) mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result) - def mock_update_params(model_info, litellm_params): + def mock_update_params(model_info, litellm_params, deployment_model=None): params = litellm_params.copy() params["messages"] = [{"role": "user", "content": "test"}] return params @@ -656,7 +656,7 @@ async def test_test_model_connection_falls_back_to_deployments_zero_without_id() mock_ahealth_check = AsyncMock(return_value=mock_health_check_result) mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result) - def mock_update_params(model_info, litellm_params): + def mock_update_params(model_info, litellm_params, deployment_model=None): params = litellm_params.copy() params["messages"] = [{"role": "user", "content": "test"}] return params @@ -938,7 +938,9 @@ async def test_test_model_connection_stored_audio_base_model_follows_the_stored_ user_api_key_dict=UserAPIKeyAuth(user_id="test-user", token="test-token"), ) - assert ("modalities" in ahealth_check.call_args.kwargs["model_params"]) is expects_audio_probe + probe_params: Final = ahealth_check.call_args.kwargs["model_params"] + assert ("modalities" in probe_params) is expects_audio_probe + assert probe_params.get("base_model") == deployment_params.get("base_model") @pytest.mark.asyncio diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index e6450f8f520..b6ce1893202 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -851,6 +851,10 @@ def test_health_check_params_apply_to_non_chat_modes(): {"base_model": "azure/gpt-4o-mini", "health_check_model": "azure/gpt-audio-mini"}, {"model": "azure/my-text-deployment"}, ), + ( + {"base_model": "gpt-4o-mini", "health_check_params": {"model": "openai/gpt-audio-mini"}}, + {"model": "openai/my-text-model"}, + ), ], ) def test_openai_and_azure_audio_chat_probe_asks_for_audio_output( @@ -879,6 +883,14 @@ def test_openai_and_azure_audio_chat_probe_asks_for_audio_output( {"health_check_model": "azure/gpt-4o-mini"}, {"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-mini"}, ), + ( + {"base_model": "gpt-audio-mini", "health_check_params": {"model": "openai/gpt-5.4-mini"}}, + {"model": "openai/gpt-audio-mini"}, + ), + ( + {"health_check_params": {"model": "openai/gpt-5.4-mini"}}, + {"model": "openai/gpt-audio-mini", "base_model": "gpt-audio-mini"}, + ), ], ) def test_probe_without_openai_audio_chat_model_asks_for_no_audio_output(