fix(health): judge the audio probe on a swapped-in health_check_params model, and keep Test Connect's stored base_model

This commit is contained in:
mateo-berri 2026-10-20 21:40:21 -07:00
parent 89b8f346d2
commit e707374599
4 changed files with 49 additions and 45 deletions

View file

@ -135,7 +135,7 @@ _AUDIO_OUTPUT_PROBE_PROVIDERS: Final[frozenset[str]] = frozenset({"openai", "azu
def _health_check_requires_audio_output(
model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None
model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None, deployment_model: object
) -> bool:
if (mode or "chat") != "chat":
return False
@ -152,16 +152,16 @@ def _health_check_requires_audio_output(
if provider not in _AUDIO_OUTPUT_PROBE_PROVIDERS:
return False
base_model: Final = model_info.get("base_model") or probe_params.get("base_model")
probes_own_model: Final = model_info.get("health_check_model") is None
return litellm.supports_audio_output(
model=base_model if isinstance(base_model, str) and probes_own_model else model, custom_llm_provider=provider
model=base_model if isinstance(base_model, str) and model == deployment_model else model,
custom_llm_provider=provider,
)
def _health_check_audio_output_params(
model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None
model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None, deployment_model: object
) -> Mapping[str, object]:
if not _health_check_requires_audio_output(model_info, probe_params, mode):
if not _health_check_requires_audio_output(model_info, probe_params, mode, deployment_model):
return MappingProxyType({})
voice: Final = model_info.get("health_check_voice")
return MappingProxyType(
@ -826,7 +826,9 @@ _PROBE_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def update_litellm_params_for_health_check(model_info: dict, litellm_params: dict) -> dict:
def update_litellm_params_for_health_check(
model_info: dict, litellm_params: dict, deployment_model: str | None = None
) -> dict:
"""
Update the litellm params for health check.
@ -842,11 +844,13 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic
- updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes
- updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models
- for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID, and pins `custom_llm_provider` to `bedrock` (only when the deployment hasn't already set one, so an explicit `bedrock_converse` survives) so the bare model id still resolves to the provider (e.g. cross-region ids like `us.cohere.embed-v4:0`)
- asks OpenAI and Azure chat models that support audio output (`base_model` from `model_info`, then
`litellm_params`, first, unless `health_check_model` replaces the probed model) for `modalities`
text + audio and a wav `audio` in the `health_check_voice` (default "alloy"), since those models reject
text-only requests. Keys already set through `health_check_params` are kept as they are
- asks OpenAI and Azure chat models that support audio output for `modalities` text + audio and a wav
`audio` in the `health_check_voice` (default "alloy"), since those models reject text-only requests.
`base_model` (from `model_info`, then `litellm_params`) decides it only while the probe still targets
`deployment_model` (default: the incoming `litellm_params` model), so a `health_check_model` or a `model`
in `health_check_params` is judged on its own. Keys already set through `health_check_params` are kept
"""
base_model_owner: Final = deployment_model or _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model")
mode: Final = resolve_health_check_mode(
model_info,
litellm_params, # any-ok: untyped router config dict
@ -922,7 +926,7 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic
probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params)
audio_output_params: Final = _health_check_audio_output_params(
_MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode
_MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode, base_model_owner
)
return {
key: value

View file

@ -203,21 +203,13 @@ def _config_base_for_health_check(
return {key: value for key, value in config_params.items() if key not in _CONFIG_CONNECTION_FIELDS}
_STORED_MODEL_INFO_KEYS: Final = frozenset(("mode", "base_model"))
_STORED_MODEL_PARAM_KEYS: Final = frozenset(("base_model",))
def _request_swaps_the_model(stored_params: Mapping[str, object], request_params: Mapping[str, object]) -> bool:
stored_model: Final = stored_params.get("model")
return stored_model is not None and request_params.get("model") not in (None, stored_model)
def _without_stored_model_keys(
fields: Mapping[str, object], stored_model_keys: frozenset[str], *, request_swaps_the_model: bool
def _model_info_for_mode_resolution(
model_info: Mapping[str, object], stored_params: Mapping[str, object], request_params: Mapping[str, object]
) -> Mapping[str, object]:
if not request_swaps_the_model:
return fields
return {key: value for key, value in fields.items() if key not in stored_model_keys}
stored_model: Final = stored_params.get("model")
if stored_model is None or request_params.get("model") in (None, stored_model):
return model_info
return {key: value for key, value in model_info.items() if key != "mode"}
def _string_mode_or_bad_request(params_mode: object) -> str | None:
@ -2314,33 +2306,27 @@ async def test_model_connection(
selected_by_id: Final = deployment_by_id is not None
reject_server_owned_wif_fields(request_litellm_params)
# Merge: config params (from proxy config) as base, request params override
request_swaps_the_model: Final = _request_swaps_the_model(
_OBJECT_MAPPING.validate_python(config_litellm_params),
_OBJECT_MAPPING.validate_python(request_litellm_params),
)
litellm_params = {
**_without_stored_model_keys(
_config_base_for_health_check(
config_litellm_params,
request_litellm_params,
allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True,
selected_by_id=selected_by_id,
),
_STORED_MODEL_PARAM_KEYS,
request_swaps_the_model=request_swaps_the_model,
**_config_base_for_health_check(
config_litellm_params,
request_litellm_params,
allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True,
selected_by_id=selected_by_id,
),
**request_litellm_params,
}
resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info
probe_model_info: Final = _without_stored_model_keys(
probe_model_info: Final = _model_info_for_mode_resolution(
_OBJECT_MAPPING.validate_python(resolved_model_info or {}),
_STORED_MODEL_INFO_KEYS,
request_swaps_the_model=request_swaps_the_model,
stored_params=_OBJECT_MAPPING.validate_python(config_litellm_params),
request_params=_OBJECT_MAPPING.validate_python(request_litellm_params),
)
stored_model: Final = _OBJECT_MAPPING.validate_python(config_litellm_params).get("model")
litellm_params = update_litellm_params_for_health_check(
model_info=dict(probe_model_info),
litellm_params=litellm_params,
deployment_model=stored_model if isinstance(stored_model, str) else None,
)
## Auth check, on the final probe params so health_check_params cannot retarget it afterwards

View file

@ -394,7 +394,7 @@ async def test_test_model_connection_loads_config_from_router():
mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result)
# Mock _update_litellm_params_for_health_check
def mock_update_params(model_info, litellm_params):
def mock_update_params(model_info, litellm_params, deployment_model=None):
# Just return params with messages added
params = litellm_params.copy()
params["messages"] = [{"role": "user", "content": "test"}]
@ -545,7 +545,7 @@ async def test_test_model_connection_uses_model_info_id_to_disambiguate_duplicat
mock_ahealth_check = AsyncMock(return_value=mock_health_check_result)
mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result)
def mock_update_params(model_info, litellm_params):
def mock_update_params(model_info, litellm_params, deployment_model=None):
params = litellm_params.copy()
params["messages"] = [{"role": "user", "content": "test"}]
return params
@ -656,7 +656,7 @@ async def test_test_model_connection_falls_back_to_deployments_zero_without_id()
mock_ahealth_check = AsyncMock(return_value=mock_health_check_result)
mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result)
def mock_update_params(model_info, litellm_params):
def mock_update_params(model_info, litellm_params, deployment_model=None):
params = litellm_params.copy()
params["messages"] = [{"role": "user", "content": "test"}]
return params
@ -938,7 +938,9 @@ async def test_test_model_connection_stored_audio_base_model_follows_the_stored_
user_api_key_dict=UserAPIKeyAuth(user_id="test-user", token="test-token"),
)
assert ("modalities" in ahealth_check.call_args.kwargs["model_params"]) is expects_audio_probe
probe_params: Final = ahealth_check.call_args.kwargs["model_params"]
assert ("modalities" in probe_params) is expects_audio_probe
assert probe_params.get("base_model") == deployment_params.get("base_model")
@pytest.mark.asyncio

View file

@ -851,6 +851,10 @@ def test_health_check_params_apply_to_non_chat_modes():
{"base_model": "azure/gpt-4o-mini", "health_check_model": "azure/gpt-audio-mini"},
{"model": "azure/my-text-deployment"},
),
(
{"base_model": "gpt-4o-mini", "health_check_params": {"model": "openai/gpt-audio-mini"}},
{"model": "openai/my-text-model"},
),
],
)
def test_openai_and_azure_audio_chat_probe_asks_for_audio_output(
@ -879,6 +883,14 @@ def test_openai_and_azure_audio_chat_probe_asks_for_audio_output(
{"health_check_model": "azure/gpt-4o-mini"},
{"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-mini"},
),
(
{"base_model": "gpt-audio-mini", "health_check_params": {"model": "openai/gpt-5.4-mini"}},
{"model": "openai/gpt-audio-mini"},
),
(
{"health_check_params": {"model": "openai/gpt-5.4-mini"}},
{"model": "openai/gpt-audio-mini", "base_model": "gpt-audio-mini"},
),
],
)
def test_probe_without_openai_audio_chat_model_asks_for_no_audio_output(