fix(health): read litellm_params.base_model, and drop the stored base_model when Test Connect swaps the model

This commit is contained in:
mateo-berri 2026-10-20 20:50:47 -07:00
parent 366b116574
commit 89b8f346d2
4 changed files with 77 additions and 17 deletions

View file

@ -151,7 +151,7 @@ def _health_check_requires_audio_output(
return False
if provider not in _AUDIO_OUTPUT_PROBE_PROVIDERS:
return False
base_model: Final = model_info.get("base_model")
base_model: Final = model_info.get("base_model") or probe_params.get("base_model")
probes_own_model: Final = model_info.get("health_check_model") is None
return litellm.supports_audio_output(
model=base_model if isinstance(base_model, str) and probes_own_model else model, custom_llm_provider=provider
@ -842,8 +842,8 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic
- updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes
- updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models
- for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID, and pins `custom_llm_provider` to `bedrock` (only when the deployment hasn't already set one, so an explicit `bedrock_converse` survives) so the bare model id still resolves to the provider (e.g. cross-region ids like `us.cohere.embed-v4:0`)
- asks OpenAI and Azure chat models that support audio output (`model_info.base_model` first, unless
`health_check_model` replaces the probed model) for `modalities`
- asks OpenAI and Azure chat models that support audio output (`base_model` from `model_info`, then
`litellm_params`, first, unless `health_check_model` replaces the probed model) for `modalities`
text + audio and a wav `audio` in the `health_check_voice` (default "alloy"), since those models reject
text-only requests. Keys already set through `health_check_params` are kept as they are
"""

View file

@ -203,13 +203,21 @@ def _config_base_for_health_check(
return {key: value for key, value in config_params.items() if key not in _CONFIG_CONNECTION_FIELDS}
def _model_info_for_mode_resolution(
model_info: Mapping[str, object], stored_params: Mapping[str, object], request_params: Mapping[str, object]
) -> Mapping[str, object]:
_STORED_MODEL_INFO_KEYS: Final = frozenset(("mode", "base_model"))
_STORED_MODEL_PARAM_KEYS: Final = frozenset(("base_model",))
def _request_swaps_the_model(stored_params: Mapping[str, object], request_params: Mapping[str, object]) -> bool:
stored_model: Final = stored_params.get("model")
if stored_model is None or request_params.get("model") in (None, stored_model):
return model_info
return {key: value for key, value in model_info.items() if key != "mode"}
return stored_model is not None and request_params.get("model") not in (None, stored_model)
def _without_stored_model_keys(
fields: Mapping[str, object], stored_model_keys: frozenset[str], *, request_swaps_the_model: bool
) -> Mapping[str, object]:
if not request_swaps_the_model:
return fields
return {key: value for key, value in fields.items() if key not in stored_model_keys}
def _string_mode_or_bad_request(params_mode: object) -> str | None:
@ -2306,21 +2314,29 @@ async def test_model_connection(
selected_by_id: Final = deployment_by_id is not None
reject_server_owned_wif_fields(request_litellm_params)
# Merge: config params (from proxy config) as base, request params override
request_swaps_the_model: Final = _request_swaps_the_model(
_OBJECT_MAPPING.validate_python(config_litellm_params),
_OBJECT_MAPPING.validate_python(request_litellm_params),
)
litellm_params = {
**_config_base_for_health_check(
config_litellm_params,
request_litellm_params,
allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True,
selected_by_id=selected_by_id,
**_without_stored_model_keys(
_config_base_for_health_check(
config_litellm_params,
request_litellm_params,
allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True,
selected_by_id=selected_by_id,
),
_STORED_MODEL_PARAM_KEYS,
request_swaps_the_model=request_swaps_the_model,
),
**request_litellm_params,
}
resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info
probe_model_info: Final = _model_info_for_mode_resolution(
probe_model_info: Final = _without_stored_model_keys(
_OBJECT_MAPPING.validate_python(resolved_model_info or {}),
stored_params=_OBJECT_MAPPING.validate_python(config_litellm_params),
request_params=_OBJECT_MAPPING.validate_python(request_litellm_params),
_STORED_MODEL_INFO_KEYS,
request_swaps_the_model=request_swaps_the_model,
)
litellm_params = update_litellm_params_for_health_check(
model_info=dict(probe_model_info),

View file

@ -902,6 +902,45 @@ async def test_test_model_connection_overridden_model_probe_params_follow_the_pr
assert "max_tokens" not in ahealth_check.call_args.kwargs["model_params"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_params", "deployment_model_info"),
[
({"model": "azure/my-audio-deployment"}, {"id": "audio-id", "base_model": "azure/gpt-audio-1.5"}),
({"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-1.5"}, {"id": "audio-id"}),
],
ids=["model_info.base_model", "litellm_params.base_model"],
)
@pytest.mark.parametrize(
("request_params", "expects_audio_probe"),
[({}, True), ({"model": "azure/my-text-deployment"}, False)],
ids=["own model", "swapped model"],
)
async def test_test_model_connection_stored_audio_base_model_follows_the_stored_model(
deployment_params: Mapping[str, str],
deployment_model_info: Mapping[str, str],
request_params: Mapping[str, str],
expects_audio_probe: bool,
):
deployment: Final = MappingProxyType(
{
"model_name": "custom-name-audio",
"litellm_params": {**deployment_params, "api_key": "fake-azure-key", "api_base": "https://x.openai.azure.com"},
"model_info": deployment_model_info,
}
)
with _test_connection_probe(deployment) as ahealth_check:
await health_test_model_connection(
request=MagicMock(),
mode=None,
litellm_params=dict(request_params),
model_info={"id": "audio-id"},
user_api_key_dict=UserAPIKeyAuth(user_id="test-user", token="test-token"),
)
assert ("modalities" in ahealth_check.call_args.kwargs["model_params"]) is expects_audio_probe
@pytest.mark.asyncio
@pytest.mark.parametrize("params_mode", [123, ["chat"], {"mode": "chat"}, False], ids=["int", "list", "dict", "bool"])
async def test_test_model_connection_non_string_params_mode_is_a_bad_request(params_mode: object):

View file

@ -846,6 +846,7 @@ def test_health_check_params_apply_to_non_chat_modes():
({}, {"model": "azure/gpt-audio-mini"}),
({}, {"model": "openai/gpt-audio-mini"}),
({"base_model": "azure/gpt-audio-mini"}, {"model": "azure/my-audio-deployment"}),
({}, {"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-1.5"}),
(
{"base_model": "azure/gpt-4o-mini", "health_check_model": "azure/gpt-audio-mini"},
{"model": "azure/my-text-deployment"},
@ -874,6 +875,10 @@ def test_openai_and_azure_audio_chat_probe_asks_for_audio_output(
{"base_model": "azure/gpt-audio-mini", "health_check_model": "azure/gpt-4o-mini"},
{"model": "azure/my-audio-deployment"},
),
(
{"health_check_model": "azure/gpt-4o-mini"},
{"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-mini"},
),
],
)
def test_probe_without_openai_audio_chat_model_asks_for_no_audio_output(