mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(health): judge the audio probe on a swapped-in health_check_params model, and keep Test Connect's stored base_model
This commit is contained in:
parent
89b8f346d2
commit
e707374599
4 changed files with 49 additions and 45 deletions
|
|
@ -135,7 +135,7 @@ _AUDIO_OUTPUT_PROBE_PROVIDERS: Final[frozenset[str]] = frozenset({"openai", "azu
|
|||
|
||||
|
||||
def _health_check_requires_audio_output(
|
||||
model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None
|
||||
model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None, deployment_model: object
|
||||
) -> bool:
|
||||
if (mode or "chat") != "chat":
|
||||
return False
|
||||
|
|
@ -152,16 +152,16 @@ def _health_check_requires_audio_output(
|
|||
if provider not in _AUDIO_OUTPUT_PROBE_PROVIDERS:
|
||||
return False
|
||||
base_model: Final = model_info.get("base_model") or probe_params.get("base_model")
|
||||
probes_own_model: Final = model_info.get("health_check_model") is None
|
||||
return litellm.supports_audio_output(
|
||||
model=base_model if isinstance(base_model, str) and probes_own_model else model, custom_llm_provider=provider
|
||||
model=base_model if isinstance(base_model, str) and model == deployment_model else model,
|
||||
custom_llm_provider=provider,
|
||||
)
|
||||
|
||||
|
||||
def _health_check_audio_output_params(
|
||||
model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None
|
||||
model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None, deployment_model: object
|
||||
) -> Mapping[str, object]:
|
||||
if not _health_check_requires_audio_output(model_info, probe_params, mode):
|
||||
if not _health_check_requires_audio_output(model_info, probe_params, mode, deployment_model):
|
||||
return MappingProxyType({})
|
||||
voice: Final = model_info.get("health_check_voice")
|
||||
return MappingProxyType(
|
||||
|
|
@ -826,7 +826,9 @@ _PROBE_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
|
|||
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def update_litellm_params_for_health_check(model_info: dict, litellm_params: dict) -> dict:
|
||||
def update_litellm_params_for_health_check(
|
||||
model_info: dict, litellm_params: dict, deployment_model: str | None = None
|
||||
) -> dict:
|
||||
"""
|
||||
Update the litellm params for health check.
|
||||
|
||||
|
|
@ -842,11 +844,13 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic
|
|||
- updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes
|
||||
- updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models
|
||||
- for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID, and pins `custom_llm_provider` to `bedrock` (only when the deployment hasn't already set one, so an explicit `bedrock_converse` survives) so the bare model id still resolves to the provider (e.g. cross-region ids like `us.cohere.embed-v4:0`)
|
||||
- asks OpenAI and Azure chat models that support audio output (`base_model` from `model_info`, then
|
||||
`litellm_params`, first, unless `health_check_model` replaces the probed model) for `modalities`
|
||||
text + audio and a wav `audio` in the `health_check_voice` (default "alloy"), since those models reject
|
||||
text-only requests. Keys already set through `health_check_params` are kept as they are
|
||||
- asks OpenAI and Azure chat models that support audio output for `modalities` text + audio and a wav
|
||||
`audio` in the `health_check_voice` (default "alloy"), since those models reject text-only requests.
|
||||
`base_model` (from `model_info`, then `litellm_params`) decides it only while the probe still targets
|
||||
`deployment_model` (default: the incoming `litellm_params` model), so a `health_check_model` or a `model`
|
||||
in `health_check_params` is judged on its own. Keys already set through `health_check_params` are kept
|
||||
"""
|
||||
base_model_owner: Final = deployment_model or _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model")
|
||||
mode: Final = resolve_health_check_mode(
|
||||
model_info,
|
||||
litellm_params, # any-ok: untyped router config dict
|
||||
|
|
@ -922,7 +926,7 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic
|
|||
|
||||
probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params)
|
||||
audio_output_params: Final = _health_check_audio_output_params(
|
||||
_MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode
|
||||
_MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode, base_model_owner
|
||||
)
|
||||
return {
|
||||
key: value
|
||||
|
|
|
|||
|
|
@ -203,21 +203,13 @@ def _config_base_for_health_check(
|
|||
return {key: value for key, value in config_params.items() if key not in _CONFIG_CONNECTION_FIELDS}
|
||||
|
||||
|
||||
_STORED_MODEL_INFO_KEYS: Final = frozenset(("mode", "base_model"))
|
||||
_STORED_MODEL_PARAM_KEYS: Final = frozenset(("base_model",))
|
||||
|
||||
|
||||
def _request_swaps_the_model(stored_params: Mapping[str, object], request_params: Mapping[str, object]) -> bool:
|
||||
stored_model: Final = stored_params.get("model")
|
||||
return stored_model is not None and request_params.get("model") not in (None, stored_model)
|
||||
|
||||
|
||||
def _without_stored_model_keys(
|
||||
fields: Mapping[str, object], stored_model_keys: frozenset[str], *, request_swaps_the_model: bool
|
||||
def _model_info_for_mode_resolution(
|
||||
model_info: Mapping[str, object], stored_params: Mapping[str, object], request_params: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
if not request_swaps_the_model:
|
||||
return fields
|
||||
return {key: value for key, value in fields.items() if key not in stored_model_keys}
|
||||
stored_model: Final = stored_params.get("model")
|
||||
if stored_model is None or request_params.get("model") in (None, stored_model):
|
||||
return model_info
|
||||
return {key: value for key, value in model_info.items() if key != "mode"}
|
||||
|
||||
|
||||
def _string_mode_or_bad_request(params_mode: object) -> str | None:
|
||||
|
|
@ -2314,33 +2306,27 @@ async def test_model_connection(
|
|||
selected_by_id: Final = deployment_by_id is not None
|
||||
reject_server_owned_wif_fields(request_litellm_params)
|
||||
# Merge: config params (from proxy config) as base, request params override
|
||||
request_swaps_the_model: Final = _request_swaps_the_model(
|
||||
_OBJECT_MAPPING.validate_python(config_litellm_params),
|
||||
_OBJECT_MAPPING.validate_python(request_litellm_params),
|
||||
)
|
||||
litellm_params = {
|
||||
**_without_stored_model_keys(
|
||||
_config_base_for_health_check(
|
||||
config_litellm_params,
|
||||
request_litellm_params,
|
||||
allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True,
|
||||
selected_by_id=selected_by_id,
|
||||
),
|
||||
_STORED_MODEL_PARAM_KEYS,
|
||||
request_swaps_the_model=request_swaps_the_model,
|
||||
**_config_base_for_health_check(
|
||||
config_litellm_params,
|
||||
request_litellm_params,
|
||||
allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True,
|
||||
selected_by_id=selected_by_id,
|
||||
),
|
||||
**request_litellm_params,
|
||||
}
|
||||
|
||||
resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info
|
||||
probe_model_info: Final = _without_stored_model_keys(
|
||||
probe_model_info: Final = _model_info_for_mode_resolution(
|
||||
_OBJECT_MAPPING.validate_python(resolved_model_info or {}),
|
||||
_STORED_MODEL_INFO_KEYS,
|
||||
request_swaps_the_model=request_swaps_the_model,
|
||||
stored_params=_OBJECT_MAPPING.validate_python(config_litellm_params),
|
||||
request_params=_OBJECT_MAPPING.validate_python(request_litellm_params),
|
||||
)
|
||||
stored_model: Final = _OBJECT_MAPPING.validate_python(config_litellm_params).get("model")
|
||||
litellm_params = update_litellm_params_for_health_check(
|
||||
model_info=dict(probe_model_info),
|
||||
litellm_params=litellm_params,
|
||||
deployment_model=stored_model if isinstance(stored_model, str) else None,
|
||||
)
|
||||
|
||||
## Auth check, on the final probe params so health_check_params cannot retarget it afterwards
|
||||
|
|
|
|||
|
|
@ -394,7 +394,7 @@ async def test_test_model_connection_loads_config_from_router():
|
|||
mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result)
|
||||
|
||||
# Mock _update_litellm_params_for_health_check
|
||||
def mock_update_params(model_info, litellm_params):
|
||||
def mock_update_params(model_info, litellm_params, deployment_model=None):
|
||||
# Just return params with messages added
|
||||
params = litellm_params.copy()
|
||||
params["messages"] = [{"role": "user", "content": "test"}]
|
||||
|
|
@ -545,7 +545,7 @@ async def test_test_model_connection_uses_model_info_id_to_disambiguate_duplicat
|
|||
mock_ahealth_check = AsyncMock(return_value=mock_health_check_result)
|
||||
mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result)
|
||||
|
||||
def mock_update_params(model_info, litellm_params):
|
||||
def mock_update_params(model_info, litellm_params, deployment_model=None):
|
||||
params = litellm_params.copy()
|
||||
params["messages"] = [{"role": "user", "content": "test"}]
|
||||
return params
|
||||
|
|
@ -656,7 +656,7 @@ async def test_test_model_connection_falls_back_to_deployments_zero_without_id()
|
|||
mock_ahealth_check = AsyncMock(return_value=mock_health_check_result)
|
||||
mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result)
|
||||
|
||||
def mock_update_params(model_info, litellm_params):
|
||||
def mock_update_params(model_info, litellm_params, deployment_model=None):
|
||||
params = litellm_params.copy()
|
||||
params["messages"] = [{"role": "user", "content": "test"}]
|
||||
return params
|
||||
|
|
@ -938,7 +938,9 @@ async def test_test_model_connection_stored_audio_base_model_follows_the_stored_
|
|||
user_api_key_dict=UserAPIKeyAuth(user_id="test-user", token="test-token"),
|
||||
)
|
||||
|
||||
assert ("modalities" in ahealth_check.call_args.kwargs["model_params"]) is expects_audio_probe
|
||||
probe_params: Final = ahealth_check.call_args.kwargs["model_params"]
|
||||
assert ("modalities" in probe_params) is expects_audio_probe
|
||||
assert probe_params.get("base_model") == deployment_params.get("base_model")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -851,6 +851,10 @@ def test_health_check_params_apply_to_non_chat_modes():
|
|||
{"base_model": "azure/gpt-4o-mini", "health_check_model": "azure/gpt-audio-mini"},
|
||||
{"model": "azure/my-text-deployment"},
|
||||
),
|
||||
(
|
||||
{"base_model": "gpt-4o-mini", "health_check_params": {"model": "openai/gpt-audio-mini"}},
|
||||
{"model": "openai/my-text-model"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_openai_and_azure_audio_chat_probe_asks_for_audio_output(
|
||||
|
|
@ -879,6 +883,14 @@ def test_openai_and_azure_audio_chat_probe_asks_for_audio_output(
|
|||
{"health_check_model": "azure/gpt-4o-mini"},
|
||||
{"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-mini"},
|
||||
),
|
||||
(
|
||||
{"base_model": "gpt-audio-mini", "health_check_params": {"model": "openai/gpt-5.4-mini"}},
|
||||
{"model": "openai/gpt-audio-mini"},
|
||||
),
|
||||
(
|
||||
{"health_check_params": {"model": "openai/gpt-5.4-mini"}},
|
||||
{"model": "openai/gpt-audio-mini", "base_model": "gpt-audio-mini"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_probe_without_openai_audio_chat_model_asks_for_no_audio_output(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue