mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(health): read litellm_params.base_model, and drop the stored base_model when Test Connect swaps the model
This commit is contained in:
parent
366b116574
commit
89b8f346d2
4 changed files with 77 additions and 17 deletions
|
|
@ -151,7 +151,7 @@ def _health_check_requires_audio_output(
|
|||
return False
|
||||
if provider not in _AUDIO_OUTPUT_PROBE_PROVIDERS:
|
||||
return False
|
||||
base_model: Final = model_info.get("base_model")
|
||||
base_model: Final = model_info.get("base_model") or probe_params.get("base_model")
|
||||
probes_own_model: Final = model_info.get("health_check_model") is None
|
||||
return litellm.supports_audio_output(
|
||||
model=base_model if isinstance(base_model, str) and probes_own_model else model, custom_llm_provider=provider
|
||||
|
|
@ -842,8 +842,8 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic
|
|||
- updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes
|
||||
- updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models
|
||||
- for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID, and pins `custom_llm_provider` to `bedrock` (only when the deployment hasn't already set one, so an explicit `bedrock_converse` survives) so the bare model id still resolves to the provider (e.g. cross-region ids like `us.cohere.embed-v4:0`)
|
||||
- asks OpenAI and Azure chat models that support audio output (`model_info.base_model` first, unless
|
||||
`health_check_model` replaces the probed model) for `modalities`
|
||||
- asks OpenAI and Azure chat models that support audio output (`base_model` from `model_info`, then
|
||||
`litellm_params`, first, unless `health_check_model` replaces the probed model) for `modalities`
|
||||
text + audio and a wav `audio` in the `health_check_voice` (default "alloy"), since those models reject
|
||||
text-only requests. Keys already set through `health_check_params` are kept as they are
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -203,13 +203,21 @@ def _config_base_for_health_check(
|
|||
return {key: value for key, value in config_params.items() if key not in _CONFIG_CONNECTION_FIELDS}
|
||||
|
||||
|
||||
def _model_info_for_mode_resolution(
|
||||
model_info: Mapping[str, object], stored_params: Mapping[str, object], request_params: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
_STORED_MODEL_INFO_KEYS: Final = frozenset(("mode", "base_model"))
|
||||
_STORED_MODEL_PARAM_KEYS: Final = frozenset(("base_model",))
|
||||
|
||||
|
||||
def _request_swaps_the_model(stored_params: Mapping[str, object], request_params: Mapping[str, object]) -> bool:
|
||||
stored_model: Final = stored_params.get("model")
|
||||
if stored_model is None or request_params.get("model") in (None, stored_model):
|
||||
return model_info
|
||||
return {key: value for key, value in model_info.items() if key != "mode"}
|
||||
return stored_model is not None and request_params.get("model") not in (None, stored_model)
|
||||
|
||||
|
||||
def _without_stored_model_keys(
|
||||
fields: Mapping[str, object], stored_model_keys: frozenset[str], *, request_swaps_the_model: bool
|
||||
) -> Mapping[str, object]:
|
||||
if not request_swaps_the_model:
|
||||
return fields
|
||||
return {key: value for key, value in fields.items() if key not in stored_model_keys}
|
||||
|
||||
|
||||
def _string_mode_or_bad_request(params_mode: object) -> str | None:
|
||||
|
|
@ -2306,21 +2314,29 @@ async def test_model_connection(
|
|||
selected_by_id: Final = deployment_by_id is not None
|
||||
reject_server_owned_wif_fields(request_litellm_params)
|
||||
# Merge: config params (from proxy config) as base, request params override
|
||||
request_swaps_the_model: Final = _request_swaps_the_model(
|
||||
_OBJECT_MAPPING.validate_python(config_litellm_params),
|
||||
_OBJECT_MAPPING.validate_python(request_litellm_params),
|
||||
)
|
||||
litellm_params = {
|
||||
**_config_base_for_health_check(
|
||||
config_litellm_params,
|
||||
request_litellm_params,
|
||||
allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True,
|
||||
selected_by_id=selected_by_id,
|
||||
**_without_stored_model_keys(
|
||||
_config_base_for_health_check(
|
||||
config_litellm_params,
|
||||
request_litellm_params,
|
||||
allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True,
|
||||
selected_by_id=selected_by_id,
|
||||
),
|
||||
_STORED_MODEL_PARAM_KEYS,
|
||||
request_swaps_the_model=request_swaps_the_model,
|
||||
),
|
||||
**request_litellm_params,
|
||||
}
|
||||
|
||||
resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info
|
||||
probe_model_info: Final = _model_info_for_mode_resolution(
|
||||
probe_model_info: Final = _without_stored_model_keys(
|
||||
_OBJECT_MAPPING.validate_python(resolved_model_info or {}),
|
||||
stored_params=_OBJECT_MAPPING.validate_python(config_litellm_params),
|
||||
request_params=_OBJECT_MAPPING.validate_python(request_litellm_params),
|
||||
_STORED_MODEL_INFO_KEYS,
|
||||
request_swaps_the_model=request_swaps_the_model,
|
||||
)
|
||||
litellm_params = update_litellm_params_for_health_check(
|
||||
model_info=dict(probe_model_info),
|
||||
|
|
|
|||
|
|
@ -902,6 +902,45 @@ async def test_test_model_connection_overridden_model_probe_params_follow_the_pr
|
|||
assert "max_tokens" not in ahealth_check.call_args.kwargs["model_params"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_params", "deployment_model_info"),
|
||||
[
|
||||
({"model": "azure/my-audio-deployment"}, {"id": "audio-id", "base_model": "azure/gpt-audio-1.5"}),
|
||||
({"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-1.5"}, {"id": "audio-id"}),
|
||||
],
|
||||
ids=["model_info.base_model", "litellm_params.base_model"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("request_params", "expects_audio_probe"),
|
||||
[({}, True), ({"model": "azure/my-text-deployment"}, False)],
|
||||
ids=["own model", "swapped model"],
|
||||
)
|
||||
async def test_test_model_connection_stored_audio_base_model_follows_the_stored_model(
|
||||
deployment_params: Mapping[str, str],
|
||||
deployment_model_info: Mapping[str, str],
|
||||
request_params: Mapping[str, str],
|
||||
expects_audio_probe: bool,
|
||||
):
|
||||
deployment: Final = MappingProxyType(
|
||||
{
|
||||
"model_name": "custom-name-audio",
|
||||
"litellm_params": {**deployment_params, "api_key": "fake-azure-key", "api_base": "https://x.openai.azure.com"},
|
||||
"model_info": deployment_model_info,
|
||||
}
|
||||
)
|
||||
with _test_connection_probe(deployment) as ahealth_check:
|
||||
await health_test_model_connection(
|
||||
request=MagicMock(),
|
||||
mode=None,
|
||||
litellm_params=dict(request_params),
|
||||
model_info={"id": "audio-id"},
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="test-user", token="test-token"),
|
||||
)
|
||||
|
||||
assert ("modalities" in ahealth_check.call_args.kwargs["model_params"]) is expects_audio_probe
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("params_mode", [123, ["chat"], {"mode": "chat"}, False], ids=["int", "list", "dict", "bool"])
|
||||
async def test_test_model_connection_non_string_params_mode_is_a_bad_request(params_mode: object):
|
||||
|
|
|
|||
|
|
@ -846,6 +846,7 @@ def test_health_check_params_apply_to_non_chat_modes():
|
|||
({}, {"model": "azure/gpt-audio-mini"}),
|
||||
({}, {"model": "openai/gpt-audio-mini"}),
|
||||
({"base_model": "azure/gpt-audio-mini"}, {"model": "azure/my-audio-deployment"}),
|
||||
({}, {"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-1.5"}),
|
||||
(
|
||||
{"base_model": "azure/gpt-4o-mini", "health_check_model": "azure/gpt-audio-mini"},
|
||||
{"model": "azure/my-text-deployment"},
|
||||
|
|
@ -874,6 +875,10 @@ def test_openai_and_azure_audio_chat_probe_asks_for_audio_output(
|
|||
{"base_model": "azure/gpt-audio-mini", "health_check_model": "azure/gpt-4o-mini"},
|
||||
{"model": "azure/my-audio-deployment"},
|
||||
),
|
||||
(
|
||||
{"health_check_model": "azure/gpt-4o-mini"},
|
||||
{"model": "azure/my-audio-deployment", "base_model": "azure/gpt-audio-mini"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_probe_without_openai_audio_chat_model_asks_for_no_audio_output(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue