From 4cc2f97062918ce0f6718f31f7ecfcefccc70a2c Mon Sep 17 00:00:00 2001 From: Aleksandr Liadov <72351793+AleksandrLiadov@users.noreply.github.com> Date: Tue, 23 Jun 2026 20:05:49 +0200 Subject: [PATCH] fix(azure): health check + chat use max_completion_tokens for gpt-chat-latest Azure gpt-chat-latest rejects max_tokens and requires max_completion_tokens, but it does not route through GPT-5/o-series config. Add a general model-map flag map_max_tokens_to_max_completion_tokens (azure/gpt-chat-latest) consumed by both AzureOpenAIConfig.map_openai_params (normal /chat/completions) and the proxy health-check probe, so neither path 400s. - Add azure/gpt-chat-latest entry + flag to both model cost maps - AzureOpenAIConfig.map_openai_params translates max_tokens -> max_completion_tokens - health_check merges model-cost metadata (suffix deployments) and routes the bounded probe token to max_completion_tokens when flagged - tests for chat + health paths; schema allowlist updated Rebased onto litellm_internal_staging. --- litellm/llms/azure/chat/gpt_transformation.py | 49 +++++++++++++- ...odel_prices_and_context_window_backup.json | 34 ++++++++++ litellm/proxy/health_check.py | 64 +++++++++++++++++-- model_prices_and_context_window.json | 34 ++++++++++ .../test_azure_chat_gpt_transformation.py | 38 +++++++++++ .../proxy/test_health_check_max_tokens.py | 40 ++++++++++++ tests/test_litellm/test_utils.py | 1 + 7 files changed, 253 insertions(+), 7 deletions(-) diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 69eda95be1b..d3c382fb25c 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -150,6 +150,44 @@ class AzureOpenAIConfig(BaseConfig): else: return api_month >= supported_month + def _should_map_max_tokens_to_max_completion_tokens(self, model: str) -> bool: + """Whether `max_tokens` should be sent as `max_completion_tokens`. + + Some Azure models (e.g. azure/gpt-chat-latest) reject `max_tokens` and + require `max_completion_tokens`, but are not part of the GPT-5/o-series + routing that already performs this translation. The model cost map flags + these deployments with `map_max_tokens_to_max_completion_tokens` so the + behavior lives in metadata rather than hardcoded model-name branches. + + The raw model cost map is read directly (rather than via + ``get_model_info``) because ``get_model_info`` returns a typed + ``ModelInfo`` that drops unknown metadata flags. + """ + candidates = [model] + if not model.startswith("azure/"): + candidates.append(f"azure/{model}") + else: + candidates.append(model.split("/", 1)[1]) + + for candidate in candidates: + metadata = litellm.model_cost.get(candidate) + if metadata is None: + try: + from litellm.litellm_core_utils.get_model_cost_map import ( + GetModelCostMap, + ) + + metadata = GetModelCostMap.load_local_model_cost_map().get( + candidate + ) + except Exception: + metadata = None + if isinstance(metadata, dict) and metadata.get( + "map_max_tokens_to_max_completion_tokens" + ): + return True + return False + def map_openai_params( self, non_default_params: dict, @@ -159,6 +197,9 @@ class AzureOpenAIConfig(BaseConfig): api_version: str = "", ) -> dict: supported_openai_params = self.get_supported_openai_params(model) + map_max_completion_tokens = ( + self._should_map_max_tokens_to_max_completion_tokens(model) + ) api_version_times = api_version.split("-") if len(api_version_times) >= 3: @@ -171,7 +212,13 @@ class AzureOpenAIConfig(BaseConfig): api_version_day = None for param, value in non_default_params.items(): - if param == "tool_choice": + if param == "max_tokens" and map_max_completion_tokens: + # Some Azure models (e.g. azure/gpt-chat-latest) reject `max_tokens` + # and require `max_completion_tokens`. The model map flags these via + # `map_max_tokens_to_max_completion_tokens` so we translate the param + # instead of letting the upstream call 400. + optional_params["max_completion_tokens"] = value + elif param == "tool_choice": """ This parameter requires API version 2023-12-01-preview or later diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6ebac7efc8d..0d7f49b9044 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4789,6 +4789,40 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/gpt-chat-latest": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 3e-05, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-openais-newest-chat-model-in-microsoft-foundry/4516848", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "map_max_tokens_to_max_completion_tokens": true + }, "azure/gpt-5-codex": { "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 488467e1b99..eb0fed75c94 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -72,6 +72,52 @@ def _resolve_health_check_mode( return None +def _resolve_health_check_model_info(model_info: dict, litellm_params: dict) -> dict: + """Merge model-cost metadata with deployment model_info for health checks. + + Model-specific health-check flags belong in model metadata, but Azure + deployment names can include suffixes that do not exist in the model map. + Try the deployment model first, then provider/base_model (for example + azure/gpt-chat-latest) when a deployment provides base_model metadata. + Deployment-level model_info overrides model-map metadata. + """ + metadata: dict = {} + candidates: list[str] = [] + deployment_model = litellm_params.get("model") + if isinstance(deployment_model, str): + candidates.append(deployment_model) + + base_model = model_info.get("base_model") + if isinstance(base_model, str): + candidates.append(base_model) + if isinstance(deployment_model, str) and "/" in deployment_model: + provider = deployment_model.split("/", 1)[0] + candidates.append(f"{provider}/{base_model}") + + for candidate in candidates: + if candidate in litellm.model_cost: + metadata = litellm.model_cost[candidate] + break + try: + from litellm.litellm_core_utils.get_model_cost_map import ( + GetModelCostMap, + ) + + local_model_cost = GetModelCostMap.load_local_model_cost_map() + if candidate in local_model_cost: + metadata = local_model_cost[candidate] + break + except Exception: + pass + try: + metadata = dict(litellm.get_model_info(model=candidate)) + break + except Exception: + continue + + return {**metadata, **dict(model_info)} + + def _should_inject_health_check_max_tokens( model_info: Mapping[str, object], mode: str | None ) -> bool: @@ -456,27 +502,33 @@ def _update_litellm_params_for_health_check( mode = _resolve_health_check_mode( model_info, litellm_params # any-ok: untyped router config dict ) + resolved_model_info = _resolve_health_check_model_info(model_info, litellm_params) litellm_params["messages"] = _get_random_llm_message() if _should_inject_health_check_max_tokens( - model_info, mode # any-ok: untyped router config dict + resolved_model_info, mode # any-ok: untyped router config dict ): _resolved_max_tokens = _resolve_health_check_max_tokens( - model_info, litellm_params + resolved_model_info, litellm_params ) if _resolved_max_tokens is not None: - litellm_params["max_tokens"] = _resolved_max_tokens + if resolved_model_info.get("map_max_tokens_to_max_completion_tokens"): + litellm_params["max_completion_tokens"] = _resolved_max_tokens + else: + litellm_params["max_tokens"] = _resolved_max_tokens # Per-model reasoning effort for health checks only (e.g. reasoning_effort=none). if mode in _HEALTH_CHECK_MODES_SUPPORTING_REASONING_EFFORT: - _hc_reasoning_effort = model_info.get("health_check_reasoning_effort", None) + _hc_reasoning_effort = resolved_model_info.get( + "health_check_reasoning_effort", None + ) if _hc_reasoning_effort is not None: litellm_params["reasoning_effort"] = _hc_reasoning_effort - _health_check_model = model_info.get("health_check_model", None) + _health_check_model = resolved_model_info.get("health_check_model", None) if _health_check_model is not None: litellm_params["model"] = _health_check_model if mode == "audio_speech": - litellm_params["voice"] = model_info.get("health_check_voice", "alloy") + litellm_params["voice"] = resolved_model_info.get("health_check_voice", "alloy") # Handle Bedrock region routing format: bedrock/region/model # This is needed because health checks bypass get_llm_provider() for the model param diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5f3f2294147..ed4eecd4a96 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4789,6 +4789,40 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/gpt-chat-latest": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 3e-05, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-openais-newest-chat-model-in-microsoft-foundry/4516848", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "map_max_tokens_to_max_completion_tokens": true + }, "azure/gpt-5-codex": { "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, diff --git a/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py index 7f837dd58b1..492b24b03ef 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -5,7 +5,10 @@ sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) ) +import litellm +from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig +from litellm.utils import _invalidate_model_cost_lowercase_map class TestAzureOpenAIConfig: @@ -54,3 +57,38 @@ def test_map_openai_params_with_preview_api_version(): assert config.map_openai_params( non_default_params, optional_params, model, drop_params, api_version ) + + +def test_map_openai_params_translates_max_tokens_for_flagged_model(monkeypatch): + """Models flagged in the cost map translate max_tokens -> max_completion_tokens. + + Azure gpt-chat-latest rejects `max_tokens` and requires + `max_completion_tokens`, but it does not route through the GPT-5/o-series + config. The `map_max_tokens_to_max_completion_tokens` flag drives the + translation for normal chat completions too (not just health checks). + """ + monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url="")) + _invalidate_model_cost_lowercase_map() + config = AzureOpenAIConfig() + optional_params = config.map_openai_params( + non_default_params={"max_tokens": 42}, + optional_params={}, + model="azure/gpt-chat-latest", + drop_params=False, + ) + assert optional_params["max_completion_tokens"] == 42 + assert "max_tokens" not in optional_params + + +def test_map_openai_params_keeps_max_tokens_for_unflagged_model(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url="")) + _invalidate_model_cost_lowercase_map() + config = AzureOpenAIConfig() + optional_params = config.map_openai_params( + non_default_params={"max_tokens": 42}, + optional_params={}, + model="azure/gpt-4o", + drop_params=False, + ) + assert optional_params["max_tokens"] == 42 + assert "max_completion_tokens" not in optional_params diff --git a/tests/test_litellm/proxy/test_health_check_max_tokens.py b/tests/test_litellm/proxy/test_health_check_max_tokens.py index e56eb9bfdd6..251390798f5 100644 --- a/tests/test_litellm/proxy/test_health_check_max_tokens.py +++ b/tests/test_litellm/proxy/test_health_check_max_tokens.py @@ -2,6 +2,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +import litellm +from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map from litellm.litellm_core_utils.health_check_helpers import HealthCheckHelpers from litellm.proxy import health_check as hc_module from litellm.proxy.health_check import ( @@ -9,6 +11,7 @@ from litellm.proxy.health_check import ( _resolve_health_check_mode, _update_litellm_params_for_health_check, ) +from litellm.utils import _invalidate_model_cost_lowercase_map @pytest.mark.asyncio @@ -516,3 +519,40 @@ def test_autodetected_embedding_skips_reasoning_effort(): assert "reasoning_effort" not in updated assert "max_tokens" not in updated + + +def test_azure_gpt_chat_latest_health_check_uses_metadata_max_completion_tokens( + monkeypatch, +): + """Azure gpt-chat-latest metadata chooses max_completion_tokens for health.""" + monkeypatch.setattr(hc_module, "BACKGROUND_HEALTH_CHECK_MAX_TOKENS", None) + monkeypatch.setattr(hc_module, "BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING", None) + monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url="")) + _invalidate_model_cost_lowercase_map() + model_info = {"base_model": "gpt-chat-latest"} + litellm_params = {"model": "azure/gpt-chat-latest-gs"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert updated["max_completion_tokens"] == 16 + assert "max_tokens" not in updated + + +def test_deployment_model_info_overrides_health_check_max_completion_tokens( + monkeypatch, +): + """Deployment model_info wins over model-map health-check metadata.""" + monkeypatch.setattr(hc_module, "BACKGROUND_HEALTH_CHECK_MAX_TOKENS", None) + monkeypatch.setattr(hc_module, "BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING", None) + monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url="")) + _invalidate_model_cost_lowercase_map() + model_info = { + "base_model": "gpt-chat-latest", + "map_max_tokens_to_max_completion_tokens": False, + } + litellm_params = {"model": "azure/gpt-chat-latest-gs"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert updated["max_tokens"] == 16 + assert "max_completion_tokens" not in updated diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index cf7afc1af68..a7f1e060354 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -860,6 +860,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_none_reasoning_effort": {"type": "boolean"}, "supports_xhigh_reasoning_effort": {"type": "boolean"}, "supports_max_reasoning_effort": {"type": "boolean"}, + "map_max_tokens_to_max_completion_tokens": {"type": "boolean"}, "supports_adaptive_thinking": {"type": "boolean"}, "supports_sampling_params": {"type": "boolean"}, "supports_service_tier": {"type": "boolean"},