diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index 0292c73f998..2967d5e3945 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -4,7 +4,10 @@ from typing import List import litellm from litellm.exceptions import UnsupportedParamsError -from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.openai.chat.gpt_5_transformation import ( + OpenAIGPT5Config, + _get_effort_level, +) from litellm.types.llms.openai import AllMessageValues from .gpt_transformation import AzureOpenAIConfig @@ -15,6 +18,21 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): GPT5_SERIES_ROUTE = "gpt5_series/" + @classmethod + def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool: + """Override to handle gpt5_series/ prefix used for Azure routing. + + The parent class calls ``_supports_factory(model, custom_llm_provider=None)`` + which fails to resolve ``gpt5_series/gpt-5.1`` to the correct Azure model + entry. Strip the prefix and prepend ``azure/`` so the lookup finds + ``azure/gpt-5.1`` in model_prices_and_context_window.json. + """ + if model.startswith(cls.GPT5_SERIES_ROUTE): + model = "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :] + elif not model.startswith("azure/"): + model = "azure/" + model + return super()._supports_reasoning_effort_level(model, level) + @classmethod def is_model_gpt_5_model(cls, model: str) -> bool: """Check if the Azure model string refers to a gpt-5 variant. @@ -46,7 +64,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): # Only gpt-5.2+ has been verified to support logprobs on Azure. # The base OpenAI class includes logprobs for gpt-5.1+, but Azure # hasn't verified support for gpt-5.1, so remove them unless gpt-5.2/5.4+. - if self.is_model_gpt_5_1_model(model) and not self.is_model_gpt_5_2_model(model): + if self._supports_reasoning_effort_level(model, "none") and not self.is_model_gpt_5_2_model(model): params = [p for p in params if p not in ["logprobs", "top_logprobs"]] elif self.is_model_gpt_5_2_model(model): azure_supported_params = ["logprobs", "top_logprobs"] @@ -66,23 +84,21 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") ) + effective_effort = _get_effort_level(reasoning_effort_value) # gpt-5.1/5.2/5.4 support reasoning_effort='none', but other gpt-5 models don't # See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning - model_for_check = model.replace(self.GPT5_SERIES_ROUTE, "") - is_gpt_5_1 = self._supports_reasoning_effort_level( - model_for_check, "none" - ) + supports_none = self._supports_reasoning_effort_level(model, "none") - if reasoning_effort_value == "none" and not is_gpt_5_1: + if effective_effort == "none" and not supports_none: if litellm.drop_params is True or ( drop_params is not None and drop_params is True ): non_default_params = non_default_params.copy() optional_params = optional_params.copy() - if non_default_params.get("reasoning_effort") == "none": + if _get_effort_level(non_default_params.get("reasoning_effort")) == "none": non_default_params.pop("reasoning_effort") - if optional_params.get("reasoning_effort") == "none": + if _get_effort_level(optional_params.get("reasoning_effort")) == "none": optional_params.pop("reasoning_effort") else: raise UnsupportedParamsError( @@ -100,14 +116,24 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): self, non_default_params=non_default_params, optional_params=optional_params, - model=model_for_check, + model=model, drop_params=drop_params, ) - # Only drop reasoning_effort='none' for non-gpt-5.1/5.2/5.4 models - if result.get("reasoning_effort") == "none" and not is_gpt_5_1: + # Only drop reasoning_effort='none' for models that don't support it + result_effort = _get_effort_level(result.get("reasoning_effort")) + if result_effort == "none" and not supports_none: result.pop("reasoning_effort") + # Azure Chat Completions: gpt-5.4+ does not support tools + reasoning together. + # Drop reasoning_effort when both are present (OpenAI routes to Responses API; Azure does not). + if self.is_model_gpt_5_4_plus_model(model): + has_tools = bool( + non_default_params.get("tools") or optional_params.get("tools") + ) + if has_tools and result_effort not in (None, "none"): + result.pop("reasoning_effort", None) + return result def transform_request( diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 9e1c2cb21c5..fe56b850ed8 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -25,6 +25,22 @@ def _normalize_reasoning_effort_for_chat_completion( return None +def _get_effort_level(value: Union[str, dict, None]) -> Optional[str]: + """Extract the effective effort level from reasoning_effort (string or dict). + + Use this for guards that compare effort level (e.g. xhigh validation, "none" checks). + Ensures dict inputs like {"effort": "none", "summary": "detailed"} are correctly + treated as effort="none" for validation purposes. + """ + if value is None: + return None + if isinstance(value, str): + return value + if isinstance(value, dict) and "effort" in value: + return value["effort"] + return None + + class OpenAIGPT5Config(OpenAIGPTConfig): """Configuration for gpt-5 models including GPT-5-Codex variants. @@ -76,6 +92,19 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model_name = model.split("/")[-1] return model_name.startswith("gpt-5.4") + @classmethod + def is_model_gpt_5_4_plus_model(cls, model: str) -> bool: + """Check if the model is gpt-5.4 or newer (5.4, 5.5, 5.6, etc., including pro).""" + model_name = model.split("/")[-1] + if not model_name.startswith("gpt-5."): + return False + try: + version_str = model_name.replace("gpt-5.", "").split("-")[0] + major = version_str.split(".")[0] + return int(major) >= 4 + except (ValueError, IndexError): + return False + @classmethod def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool: """Check if the model supports a specific reasoning_effort level. @@ -156,38 +185,32 @@ class OpenAIGPT5Config(OpenAIGPTConfig): drop_params=drop_params, ) - # Normalize reasoning_effort: chat completion API expects a string, not a dict - # (e.g. {'effort': 'high'} -> 'high') - # BUT: preserve dict format if it has additional fields like 'summary' for Responses API + # Get raw reasoning_effort and effective effort level for all guards. + # Use effective_effort (extracted string) for xhigh validation, "none" checks, and + # tool/sampling guards — dict inputs like {"effort": "none", "summary": "detailed"} + # must be treated as effort="none" to avoid incorrect tool-drop or sampling errors. raw_reasoning_effort = ( non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") ) - - # Only normalize if it's a simple dict with just 'effort' key - # Preserve dict format if it has additional fields (e.g., 'summary') for Responses API - should_normalize = False - if isinstance(raw_reasoning_effort, dict): - # Only normalize if dict has only 'effort' key (or is empty) - if set(raw_reasoning_effort.keys()) == {"effort"} or len(raw_reasoning_effort) == 0: - should_normalize = True - elif isinstance(raw_reasoning_effort, str): - # String format is already normalized - should_normalize = False - - if should_normalize: + effective_effort = _get_effort_level(raw_reasoning_effort) + + # Normalize to string for Chat Completions API when dict has only "effort". + # Preserve full dict (e.g. {"effort": "high", "summary": "detailed"}) for Responses API. + if isinstance(raw_reasoning_effort, dict) and set(raw_reasoning_effort.keys()) <= {"effort"}: normalized = _normalize_reasoning_effort_for_chat_completion(raw_reasoning_effort) - if raw_reasoning_effort is not None and normalized is not None: + if normalized is not None: if "reasoning_effort" in non_default_params: non_default_params["reasoning_effort"] = normalized if "reasoning_effort" in optional_params: optional_params["reasoning_effort"] = normalized - else: - # Keep the original format (string or dict with additional fields) - normalized = raw_reasoning_effort - reasoning_effort = normalized or raw_reasoning_effort - if reasoning_effort is not None and reasoning_effort == "xhigh": + reasoning_effort = ( + non_default_params.get("reasoning_effort") + or optional_params.get("reasoning_effort") + or raw_reasoning_effort + ) + if effective_effort is not None and effective_effort == "xhigh": if not self._supports_reasoning_effort_level(model, "xhigh"): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) @@ -215,23 +238,10 @@ class OpenAIGPT5Config(OpenAIGPTConfig): has_tools = bool( non_default_params.get("tools") or optional_params.get("tools") ) - if has_tools and reasoning_effort not in (None, "none"): + if has_tools and effective_effort not in (None, "none"): # Check if this will be routed to Responses API # If so, keep reasoning_effort; otherwise drop it for chat completions API - model_name = model.split("/")[-1] - will_route_to_responses = False - if model_name.startswith("gpt-5."): - try: - version_str = model_name.replace("gpt-5.", "").split("-")[0] - if "." in version_str: - major_version = int(version_str.split(".")[0]) - else: - major_version = int(version_str) - will_route_to_responses = major_version >= 4 - except (ValueError, IndexError): - pass - - if not will_route_to_responses: + if not self.is_model_gpt_5_4_plus_model(model): non_default_params.pop("reasoning_effort", None) optional_params.pop("reasoning_effort", None) reasoning_effort = None @@ -241,7 +251,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if supports_none: sampling_params = ["logprobs", "top_logprobs", "top_p"] has_sampling = any(p in non_default_params for p in sampling_params) - if has_sampling and reasoning_effort not in (None, "none"): + if has_sampling and effective_effort not in (None, "none"): if litellm.drop_params or drop_params: for p in sampling_params: non_default_params.pop(p, None) @@ -251,7 +261,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): "gpt-5.1/5.2/5.4 only support logprobs, top_p, top_logprobs when " "reasoning_effort='none'. Current reasoning_effort='{}'. " "To drop unsupported params set `litellm.drop_params = True`" - ).format(reasoning_effort), + ).format(effective_effort), status_code=400, ) @@ -259,7 +269,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): temperature_value: Optional[float] = non_default_params.pop("temperature") if temperature_value is not None: # models supporting reasoning_effort="none" also support flexible temperature - if supports_none and (reasoning_effort == "none" or reasoning_effort is None): + if supports_none and (effective_effort == "none" or effective_effort is None): optional_params["temperature"] = temperature_value elif temperature_value == 1: optional_params["temperature"] = temperature_value diff --git a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py index 25f3d1364f6..635359563ba 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py @@ -192,6 +192,23 @@ def test_azure_gpt5_1_series_temperature_handling(config: AzureOpenAIGPT5Config) assert params["temperature"] == 0.6 +def test_azure_gpt5_4_drops_reasoning_effort_when_tools_present(config: AzureOpenAIGPT5Config): + """Azure Chat Completions: gpt-5.4+ drops reasoning_effort when tools are present. + + OpenAI routes tools+reasoning to Responses API; Azure does not, so we drop reasoning_effort. + """ + tools = [{"type": "function", "function": {"name": "test", "description": "test"}}] + params = config.map_openai_params( + non_default_params={"reasoning_effort": "high", "tools": tools}, + optional_params={}, + model="gpt5_series/gpt-5.4", + drop_params=False, + api_version="2024-05-01-preview", + ) + assert "reasoning_effort" not in params + assert params["tools"] == tools + + def test_azure_gpt5_reasoning_effort_none_error(config: AzureOpenAIGPT5Config): """Test that Azure GPT-5 (non-5.1) raises error for reasoning_effort='none' when drop_params=False.""" with pytest.raises(litellm.utils.UnsupportedParamsError): diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index ea2a109e889..b45d3a24a38 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -426,3 +426,94 @@ class TestGPT5ReasoningEffortPreservation: assert reasoning["effort"] == "high" assert reasoning["summary"] == "detailed" assert reasoning["generate_summary"] == "concise" + + def test_reasoning_effort_dict_xhigh_triggers_validation(self): + """xhigh-dict: effective effort is extracted for model-support validation. + + When reasoning_effort={"effort": "xhigh", "summary": "detailed"} is passed to a model + that doesn't support xhigh (e.g. gpt-5.1), the xhigh guard must fire. + """ + import litellm + + non_default_params = {"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}} + optional_params = {} + + with pytest.raises(litellm.utils.UnsupportedParamsError): + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.1", + drop_params=False, + ) + + def test_reasoning_effort_dict_xhigh_dropped_when_requested(self): + """xhigh-dict with drop_params=True: reasoning_effort is dropped.""" + non_default_params = {"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.1", + drop_params=True, + ) + + assert "reasoning_effort" not in non_default_params + + def test_reasoning_effort_dict_none_treated_as_none_for_tools(self): + """none-dict: {"effort": "none", "summary": "detailed"} is treated as effort=none. + + Tool-drop guard should NOT fire; reasoning_effort should be kept. + """ + tools = [{"type": "function", "function": {"name": "test", "description": "test"}}] + non_default_params = {"reasoning_effort": {"effort": "none", "summary": "detailed"}, "tools": tools} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.4", + drop_params=False, + ) + + assert non_default_params.get("reasoning_effort") == {"effort": "none", "summary": "detailed"} + assert non_default_params.get("tools") == tools + + def test_reasoning_effort_dict_none_treated_as_none_for_sampling(self): + """none-dict: {"effort": "none", "summary": "detailed"} allows logprobs/top_p. + + Sampling-param guard should NOT fire; logprobs should be kept. + """ + non_default_params = { + "reasoning_effort": {"effort": "none", "summary": "detailed"}, + "logprobs": True, + } + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.1", + drop_params=False, + ) + + assert non_default_params.get("reasoning_effort") == {"effort": "none", "summary": "detailed"} + assert non_default_params.get("logprobs") is True + + def test_reasoning_effort_dict_none_allows_temperature(self): + """none-dict: {"effort": "none", "summary": "detailed"} allows non-default temperature.""" + non_default_params = { + "reasoning_effort": {"effort": "none", "summary": "detailed"}, + "temperature": 0.5, + } + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.1", + drop_params=False, + ) + + assert optional_params.get("temperature") == 0.5 + assert non_default_params.get("reasoning_effort") == {"effort": "none", "summary": "detailed"} diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index f12e73aaeb1..f1793e8ebf6 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -324,10 +324,11 @@ def test_gpt5_4_pro_allows_reasoning_effort_xhigh(config: OpenAIConfig): assert params["reasoning_effort"] == "xhigh" -def test_gpt5_normalizes_reasoning_effort_dict_to_string(config: OpenAIConfig): - """Chat completion API expects reasoning_effort as a string, not a dict. +def test_gpt5_preserves_reasoning_effort_dict_with_summary(config: OpenAIConfig): + """Dict with summary/generate_summary is preserved for Responses API. Config/deployments may pass Responses API format: {'effort': 'high', 'summary': 'detailed'}. + We preserve the full dict so it reaches the Responses API transformation. """ params = config.map_openai_params( non_default_params={"reasoning_effort": {"effort": "high", "summary": "detailed"}}, @@ -335,18 +336,82 @@ def test_gpt5_normalizes_reasoning_effort_dict_to_string(config: OpenAIConfig): model="gpt-5.4", drop_params=False, ) - assert params["reasoning_effort"] == "high" + assert params["reasoning_effort"] == {"effort": "high", "summary": "detailed"} -def test_gpt5_normalizes_reasoning_effort_dict_from_optional_params(config: OpenAIConfig): - """reasoning_effort dict in optional_params (e.g. from model config) is normalized.""" +def test_gpt5_xhigh_dict_triggers_validation(config: OpenAIConfig): + """Dict with effort='xhigh' triggers xhigh model-support validation. + + Regression: when reasoning_effort is a dict, effective_effort must be used for + the xhigh guard so validation is not silently skipped. + """ + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + + +def test_gpt5_xhigh_dict_accepted_for_supported_model(config: OpenAIConfig): + """Dict with effort='xhigh' passes through for gpt-5.4+.""" + params = config.map_openai_params( + non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, + optional_params={}, + model="gpt-5.4", + drop_params=False, + ) + assert params["reasoning_effort"] == {"effort": "xhigh", "summary": "detailed"} + + +def test_gpt5_none_dict_with_tools_no_tool_drop(config: OpenAIConfig): + """Dict with effort='none' and tools: no tool-drop, reasoning_effort preserved. + + Regression: effective_effort='none' must be used for tool-drop guard so + {"effort": "none", "summary": "detailed"} is not incorrectly treated as non-none. + """ + tools = [{"type": "function", "function": {"name": "test", "description": "test"}}] + params = config.map_openai_params( + non_default_params={"reasoning_effort": {"effort": "none", "summary": "detailed"}, "tools": tools}, + optional_params={}, + model="gpt-5.4", + drop_params=False, + ) + assert params["reasoning_effort"] == {"effort": "none", "summary": "detailed"} + assert params["tools"] == tools + + +def test_gpt5_none_dict_with_sampling_params_allowed(config: OpenAIConfig): + """Dict with effort='none' allows logprobs/top_p/top_logprobs. + + Regression: effective_effort='none' must be used for sampling guard so + {"effort": "none", "summary": "detailed"} does not incorrectly trigger sampling errors. + """ + params = config.map_openai_params( + non_default_params={ + "reasoning_effort": {"effort": "none", "summary": "detailed"}, + "logprobs": True, + "top_p": 0.9, + }, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + assert params["reasoning_effort"] == {"effort": "none", "summary": "detailed"} + assert params["logprobs"] is True + assert params["top_p"] == 0.9 + + +def test_gpt5_preserves_reasoning_effort_dict_with_summary_from_optional_params(config: OpenAIConfig): + """reasoning_effort dict with summary in optional_params is preserved.""" params = config.map_openai_params( non_default_params={}, optional_params={"reasoning_effort": {"effort": "medium", "summary": "detailed"}}, model="gpt-5.4", drop_params=False, ) - assert params["reasoning_effort"] == "medium" + assert params["reasoning_effort"] == {"effort": "medium", "summary": "detailed"} def test_gpt5_4_drops_reasoning_effort_when_tools_present(config: OpenAIConfig):