diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 23342550e17..af02235c342 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -1,12 +1,30 @@ """Support for OpenAI gpt-5 model family.""" -from typing import Optional +from typing import Optional, Union import litellm +from litellm.utils import _supports_factory from .gpt_transformation import OpenAIGPTConfig +def _normalize_reasoning_effort_for_chat_completion( + value: Union[str, dict, None], +) -> Optional[str]: + """Convert reasoning_effort to the string format expected by OpenAI chat completion API. + + The chat completion API expects a simple string: 'none', 'low', 'medium', 'high', or 'xhigh'. + Config/deployments may pass the Responses API format: {'effort': 'high', 'summary': 'detailed'}. + """ + if value is None: + return None + if isinstance(value, str): + return value + if isinstance(value, dict) and "effort" in value: + return value["effort"] + return None + + class OpenAIGPT5Config(OpenAIGPTConfig): """Configuration for gpt-5 models including GPT-5-Codex variants. @@ -23,53 +41,61 @@ class OpenAIGPT5Config(OpenAIGPTConfig): # Don't route it through GPT-5 reasoning-specific parameter restrictions. return "gpt-5" in model and "gpt-5-chat" not in model + @classmethod + def is_model_gpt_5_search_model(cls, model: str) -> bool: + """Check if the model is a GPT-5 search variant (e.g. gpt-5-search-api). + + Search-only models have a severely restricted parameter set compared to + regular GPT-5 models. They are identified by name convention (contain + both ``gpt-5`` and ``search``). Note: ``supports_web_search`` in model + info is a *different* concept — it indicates a model can *use* web + search as a tool, which many non-search-only models also support. + """ + return "gpt-5" in model and "search" in model + @classmethod def is_model_gpt_5_codex_model(cls, model: str) -> bool: """Check if the model is specifically a GPT-5 Codex variant.""" return "gpt-5-codex" in model - @classmethod - def is_model_gpt_5_1_codex_max_model(cls, model: str) -> bool: - """Check if the model is the gpt-5.1-codex-max variant.""" - model_name = model.split("/")[-1] # handle provider prefixes - return model_name == "gpt-5.1-codex-max" - - @classmethod - def is_model_gpt_5_1_model(cls, model: str) -> bool: - """Check if the model is a gpt-5.1, gpt-5.2, or gpt-5.4 chat variant. - - gpt-5.1/5.2/5.4 support temperature when reasoning_effort="none", - unlike base gpt-5 which only supports temperature=1. Excludes - pro variants which keep stricter knobs and chat-only variants - which only support temperature=1. - """ - model_name = model.split("/")[-1] - is_gpt_5_1 = model_name.startswith("gpt-5.1") - is_gpt_5_2 = ( - model_name.startswith("gpt-5.2") - and "pro" not in model_name - and not model_name.startswith("gpt-5.2-chat") - ) - is_gpt_5_4 = ( - model_name.startswith("gpt-5.4") - and "pro" not in model_name - and not model_name.startswith("gpt-5.4-chat") - ) - return is_gpt_5_1 or is_gpt_5_2 or is_gpt_5_4 - - @classmethod - def is_model_gpt_5_2_pro_model(cls, model: str) -> bool: - """Check if the model is the gpt-5.2-pro snapshot/alias.""" - model_name = model.split("/")[-1] - return model_name.startswith("gpt-5.2-pro") - @classmethod def is_model_gpt_5_2_model(cls, model: str) -> bool: """Check if the model is a gpt-5.2 variant (including pro).""" model_name = model.split("/")[-1] return model_name.startswith("gpt-5.2") or model_name.startswith("gpt-5.4") + @classmethod + def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool: + """Check if the model supports a specific reasoning_effort level. + + Looks up ``supports_{level}_reasoning_effort`` in the model map via + the shared ``_supports_factory`` helper. + Returns False for unknown models (safe fallback). + """ + return _supports_factory( + model=model, + custom_llm_provider=None, + key=f"supports_{level}_reasoning_effort", + ) + def get_supported_openai_params(self, model: str) -> list: + if self.is_model_gpt_5_search_model(model): + return [ + "max_tokens", + "max_completion_tokens", + "stream", + "stream_options", + "web_search_options", + "service_tier", + "safety_identifier", + "response_format", + "user", + "store", + "verbosity", + "max_retries", + "extra_headers", + ] + from litellm.utils import supports_tool_choice base_gpt_series_params = super().get_supported_openai_params(model=model) @@ -89,9 +115,10 @@ class OpenAIGPT5Config(OpenAIGPTConfig): "web_search_options", ] - # gpt-5.1/5.2/5.4 support logprobs, top_p, top_logprobs when reasoning_effort="none" - if not self.is_model_gpt_5_1_model(model): + # gpt-5.1/5.2 support logprobs, top_p, top_logprobs when reasoning_effort="none" + if not self._supports_reasoning_effort_level(model, "none"): non_supported_params.extend(["logprobs", "top_p", "top_logprobs"]) + return [ param for param in base_gpt_series_params @@ -105,15 +132,34 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model: str, drop_params: bool, ) -> dict: - reasoning_effort = ( + if self.is_model_gpt_5_search_model(model): + if "max_tokens" in non_default_params: + optional_params["max_completion_tokens"] = non_default_params.pop( + "max_tokens" + ) + return super()._map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + ) + + # Normalize reasoning_effort: chat completion API expects a string, not a dict + # (e.g. {'effort': 'high', 'summary': 'detailed'} -> 'high') + raw_reasoning_effort = ( non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") ) + normalized = _normalize_reasoning_effort_for_chat_completion(raw_reasoning_effort) + if raw_reasoning_effort is not None and normalized is not None: + if "reasoning_effort" in non_default_params: + non_default_params["reasoning_effort"] = normalized + if "reasoning_effort" in optional_params: + optional_params["reasoning_effort"] = normalized + + reasoning_effort = normalized or raw_reasoning_effort if reasoning_effort is not None and reasoning_effort == "xhigh": - if not ( - self.is_model_gpt_5_1_codex_max_model(model) - or self.is_model_gpt_5_2_model(model) - ): + if not self._supports_reasoning_effort_level(model, "xhigh"): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) else: @@ -133,8 +179,9 @@ class OpenAIGPT5Config(OpenAIGPTConfig): "max_tokens" ) - # gpt-5.1/5.2/5.4 support logprobs, top_p, top_logprobs only when reasoning_effort="none" - if self.is_model_gpt_5_1_model(model): + # gpt-5.1/5.2 support logprobs, top_p, top_logprobs only when reasoning_effort="none" + supports_none = self._supports_reasoning_effort_level(model, "none") + if supports_none: sampling_params = ["logprobs", "top_logprobs", "top_p"] has_sampling = any(p in non_default_params for p in sampling_params) if has_sampling and reasoning_effort not in (None, "none"): @@ -151,14 +198,11 @@ class OpenAIGPT5Config(OpenAIGPTConfig): status_code=400, ) - if "temperature" in non_default_params: temperature_value: Optional[float] = non_default_params.pop("temperature") if temperature_value is not None: - is_gpt_5_1 = self.is_model_gpt_5_1_model(model) - - # gpt-5.1 supports any temperature when reasoning_effort="none" (or not specified, as it defaults to "none") - if is_gpt_5_1 and (reasoning_effort == "none" or reasoning_effort is None): + # models supporting reasoning_effort="none" also support flexible temperature + if supports_none and (reasoning_effort == "none" or reasoning_effort is None): optional_params["temperature"] = temperature_value elif temperature_value == 1: optional_params["temperature"] = temperature_value diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index 30647f58687..a995fb47c4a 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -24,6 +24,7 @@ from litellm.utils import ( supports_system_messages, ) +from .gpt_5_transformation import _normalize_reasoning_effort_for_chat_completion from .gpt_transformation import OpenAIGPTConfig @@ -104,6 +105,18 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): model: str, drop_params: bool, ): + # Normalize reasoning_effort: chat completion API expects a string, not a dict + raw_reasoning_effort = ( + non_default_params.get("reasoning_effort") + or optional_params.get("reasoning_effort") + ) + normalized = _normalize_reasoning_effort_for_chat_completion(raw_reasoning_effort) + if raw_reasoning_effort is not None and normalized is not None: + if "reasoning_effort" in non_default_params: + non_default_params["reasoning_effort"] = normalized + if "reasoning_effort" in optional_params: + optional_params["reasoning_effort"] = normalized + if "max_tokens" in non_default_params: optional_params["max_completion_tokens"] = non_default_params.pop( "max_tokens" diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 386f264a4dd..415325d0da7 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -1,8 +1,8 @@ import pytest import litellm -from litellm.llms.openai.openai import OpenAIConfig from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.openai.openai import OpenAIConfig @pytest.fixture() @@ -260,18 +260,21 @@ def test_gpt5_drops_reasoning_effort_xhigh_when_requested(config: OpenAIConfig): # GPT-5.1 temperature handling tests def test_gpt5_1_model_detection(gpt5_config: OpenAIGPT5Config): - """Test that GPT-5.1 models are correctly detected.""" - assert gpt5_config.is_model_gpt_5_1_model("gpt-5.1") - assert gpt5_config.is_model_gpt_5_1_model("gpt-5.1-codex") - assert gpt5_config.is_model_gpt_5_1_model("gpt-5.1-codex-max") - assert gpt5_config.is_model_gpt_5_1_model("gpt-5.1-chat") - assert gpt5_config.is_model_gpt_5_1_model("gpt-5.2") - assert gpt5_config.is_model_gpt_5_1_model("gpt-5.2-2025-12-11") - assert gpt5_config.is_model_gpt_5_1_model("gpt-5.2-chat-latest") - assert not gpt5_config.is_model_gpt_5_1_model("gpt-5.2-pro") - assert not gpt5_config.is_model_gpt_5_1_model("gpt-5") - assert not gpt5_config.is_model_gpt_5_1_model("gpt-5-mini") - assert not gpt5_config.is_model_gpt_5_1_model("gpt-5-codex") + """Test that models supporting reasoning_effort='none' are correctly detected via model map.""" + # gpt-5.1 and gpt-5.2 chat variants support none + assert gpt5_config._supports_reasoning_effort_level("gpt-5.1", "none") + assert gpt5_config._supports_reasoning_effort_level("gpt-5.1-2025-11-13", "none") + assert gpt5_config._supports_reasoning_effort_level("gpt-5.1-chat-latest", "none") + assert gpt5_config._supports_reasoning_effort_level("gpt-5.2", "none") + assert gpt5_config._supports_reasoning_effort_level("gpt-5.2-2025-12-11", "none") + # codex/pro/chat variants do not support none + assert not gpt5_config._supports_reasoning_effort_level("gpt-5.1-codex", "none") + assert not gpt5_config._supports_reasoning_effort_level("gpt-5.1-codex-max", "none") + assert not gpt5_config._supports_reasoning_effort_level("gpt-5.2-chat-latest", "none") + assert not gpt5_config._supports_reasoning_effort_level("gpt-5.2-pro", "none") + assert not gpt5_config._supports_reasoning_effort_level("gpt-5", "none") + assert not gpt5_config._supports_reasoning_effort_level("gpt-5-mini", "none") + assert not gpt5_config._supports_reasoning_effort_level("gpt-5-codex", "none") def test_gpt5_1_temperature_with_reasoning_effort_none(config: OpenAIConfig): @@ -301,6 +304,61 @@ def test_gpt5_2_temperature_with_reasoning_effort_none(config: OpenAIConfig): assert params["reasoning_effort"] == "none" +def test_gpt5_4_allows_reasoning_effort_xhigh(config: OpenAIConfig): + params = config.map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, + optional_params={}, + model="gpt-5.4", + drop_params=False, + ) + assert params["reasoning_effort"] == "xhigh" + + +def test_gpt5_4_pro_allows_reasoning_effort_xhigh(config: OpenAIConfig): + params = config.map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, + optional_params={}, + model="gpt-5.4-pro", + drop_params=False, + ) + assert params["reasoning_effort"] == "xhigh" + + +def test_gpt5_normalizes_reasoning_effort_dict_to_string(config: OpenAIConfig): + """Chat completion API expects reasoning_effort as a string, not a dict. + + Config/deployments may pass Responses API format: {'effort': 'high', 'summary': 'detailed'}. + """ + params = config.map_openai_params( + non_default_params={"reasoning_effort": {"effort": "high", "summary": "detailed"}}, + optional_params={}, + model="gpt-5.4", + drop_params=False, + ) + assert params["reasoning_effort"] == "high" + + +def test_gpt5_normalizes_reasoning_effort_dict_from_optional_params(config: OpenAIConfig): + """reasoning_effort dict in optional_params (e.g. from model config) is normalized.""" + params = config.map_openai_params( + non_default_params={}, + optional_params={"reasoning_effort": {"effort": "medium", "summary": "detailed"}}, + model="gpt-5.4", + drop_params=False, + ) + assert params["reasoning_effort"] == "medium" + + +def test_gpt5_4_pro_rejects_non_default_temperature(config: OpenAIConfig): + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"temperature": 0.5}, + optional_params={}, + model="gpt-5.4-pro", + drop_params=False, + ) + + def test_gpt5_1_temperature_without_reasoning_effort(config: OpenAIConfig): """Test that GPT-5.1 supports any temperature when reasoning_effort is not specified. @@ -395,7 +453,38 @@ def test_gpt5_temperature_still_restricted(config: OpenAIConfig): assert params["temperature"] == 1.0 -def test_gpt5_2_pro_allows_reasoning_effort_xhigh(config: OpenAIConfig): +def test_gpt5_2_chat_temperature_restricted(config: OpenAIConfig): + """Test that gpt-5.2-chat only supports temperature=1, like base gpt-5. + + Regression test for https://github.com/BerriAI/litellm/issues/21911 + """ + # gpt-5.2-chat should reject non-1 temperature when drop_params=False + for model in ["gpt-5.2-chat", "gpt-5.2-chat-latest"]: + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"temperature": 0.7}, + optional_params={}, + model=model, + drop_params=False, + ) + + # temperature=1 should still work + params = config.map_openai_params( + non_default_params={"temperature": 1.0}, + optional_params={}, + model=model, + drop_params=False, + ) + assert params["temperature"] == 1.0 + + # drop_params=True should silently drop non-1 temperature + params = config.map_openai_params( + non_default_params={"temperature": 0.5}, + optional_params={}, + model=model, + drop_params=True, + ) + assert "temperature" not in params params = config.map_openai_params( non_default_params={"reasoning_effort": "xhigh"}, optional_params={}, @@ -414,3 +503,174 @@ def test_gpt5_2_allows_reasoning_effort_xhigh(config: OpenAIConfig): drop_params=False, ) assert params["reasoning_effort"] == "xhigh" + + +# GPT-5-Search specific tests +def test_gpt5_search_model_detection(gpt5_config: OpenAIGPT5Config): + """Test that GPT-5 search models are correctly detected.""" + assert gpt5_config.is_model_gpt_5_search_model("gpt-5-search-api") + assert gpt5_config.is_model_gpt_5_search_model("gpt-5-search-mini-api") + + assert not gpt5_config.is_model_gpt_5_search_model("gpt-5") + assert not gpt5_config.is_model_gpt_5_search_model("gpt-5-codex") + assert not gpt5_config.is_model_gpt_5_search_model("gpt-5-mini") + + +def test_gpt5_search_supported_params(gpt5_config: OpenAIGPT5Config): + """Test that search models do NOT list reasoning/tool params as supported.""" + supported = gpt5_config.get_supported_openai_params(model="gpt-5-search-api") + rejected = [ + "logit_bias", + "modalities", + "prediction", + "n", + "seed", + "temperature", + "tools", + "tool_choice", + "function_call", + "functions", + "parallel_tool_calls", + "audio", + "reasoning_effort", + ] + for param in rejected: + assert param not in supported, f"{param} should not be supported for search models" + + +def test_gpt5_search_has_expected_params(gpt5_config: OpenAIGPT5Config): + """Test that search models DO list the correct supported params.""" + supported = gpt5_config.get_supported_openai_params(model="gpt-5-search-api") + expected = [ + "max_tokens", + "max_completion_tokens", + "stream", + "stream_options", + "web_search_options", + "service_tier", + "response_format", + "user", + "store", + "verbosity", + "extra_headers", + ] + for param in expected: + assert param in supported, f"{param} should be supported for search models" + + +def test_gpt5_search_maps_max_tokens(config: OpenAIConfig): + """Test that search models map max_tokens -> max_completion_tokens.""" + params = config.map_openai_params( + non_default_params={"max_tokens": 200}, + optional_params={}, + model="gpt-5-search-api", + drop_params=False, + ) + assert params["max_completion_tokens"] == 200 + assert "max_tokens" not in params + + +def test_gpt5_search_drops_unsupported_params(config: OpenAIConfig): + """Test that search models drop unsupported params via map_openai_params.""" + params = config.map_openai_params( + non_default_params={"n": 2, "temperature": 0.7, "tools": [{"type": "function"}]}, + optional_params={}, + model="gpt-5-search-api", + drop_params=True, + ) + assert "n" not in params + assert "temperature" not in params + assert "tools" not in params +# GPT-5 unsupported params audit (validated via direct API calls) +def test_gpt5_rejects_params_unsupported_by_openai(config: OpenAIConfig): + """Params that OpenAI rejects for all GPT-5 reasoning models.""" + rejected_params = [ + "logit_bias", + "modalities", + "prediction", + "audio", + "web_search_options", + ] + for model in ["gpt-5", "gpt-5-mini", "gpt-5-codex", "gpt-5.1", "gpt-5.2"]: + supported = config.get_supported_openai_params(model=model) + for param in rejected_params: + assert param not in supported, ( + f"{param} should not be supported for {model}" + ) + + +def test_gpt5_1_supports_logprobs_top_p(config: OpenAIConfig): + """gpt-5.1/5.2 support logprobs, top_p, top_logprobs when reasoning_effort='none'.""" + for model in ["gpt-5.1", "gpt-5.2"]: + supported = config.get_supported_openai_params(model=model) + assert "logprobs" in supported, f"logprobs should be supported for {model}" + assert "top_p" in supported, f"top_p should be supported for {model}" + assert "top_logprobs" in supported, f"top_logprobs should be supported for {model}" + + +def test_gpt5_base_does_not_support_logprobs_top_p(config: OpenAIConfig): + """Base gpt-5/gpt-5-mini do NOT support logprobs, top_p, top_logprobs.""" + for model in ["gpt-5", "gpt-5-mini", "gpt-5-codex"]: + supported = config.get_supported_openai_params(model=model) + assert "logprobs" not in supported, f"logprobs should not be supported for {model}" + assert "top_p" not in supported, f"top_p should not be supported for {model}" + assert "top_logprobs" not in supported, f"top_logprobs should not be supported for {model}" + + +def test_gpt5_1_logprobs_passthrough(config: OpenAIConfig): + """Test that logprobs passes through for gpt-5.1.""" + params = config.map_openai_params( + non_default_params={"logprobs": True, "top_logprobs": 3}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + assert params["logprobs"] is True + assert params["top_logprobs"] == 3 + + +def test_gpt5_1_top_p_passthrough(config: OpenAIConfig): + """Test that top_p passes through for gpt-5.1.""" + params = config.map_openai_params( + non_default_params={"top_p": 0.9}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + assert params["top_p"] == 0.9 + + +def test_gpt5_1_logprobs_rejected_with_reasoning_effort(config: OpenAIConfig): + """logprobs/top_p/top_logprobs are rejected when reasoning_effort != 'none'.""" + for effort in ["low", "medium", "high"]: + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"logprobs": True, "reasoning_effort": effort}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + + +def test_gpt5_1_top_p_rejected_with_reasoning_effort(config: OpenAIConfig): + """top_p is rejected when reasoning_effort != 'none'.""" + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"top_p": 0.9, "reasoning_effort": "high"}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + + +def test_gpt5_1_logprobs_dropped_with_reasoning_effort(config: OpenAIConfig): + """logprobs/top_p are dropped when reasoning_effort != 'none' and drop_params=True.""" + params = config.map_openai_params( + non_default_params={"logprobs": True, "top_p": 0.9, "reasoning_effort": "high"}, + optional_params={}, + model="gpt-5.1", + drop_params=True, + ) + assert "logprobs" not in params + assert "top_p" not in params + assert params["reasoning_effort"] == "high" \ No newline at end of file