diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 16fd042cb2f..99dbfc2c508 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -3,10 +3,8 @@ Support for gpt model family """ import json -import os from collections.abc import AsyncIterator, Coroutine, Iterator from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload -from urllib.parse import urlparse import httpx @@ -50,7 +48,7 @@ from litellm.types.utils import ( ) from litellm.utils import convert_to_model_response_object -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, should_preserve_cache_control_for_endpoint if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -394,21 +392,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): custom_llm_provider: str | None, api_base: str | None, ) -> bool: - """ - The generic `openai` provider also reaches OpenAI-compatible endpoints - (a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) via a custom - api_base. Those can understand cache_control, so it must survive there. - Real OpenAI cannot, so it is still stripped for an openai.com host. - """ - if custom_llm_provider != "openai": - return False - resolved_api_base = api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") - if not resolved_api_base: - return False - hostname: Final = urlparse(resolved_api_base).hostname - if hostname is None: - return False - return hostname != "openai.com" and not hostname.endswith(".openai.com") + """See `should_preserve_cache_control_for_endpoint`; the responses path + applies the same rule to the same deployment.""" + return should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base) def transform_request( self, diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 1b1ab80e85d..a93e7236dc5 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -11,6 +11,7 @@ import time import uuid from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Optional +from urllib.parse import urlparse import httpx import openai @@ -34,6 +35,32 @@ from litellm.llms.custom_httpx.http_handler import ( ) +def should_preserve_cache_control_for_endpoint( + custom_llm_provider: str | None, + api_base: str | None, +) -> bool: + """ + The generic `openai` provider also reaches OpenAI-compatible endpoints + (a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) via a custom + api_base. Those can understand cache_control, so it must survive there. + Real OpenAI cannot, so it is still stripped for an openai.com host. + + Shared by the chat completions and responses transformers so the two + paths make the same call for the same deployment. + """ + if custom_llm_provider != "openai": + return False + resolved_api_base: Final = ( + api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") + ) + if not resolved_api_base: + return False + hostname: Final = urlparse(resolved_api_base).hostname + if hostname is None: + return False + return hostname != "openai.com" and not hostname.endswith(".openai.com") + + def _get_client_init_params(cls: type) -> tuple[str, ...]: """Extract __init__ parameter names (excluding 'self') from a class.""" return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self") diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index b2a69564908..a58af2dfaec 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -18,7 +18,7 @@ from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, should_preserve_cache_control_for_endpoint OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 @@ -147,12 +147,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): OpenAI's Responses API rejects unknown fields on input content blocks with HTTP 400 ("Unknown parameter: 'input[0].content[0].cache_control'"). Chat Completions strips these in - `remove_cache_control_flag_from_messages_and_tools`; mirror that here. + `remove_cache_control_flag_from_messages_and_tools`; mirror that here, + including its carve-out for an openai-provider deployment on a custom + api_base, which may well understand cache_control. """ input = self._validate_input_param(input) tools = response_api_optional_request_params.get("tools") - input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools) + if not should_preserve_cache_control_for_endpoint(litellm_params.custom_llm_provider, litellm_params.api_base): + input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools) if tools is not None: response_api_optional_request_params["tools"] = tools final_request_params: Final = dict( @@ -631,7 +634,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): input = self._validate_input_param(input) tools = response_api_optional_request_params.get("tools") - input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools) + if not should_preserve_cache_control_for_endpoint(litellm_params.custom_llm_provider, litellm_params.api_base): + input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools) if tools is not None: response_api_optional_request_params["tools"] = tools data: Final = dict(ResponsesAPIRequestParams(model=model, input=input, **response_api_optional_request_params)) diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index c03c632363d..df3008dcb8a 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -120,7 +120,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_text, response_api_optional_request_params=optional_params, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -162,7 +162,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_with_cache_control, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -193,7 +193,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input="hi", response_api_optional_request_params={"tools": tools_with_cache_control}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -213,7 +213,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_clean, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -508,7 +508,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_text, response_api_optional_request_params=optional_params, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -538,7 +538,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_text, response_api_optional_request_params=optional_params, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -752,6 +752,132 @@ class TestOpenAIResponsesAPIConfig: assert "namespace" not in norm["input"][1] +class TestResponsesCacheControlPreservationForCustomEndpoint: + """ + Regression tests for https://github.com/BerriAI/litellm/issues/37474 + + The chat completions transformer keeps `cache_control` for the generic + `openai` provider on a custom api_base, because such an endpoint (a LiteLLM + proxy, vLLM, an Anthropic-compatible gateway) can understand it. The + responses transformer stripped it unconditionally, so the same deployment + lost prompt caching on whichever of the two paths a request happened to + take. Both paths now ask the same question. + """ + + def setup_method(self): + self.config = OpenAIResponsesAPIConfig() + self.model = "claude-sonnet-4" + + @pytest.fixture(autouse=True) + def _clean_openai_base_env(self, monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None, raising=False) + + @staticmethod + def _cache_controlled_input(): + return [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + + @staticmethod + def _cache_controlled_tools(): + return [ + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + "cache_control": {"type": "ephemeral"}, + } + ] + + def _transform(self, custom_llm_provider, api_base, tools=None): + return self.config.transform_responses_api_request( + model=self.model, + input=self._cache_controlled_input(), + response_api_optional_request_params={"tools": tools} if tools else {}, + litellm_params=GenericLiteLLMParams( + custom_llm_provider=custom_llm_provider, api_base=api_base + ), + headers={}, + ) + + def test_preserves_for_openai_provider_on_custom_api_base(self): + result = self._transform("openai", "http://localhost:4000/v1") + assert result["input"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + + def test_preserves_tool_cache_control_for_custom_api_base(self): + result = self._transform( + "openai", "http://localhost:4000/v1", tools=self._cache_controlled_tools() + ) + assert result["tools"][0]["cache_control"] == {"type": "ephemeral"} + + def test_strips_for_real_openai_without_api_base(self): + result = self._transform("openai", None) + assert "cache_control" not in result["input"][0]["content"][0] + + def test_strips_for_explicit_openai_host(self): + result = self._transform("openai", "https://api.openai.com/v1") + assert "cache_control" not in result["input"][0]["content"][0] + + def test_strips_for_non_openai_provider(self): + result = self._transform("azure", "https://example.openai.azure.com") + assert "cache_control" not in result["input"][0]["content"][0] + + def test_compact_request_follows_the_same_rule(self): + _, preserved = self.config.transform_compact_response_api_request( + model=self.model, + input=self._cache_controlled_input(), + response_api_optional_request_params={}, + api_base="http://localhost:4000/v1/responses", + litellm_params=GenericLiteLLMParams( + custom_llm_provider="openai", api_base="http://localhost:4000/v1" + ), + headers={}, + ) + assert preserved["input"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + + _, stripped = self.config.transform_compact_response_api_request( + model=self.model, + input=self._cache_controlled_input(), + response_api_optional_request_params={}, + api_base="https://api.openai.com/v1/responses", + litellm_params=GenericLiteLLMParams( + custom_llm_provider="openai", api_base="https://api.openai.com/v1" + ), + headers={}, + ) + assert "cache_control" not in stripped["input"][0]["content"][0] + + def test_chat_and_responses_paths_agree_for_the_same_deployment(self): + """The bug was the two paths disagreeing, so assert them together.""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + for provider, api_base, expected in [ + ("openai", "http://localhost:4000/v1", True), + ("openai", None, False), + ("openai", "https://api.openai.com/v1", False), + ("fireworks_ai", "https://api.fireworks.ai/inference/v1", False), + ]: + chat_keeps = OpenAIGPTConfig()._should_preserve_cache_control_for_endpoint( + provider, api_base + ) + responses_keeps = "cache_control" in self._transform(provider, api_base)["input"][0][ + "content" + ][0] + assert chat_keeps is expected, (provider, api_base) + assert responses_keeps is expected, (provider, api_base) + + class TestAzureResponsesAPIConfig: def setup_method(self): self.config = AzureOpenAIResponsesAPIConfig() diff --git a/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py b/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py index 534176e381a..3d0e87a73f7 100644 --- a/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py +++ b/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py @@ -15,6 +15,7 @@ import pytest from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.router import GenericLiteLLMParams from litellm.llms.perplexity.responses.transformation import PerplexityResponsesConfig from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams from litellm.types.utils import LlmProviders @@ -325,7 +326,7 @@ class TestPerplexityResponsesTransformation: model="preset/pro-search", input="What is AI?", response_api_optional_request_params={"temperature": 0.7}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -345,7 +346,7 @@ class TestPerplexityResponsesTransformation: model="preset/pro-search", input=list_input, response_api_optional_request_params={"temperature": 0.7}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -366,7 +367,7 @@ class TestPerplexityResponsesTransformation: model="openai/gpt-5.2", input=list_input, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -386,7 +387,7 @@ class TestPerplexityResponsesTransformation: model="openai/gpt-5.2", input=list_input, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -406,7 +407,7 @@ class TestPerplexityResponsesTransformation: model="openai/gpt-5.2", input=list_input, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, )