diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 9afc6331d96..c99895bd50a 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -53,7 +53,7 @@ from litellm.types.utils import ( ) from litellm.utils import convert_to_model_response_object -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, should_preserve_cache_control_for_endpoint if TYPE_CHECKING: import tiktoken @@ -422,15 +422,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): custom_llm_provider: str | None, api_base: str | None, ) -> bool: - """ - The generic `openai` provider also reaches OpenAI-compatible endpoints - (a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) via a custom - api_base. Those can understand cache_control, so it must survive there. - Real OpenAI cannot, so it is still stripped for an openai.com host. - """ - return custom_llm_provider == "openai" and not self._targets_openai_hosted_endpoint( - custom_llm_provider, api_base - ) + """See `should_preserve_cache_control_for_endpoint`; the responses path + applies the same rule to the same deployment.""" + return should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base) def _flattened_tools_update_for_openai( self, diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 2db6d78a218..1dae08cf67d 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -35,6 +35,32 @@ from litellm.llms.custom_httpx.http_handler import ( ) +def should_preserve_cache_control_for_endpoint( + custom_llm_provider: str | None, + api_base: str | None, +) -> bool: + """ + The generic `openai` provider also reaches OpenAI-compatible endpoints + (a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) via a custom + api_base. Those can understand cache_control, so it must survive there. + Real OpenAI cannot, so it is still stripped for an openai.com host. + + Shared by the chat completions and responses transformers so the two + paths make the same call for the same deployment. + """ + if custom_llm_provider != "openai": + return False + resolved_api_base: Final = ( + api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") + ) + if not resolved_api_base: + return False + hostname: Final = urlsplit(resolved_api_base).hostname + if hostname is None: + return False + return hostname != "openai.com" and not hostname.endswith(".openai.com") + + def _get_client_init_params(cls: type) -> tuple[str, ...]: """Extract __init__ parameter names (excluding 'self') from a class.""" return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self") diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 926de3e8854..ec68d476db9 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -26,7 +26,7 @@ from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, should_preserve_cache_control_for_endpoint from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 @@ -289,9 +289,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): litellm_params: GenericLiteLLMParams, ) -> tuple[str | ResponseInputParam, Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None]: validated_input: Final = self._validate_input_param(input) - stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools( - model=model, input=validated_input, tools=tools - ) + # An OpenAI-compatible endpoint behind the generic `openai` provider can understand + # cache_control, and the chat path already keeps it there. Stripping it here anyway + # is what makes the same deployment lose prompt caching on the responses route. + if should_preserve_cache_control_for_endpoint(litellm_params.custom_llm_provider, litellm_params.api_base): + stripped_input, stripped_tools = validated_input, tools + else: + stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools( + model=model, input=validated_input, tools=tools + ) object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools) sanitized_tools: Final = self._flatten_tool_schema_combinators_for_openai( model=model, tools=object_schema_tools, litellm_params=litellm_params diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index c5902b32a06..df2644f36ae 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -121,7 +121,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_text, response_api_optional_request_params=optional_params, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -163,7 +163,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_with_cache_control, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -194,7 +194,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input="hi", response_api_optional_request_params={"tools": tools_with_cache_control}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -214,7 +214,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_clean, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -257,7 +257,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=replayed_input, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -294,7 +294,7 @@ class TestOpenAIResponsesAPIConfig: model="openrouter/some-model", input=replayed_input, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -538,7 +538,7 @@ class TestOpenAIResponsesAPIConfig: result = self.config.get_complete_url( api_base=api_base, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), ) assert result == "https://custom-openai.example.com/v1/responses" @@ -547,7 +547,7 @@ class TestOpenAIResponsesAPIConfig: with patch("litellm.api_base", "https://litellm-api-base.example.com/v1"): result = self.config.get_complete_url( api_base=None, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), ) assert result == "https://litellm-api-base.example.com/v1/responses" @@ -560,7 +560,7 @@ class TestOpenAIResponsesAPIConfig: ): result = self.config.get_complete_url( api_base=None, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), ) assert result == "https://env-api-base.example.com/v1/responses" @@ -573,7 +573,7 @@ class TestOpenAIResponsesAPIConfig: ): result = self.config.get_complete_url( api_base=None, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), ) assert result == "https://api.openai.com/v1/responses" @@ -583,7 +583,7 @@ class TestOpenAIResponsesAPIConfig: result = self.config.get_complete_url( api_base=api_base, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), ) assert result == "https://custom-openai.example.com/v1/responses" @@ -697,7 +697,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_text, response_api_optional_request_params=optional_params, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -727,7 +727,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_text, response_api_optional_request_params=optional_params, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -941,6 +941,121 @@ class TestOpenAIResponsesAPIConfig: assert "namespace" not in norm["input"][1] +class TestResponsesCacheControlPreservationForCustomEndpoint: + """ + Regression tests for https://github.com/BerriAI/litellm/issues/37474 + + The chat completions transformer keeps `cache_control` for the generic + `openai` provider on a custom api_base, because such an endpoint (a LiteLLM + proxy, vLLM, an Anthropic-compatible gateway) can understand it. The + responses transformer stripped it unconditionally, so the same deployment + lost prompt caching on whichever of the two paths a request happened to + take. Both paths now ask the same question. + """ + + def setup_method(self): + self.config = OpenAIResponsesAPIConfig() + self.model = "claude-sonnet-4" + + @pytest.fixture(autouse=True) + def _clean_openai_base_env(self, monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None, raising=False) + + @staticmethod + def _cache_controlled_input(): + return [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + + @staticmethod + def _cache_controlled_tools(): + return [ + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + "cache_control": {"type": "ephemeral"}, + } + ] + + def _transform(self, custom_llm_provider, api_base, tools=None): + return self.config.transform_responses_api_request( + model=self.model, + input=self._cache_controlled_input(), + response_api_optional_request_params={"tools": tools} if tools else {}, + litellm_params=GenericLiteLLMParams(custom_llm_provider=custom_llm_provider, api_base=api_base), + headers={}, + ) + + def test_preserves_for_openai_provider_on_custom_api_base(self): + result = self._transform("openai", "http://localhost:4000/v1") + assert result["input"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + + def test_preserves_tool_cache_control_for_custom_api_base(self): + result = self._transform("openai", "http://localhost:4000/v1", tools=self._cache_controlled_tools()) + assert result["tools"][0]["cache_control"] == {"type": "ephemeral"} + + def test_strips_for_real_openai_without_api_base(self): + result = self._transform("openai", None) + assert "cache_control" not in result["input"][0]["content"][0] + + def test_strips_for_explicit_openai_host(self): + result = self._transform("openai", "https://api.openai.com/v1") + assert "cache_control" not in result["input"][0]["content"][0] + + def test_strips_for_non_openai_provider(self): + result = self._transform("azure", "https://example.openai.azure.com") + assert "cache_control" not in result["input"][0]["content"][0] + + def test_compact_request_follows_the_same_rule(self): + _, preserved = self.config.transform_compact_response_api_request( + model=self.model, + input=self._cache_controlled_input(), + response_api_optional_request_params={}, + api_base="http://localhost:4000/v1/responses", + litellm_params=GenericLiteLLMParams(custom_llm_provider="openai", api_base="http://localhost:4000/v1"), + headers={}, + ) + assert preserved["input"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + + _, stripped = self.config.transform_compact_response_api_request( + model=self.model, + input=self._cache_controlled_input(), + response_api_optional_request_params={}, + api_base="https://api.openai.com/v1/responses", + litellm_params=GenericLiteLLMParams(custom_llm_provider="openai", api_base="https://api.openai.com/v1"), + headers={}, + ) + assert "cache_control" not in stripped["input"][0]["content"][0] + + def test_chat_and_responses_paths_agree_for_the_same_deployment(self): + """The bug was the two paths disagreeing, so assert them together.""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + for provider, api_base, expected in [ + ("openai", "http://localhost:4000/v1", True), + ("openai", None, False), + ("openai", "https://api.openai.com/v1", False), + ("fireworks_ai", "https://api.fireworks.ai/inference/v1", False), + ]: + chat_keeps = OpenAIGPTConfig()._should_preserve_cache_control_for_endpoint(provider, api_base) + responses_keeps = "cache_control" in self._transform(provider, api_base)["input"][0]["content"][0] + assert chat_keeps is expected, (provider, api_base) + assert responses_keeps is expected, (provider, api_base) + + + class TestAzureResponsesAPIConfig: def setup_method(self): self.config = AzureOpenAIResponsesAPIConfig() @@ -1766,7 +1881,7 @@ class TestPromptCacheOptionsOnResponsesPath: model="gpt-5.6", input="hi", response_api_optional_request_params=mapped, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) assert result["prompt_cache_options"] == self.OPTIONS @@ -1788,7 +1903,7 @@ class TestPromptCacheOptionsOnResponsesPath: } ], response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) assert result["input"][0]["content"][0] == { diff --git a/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py b/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py index 534176e381a..7f29546180e 100644 --- a/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py +++ b/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py @@ -15,6 +15,7 @@ import pytest from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.router import GenericLiteLLMParams from litellm.llms.perplexity.responses.transformation import PerplexityResponsesConfig from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams from litellm.types.utils import LlmProviders @@ -325,7 +326,7 @@ class TestPerplexityResponsesTransformation: model="preset/pro-search", input="What is AI?", response_api_optional_request_params={"temperature": 0.7}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -345,7 +346,7 @@ class TestPerplexityResponsesTransformation: model="preset/pro-search", input=list_input, response_api_optional_request_params={"temperature": 0.7}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -366,7 +367,7 @@ class TestPerplexityResponsesTransformation: model="openai/gpt-5.2", input=list_input, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -386,7 +387,7 @@ class TestPerplexityResponsesTransformation: model="openai/gpt-5.2", input=list_input, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -406,7 +407,7 @@ class TestPerplexityResponsesTransformation: model="openai/gpt-5.2", input=list_input, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -416,18 +417,18 @@ class TestPerplexityResponsesTransformation: """Correct endpoint URL""" config = PerplexityResponsesConfig() - url = config.get_complete_url(api_base=None, litellm_params={}) + url = config.get_complete_url(api_base=None, litellm_params=GenericLiteLLMParams()) assert url == "https://api.perplexity.ai/v1/responses" custom_url = config.get_complete_url( api_base="https://custom.perplexity.ai", - litellm_params={}, + litellm_params=GenericLiteLLMParams(), ) assert custom_url == "https://custom.perplexity.ai/v1/responses" url_with_slash = config.get_complete_url( api_base="https://api.perplexity.ai/", - litellm_params={}, + litellm_params=GenericLiteLLMParams(), ) assert url_with_slash == "https://api.perplexity.ai/v1/responses"