From 01347ffb3f06e567322eebe832e527a78852d6fa Mon Sep 17 00:00:00 2001 From: Vineeth Sai Date: Wed, 19 Aug 2026 09:58:19 -0700 Subject: [PATCH] Keep cache_control on the responses path for a custom openai endpoint The chat completions transformer already carves out the generic `openai` provider on a non-openai.com api_base: a LiteLLM proxy, vLLM or an Anthropic-compatible gateway can understand `cache_control`, so it must survive there, and only a real openai.com host gets it stripped. transform_responses_api_request and transform_compact_response_api_request had no such carve-out and called remove_cache_control_flag_from_input_and_tools unconditionally. The same deployment therefore kept its cache breakpoints or lost them depending on which of the two paths a request happened to take, and a client that routes through the Responses API never got a cache hit. Move the predicate into llms/openai/common_utils so both transformers ask the same question of the same deployment, and apply it on the responses path. The regression tests use GenericLiteLLMParams, which is what the signature asks for and what production passes; a few existing tests handed in a bare dict, which only worked while nothing read the object. --- .../llms/openai/chat/gpt_transformation.py | 22 +-- litellm/llms/openai/common_utils.py | 27 ++++ .../llms/openai/responses/transformation.py | 12 +- .../test_openai_responses_transformation.py | 138 +++++++++++++++++- ...est_perplexity_responses_transformation.py | 11 +- 5 files changed, 177 insertions(+), 33 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 16fd042cb2f..99dbfc2c508 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -3,10 +3,8 @@ Support for gpt model family """ import json -import os from collections.abc import AsyncIterator, Coroutine, Iterator from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload -from urllib.parse import urlparse import httpx @@ -50,7 +48,7 @@ from litellm.types.utils import ( ) from litellm.utils import convert_to_model_response_object -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, should_preserve_cache_control_for_endpoint if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -394,21 +392,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): custom_llm_provider: str | None, api_base: str | None, ) -> bool: - """ - The generic `openai` provider also reaches OpenAI-compatible endpoints - (a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) via a custom - api_base. Those can understand cache_control, so it must survive there. - Real OpenAI cannot, so it is still stripped for an openai.com host. - """ - if custom_llm_provider != "openai": - return False - resolved_api_base = api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") - if not resolved_api_base: - return False - hostname: Final = urlparse(resolved_api_base).hostname - if hostname is None: - return False - return hostname != "openai.com" and not hostname.endswith(".openai.com") + """See `should_preserve_cache_control_for_endpoint`; the responses path + applies the same rule to the same deployment.""" + return should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base) def transform_request( self, diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 1b1ab80e85d..a93e7236dc5 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -11,6 +11,7 @@ import time import uuid from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Optional +from urllib.parse import urlparse import httpx import openai @@ -34,6 +35,32 @@ from litellm.llms.custom_httpx.http_handler import ( ) +def should_preserve_cache_control_for_endpoint( + custom_llm_provider: str | None, + api_base: str | None, +) -> bool: + """ + The generic `openai` provider also reaches OpenAI-compatible endpoints + (a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) via a custom + api_base. Those can understand cache_control, so it must survive there. + Real OpenAI cannot, so it is still stripped for an openai.com host. + + Shared by the chat completions and responses transformers so the two + paths make the same call for the same deployment. + """ + if custom_llm_provider != "openai": + return False + resolved_api_base: Final = ( + api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") + ) + if not resolved_api_base: + return False + hostname: Final = urlparse(resolved_api_base).hostname + if hostname is None: + return False + return hostname != "openai.com" and not hostname.endswith(".openai.com") + + def _get_client_init_params(cls: type) -> tuple[str, ...]: """Extract __init__ parameter names (excluding 'self') from a class.""" return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self") diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index b2a69564908..a58af2dfaec 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -18,7 +18,7 @@ from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, should_preserve_cache_control_for_endpoint OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 @@ -147,12 +147,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): OpenAI's Responses API rejects unknown fields on input content blocks with HTTP 400 ("Unknown parameter: 'input[0].content[0].cache_control'"). Chat Completions strips these in - `remove_cache_control_flag_from_messages_and_tools`; mirror that here. + `remove_cache_control_flag_from_messages_and_tools`; mirror that here, + including its carve-out for an openai-provider deployment on a custom + api_base, which may well understand cache_control. """ input = self._validate_input_param(input) tools = response_api_optional_request_params.get("tools") - input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools) + if not should_preserve_cache_control_for_endpoint(litellm_params.custom_llm_provider, litellm_params.api_base): + input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools) if tools is not None: response_api_optional_request_params["tools"] = tools final_request_params: Final = dict( @@ -631,7 +634,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): input = self._validate_input_param(input) tools = response_api_optional_request_params.get("tools") - input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools) + if not should_preserve_cache_control_for_endpoint(litellm_params.custom_llm_provider, litellm_params.api_base): + input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools) if tools is not None: response_api_optional_request_params["tools"] = tools data: Final = dict(ResponsesAPIRequestParams(model=model, input=input, **response_api_optional_request_params)) diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index 151c51f1ca0..96128dbdb02 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -125,7 +125,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_text, response_api_optional_request_params=optional_params, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -167,7 +167,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_with_cache_control, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -198,7 +198,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input="hi", response_api_optional_request_params={"tools": tools_with_cache_control}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -218,7 +218,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_clean, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -513,7 +513,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_text, response_api_optional_request_params=optional_params, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -543,7 +543,7 @@ class TestOpenAIResponsesAPIConfig: model=self.model, input=input_text, response_api_optional_request_params=optional_params, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -757,6 +757,132 @@ class TestOpenAIResponsesAPIConfig: assert "namespace" not in norm["input"][1] +class TestResponsesCacheControlPreservationForCustomEndpoint: + """ + Regression tests for https://github.com/BerriAI/litellm/issues/37474 + + The chat completions transformer keeps `cache_control` for the generic + `openai` provider on a custom api_base, because such an endpoint (a LiteLLM + proxy, vLLM, an Anthropic-compatible gateway) can understand it. The + responses transformer stripped it unconditionally, so the same deployment + lost prompt caching on whichever of the two paths a request happened to + take. Both paths now ask the same question. + """ + + def setup_method(self): + self.config = OpenAIResponsesAPIConfig() + self.model = "claude-sonnet-4" + + @pytest.fixture(autouse=True) + def _clean_openai_base_env(self, monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None, raising=False) + + @staticmethod + def _cache_controlled_input(): + return [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + + @staticmethod + def _cache_controlled_tools(): + return [ + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + "cache_control": {"type": "ephemeral"}, + } + ] + + def _transform(self, custom_llm_provider, api_base, tools=None): + return self.config.transform_responses_api_request( + model=self.model, + input=self._cache_controlled_input(), + response_api_optional_request_params={"tools": tools} if tools else {}, + litellm_params=GenericLiteLLMParams( + custom_llm_provider=custom_llm_provider, api_base=api_base + ), + headers={}, + ) + + def test_preserves_for_openai_provider_on_custom_api_base(self): + result = self._transform("openai", "http://localhost:4000/v1") + assert result["input"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + + def test_preserves_tool_cache_control_for_custom_api_base(self): + result = self._transform( + "openai", "http://localhost:4000/v1", tools=self._cache_controlled_tools() + ) + assert result["tools"][0]["cache_control"] == {"type": "ephemeral"} + + def test_strips_for_real_openai_without_api_base(self): + result = self._transform("openai", None) + assert "cache_control" not in result["input"][0]["content"][0] + + def test_strips_for_explicit_openai_host(self): + result = self._transform("openai", "https://api.openai.com/v1") + assert "cache_control" not in result["input"][0]["content"][0] + + def test_strips_for_non_openai_provider(self): + result = self._transform("azure", "https://example.openai.azure.com") + assert "cache_control" not in result["input"][0]["content"][0] + + def test_compact_request_follows_the_same_rule(self): + _, preserved = self.config.transform_compact_response_api_request( + model=self.model, + input=self._cache_controlled_input(), + response_api_optional_request_params={}, + api_base="http://localhost:4000/v1/responses", + litellm_params=GenericLiteLLMParams( + custom_llm_provider="openai", api_base="http://localhost:4000/v1" + ), + headers={}, + ) + assert preserved["input"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + + _, stripped = self.config.transform_compact_response_api_request( + model=self.model, + input=self._cache_controlled_input(), + response_api_optional_request_params={}, + api_base="https://api.openai.com/v1/responses", + litellm_params=GenericLiteLLMParams( + custom_llm_provider="openai", api_base="https://api.openai.com/v1" + ), + headers={}, + ) + assert "cache_control" not in stripped["input"][0]["content"][0] + + def test_chat_and_responses_paths_agree_for_the_same_deployment(self): + """The bug was the two paths disagreeing, so assert them together.""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + for provider, api_base, expected in [ + ("openai", "http://localhost:4000/v1", True), + ("openai", None, False), + ("openai", "https://api.openai.com/v1", False), + ("fireworks_ai", "https://api.fireworks.ai/inference/v1", False), + ]: + chat_keeps = OpenAIGPTConfig()._should_preserve_cache_control_for_endpoint( + provider, api_base + ) + responses_keeps = "cache_control" in self._transform(provider, api_base)["input"][0][ + "content" + ][0] + assert chat_keeps is expected, (provider, api_base) + assert responses_keeps is expected, (provider, api_base) + + class TestAzureResponsesAPIConfig: def setup_method(self): self.config = AzureOpenAIResponsesAPIConfig() diff --git a/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py b/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py index a3ec81c569c..4f9bfe415d3 100644 --- a/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py +++ b/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py @@ -18,6 +18,7 @@ sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.router import GenericLiteLLMParams from litellm.llms.perplexity.responses.transformation import PerplexityResponsesConfig from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams from litellm.types.utils import LlmProviders @@ -328,7 +329,7 @@ class TestPerplexityResponsesTransformation: model="preset/pro-search", input="What is AI?", response_api_optional_request_params={"temperature": 0.7}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -348,7 +349,7 @@ class TestPerplexityResponsesTransformation: model="preset/pro-search", input=list_input, response_api_optional_request_params={"temperature": 0.7}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -369,7 +370,7 @@ class TestPerplexityResponsesTransformation: model="openai/gpt-5.2", input=list_input, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -389,7 +390,7 @@ class TestPerplexityResponsesTransformation: model="openai/gpt-5.2", input=list_input, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, ) @@ -409,7 +410,7 @@ class TestPerplexityResponsesTransformation: model="openai/gpt-5.2", input=list_input, response_api_optional_request_params={}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers={}, )