Merge 01347ffb3f into 80843ae7cb

litellm/llms/openai/chat/gpt_transformation.py

@@ -3,10 +3,8 @@ Support for gpt model family
 """
 
 import json
-import os
 from collections.abc import AsyncIterator, Coroutine, Iterator
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload
-from urllib.parse import urlparse
 
 import httpx
 
@@ -50,7 +48,7 @@ from litellm.types.utils import (
 )
 from litellm.utils import convert_to_model_response_object
 
-from ..common_utils import OpenAIError
+from ..common_utils import OpenAIError, should_preserve_cache_control_for_endpoint
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -394,21 +392,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         custom_llm_provider: str | None,
         api_base: str | None,
     ) -> bool:
-        """
-        The generic `openai` provider also reaches OpenAI-compatible endpoints
-        (a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) via a custom
-        api_base. Those can understand cache_control, so it must survive there.
-        Real OpenAI cannot, so it is still stripped for an openai.com host.
-        """
-        if custom_llm_provider != "openai":
-            return False
-        resolved_api_base = api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE")
-        if not resolved_api_base:
-            return False
-        hostname: Final = urlparse(resolved_api_base).hostname
-        if hostname is None:
-            return False
-        return hostname != "openai.com" and not hostname.endswith(".openai.com")
+        """See `should_preserve_cache_control_for_endpoint`; the responses path
+        applies the same rule to the same deployment."""
+        return should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base)
 
     def transform_request(
         self,

litellm/llms/openai/common_utils.py

@@ -11,6 +11,7 @@ import time
 import uuid
 from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Optional
+from urllib.parse import urlparse
 
 import httpx
 import openai
@@ -34,6 +35,32 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 
 
+def should_preserve_cache_control_for_endpoint(
+    custom_llm_provider: str | None,
+    api_base: str | None,
+) -> bool:
+    """
+    The generic `openai` provider also reaches OpenAI-compatible endpoints
+    (a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) via a custom
+    api_base. Those can understand cache_control, so it must survive there.
+    Real OpenAI cannot, so it is still stripped for an openai.com host.
+
+    Shared by the chat completions and responses transformers so the two
+    paths make the same call for the same deployment.
+    """
+    if custom_llm_provider != "openai":
+        return False
+    resolved_api_base: Final = (
+        api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE")
+    )
+    if not resolved_api_base:
+        return False
+    hostname: Final = urlparse(resolved_api_base).hostname
+    if hostname is None:
+        return False
+    return hostname != "openai.com" and not hostname.endswith(".openai.com")
+
+
 def _get_client_init_params(cls: type) -> tuple[str, ...]:
     """Extract __init__ parameter names (excluding 'self') from a class."""
     return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")

litellm/llms/openai/responses/transformation.py

@@ -18,7 +18,7 @@ from litellm.types.responses.main import *
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import LlmProviders
 
-from ..common_utils import OpenAIError
+from ..common_utils import OpenAIError, should_preserve_cache_control_for_endpoint
 
 OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
 
@@ -147,12 +147,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         OpenAI's Responses API rejects unknown fields on input content blocks
         with HTTP 400 ("Unknown parameter: 'input[0].content[0].cache_control'").
         Chat Completions strips these in
-        `remove_cache_control_flag_from_messages_and_tools`; mirror that here.
+        `remove_cache_control_flag_from_messages_and_tools`; mirror that here,
+        including its carve-out for an openai-provider deployment on a custom
+        api_base, which may well understand cache_control.
         """
 
         input = self._validate_input_param(input)
         tools = response_api_optional_request_params.get("tools")
-        input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools)
+        if not should_preserve_cache_control_for_endpoint(litellm_params.custom_llm_provider, litellm_params.api_base):
+            input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools)
         if tools is not None:
             response_api_optional_request_params["tools"] = tools
         final_request_params: Final = dict(
@@ -631,7 +634,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
 
         input = self._validate_input_param(input)
         tools = response_api_optional_request_params.get("tools")
-        input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools)
+        if not should_preserve_cache_control_for_endpoint(litellm_params.custom_llm_provider, litellm_params.api_base):
+            input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools)
         if tools is not None:
             response_api_optional_request_params["tools"] = tools
         data: Final = dict(ResponsesAPIRequestParams(model=model, input=input, **response_api_optional_request_params))

tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py

@@ -120,7 +120,7 @@ class TestOpenAIResponsesAPIConfig:
             model=self.model,
             input=input_text,
             response_api_optional_request_params=optional_params,
-            litellm_params={},
+            litellm_params=GenericLiteLLMParams(),
             headers={},
         )
 
@@ -162,7 +162,7 @@ class TestOpenAIResponsesAPIConfig:
             model=self.model,
             input=input_with_cache_control,
             response_api_optional_request_params={},
-            litellm_params={},
+            litellm_params=GenericLiteLLMParams(),
             headers={},
         )
 
@@ -193,7 +193,7 @@ class TestOpenAIResponsesAPIConfig:
             model=self.model,
             input="hi",
             response_api_optional_request_params={"tools": tools_with_cache_control},
-            litellm_params={},
+            litellm_params=GenericLiteLLMParams(),
             headers={},
         )
 
@@ -213,7 +213,7 @@ class TestOpenAIResponsesAPIConfig:
             model=self.model,
             input=input_clean,
             response_api_optional_request_params={},
-            litellm_params={},
+            litellm_params=GenericLiteLLMParams(),
             headers={},
         )
 
@@ -508,7 +508,7 @@ class TestOpenAIResponsesAPIConfig:
             model=self.model,
             input=input_text,
             response_api_optional_request_params=optional_params,
-            litellm_params={},
+            litellm_params=GenericLiteLLMParams(),
             headers={},
         )
 
@@ -538,7 +538,7 @@ class TestOpenAIResponsesAPIConfig:
                 model=self.model,
                 input=input_text,
                 response_api_optional_request_params=optional_params,
-                litellm_params={},
+                litellm_params=GenericLiteLLMParams(),
                 headers={},
             )
 
@@ -752,6 +752,132 @@ class TestOpenAIResponsesAPIConfig:
         assert "namespace" not in norm["input"][1]
 
 
+class TestResponsesCacheControlPreservationForCustomEndpoint:
+    """
+    Regression tests for https://github.com/BerriAI/litellm/issues/37474
+
+    The chat completions transformer keeps `cache_control` for the generic
+    `openai` provider on a custom api_base, because such an endpoint (a LiteLLM
+    proxy, vLLM, an Anthropic-compatible gateway) can understand it. The
+    responses transformer stripped it unconditionally, so the same deployment
+    lost prompt caching on whichever of the two paths a request happened to
+    take. Both paths now ask the same question.
+    """
+
+    def setup_method(self):
+        self.config = OpenAIResponsesAPIConfig()
+        self.model = "claude-sonnet-4"
+
+    @pytest.fixture(autouse=True)
+    def _clean_openai_base_env(self, monkeypatch):
+        monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
+        monkeypatch.delenv("OPENAI_API_BASE", raising=False)
+        monkeypatch.setattr(litellm, "api_base", None, raising=False)
+
+    @staticmethod
+    def _cache_controlled_input():
+        return [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "input_text",
+                        "text": "Hello",
+                        "cache_control": {"type": "ephemeral"},
+                    }
+                ],
+            }
+        ]
+
+    @staticmethod
+    def _cache_controlled_tools():
+        return [
+            {
+                "type": "function",
+                "name": "get_weather",
+                "parameters": {"type": "object", "properties": {}},
+                "cache_control": {"type": "ephemeral"},
+            }
+        ]
+
+    def _transform(self, custom_llm_provider, api_base, tools=None):
+        return self.config.transform_responses_api_request(
+            model=self.model,
+            input=self._cache_controlled_input(),
+            response_api_optional_request_params={"tools": tools} if tools else {},
+            litellm_params=GenericLiteLLMParams(
+                custom_llm_provider=custom_llm_provider, api_base=api_base
+            ),
+            headers={},
+        )
+
+    def test_preserves_for_openai_provider_on_custom_api_base(self):
+        result = self._transform("openai", "http://localhost:4000/v1")
+        assert result["input"][0]["content"][0]["cache_control"] == {"type": "ephemeral"}
+
+    def test_preserves_tool_cache_control_for_custom_api_base(self):
+        result = self._transform(
+            "openai", "http://localhost:4000/v1", tools=self._cache_controlled_tools()
+        )
+        assert result["tools"][0]["cache_control"] == {"type": "ephemeral"}
+
+    def test_strips_for_real_openai_without_api_base(self):
+        result = self._transform("openai", None)
+        assert "cache_control" not in result["input"][0]["content"][0]
+
+    def test_strips_for_explicit_openai_host(self):
+        result = self._transform("openai", "https://api.openai.com/v1")
+        assert "cache_control" not in result["input"][0]["content"][0]
+
+    def test_strips_for_non_openai_provider(self):
+        result = self._transform("azure", "https://example.openai.azure.com")
+        assert "cache_control" not in result["input"][0]["content"][0]
+
+    def test_compact_request_follows_the_same_rule(self):
+        _, preserved = self.config.transform_compact_response_api_request(
+            model=self.model,
+            input=self._cache_controlled_input(),
+            response_api_optional_request_params={},
+            api_base="http://localhost:4000/v1/responses",
+            litellm_params=GenericLiteLLMParams(
+                custom_llm_provider="openai", api_base="http://localhost:4000/v1"
+            ),
+            headers={},
+        )
+        assert preserved["input"][0]["content"][0]["cache_control"] == {"type": "ephemeral"}
+
+        _, stripped = self.config.transform_compact_response_api_request(
+            model=self.model,
+            input=self._cache_controlled_input(),
+            response_api_optional_request_params={},
+            api_base="https://api.openai.com/v1/responses",
+            litellm_params=GenericLiteLLMParams(
+                custom_llm_provider="openai", api_base="https://api.openai.com/v1"
+            ),
+            headers={},
+        )
+        assert "cache_control" not in stripped["input"][0]["content"][0]
+
+    def test_chat_and_responses_paths_agree_for_the_same_deployment(self):
+        """The bug was the two paths disagreeing, so assert them together."""
+        from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+
+        for provider, api_base, expected in [
+            ("openai", "http://localhost:4000/v1", True),
+            ("openai", None, False),
+            ("openai", "https://api.openai.com/v1", False),
+            ("fireworks_ai", "https://api.fireworks.ai/inference/v1", False),
+        ]:
+            chat_keeps = OpenAIGPTConfig()._should_preserve_cache_control_for_endpoint(
+                provider, api_base
+            )
+            responses_keeps = "cache_control" in self._transform(provider, api_base)["input"][0][
+                "content"
+            ][0]
+            assert chat_keeps is expected, (provider, api_base)
+            assert responses_keeps is expected, (provider, api_base)
+
+
 class TestAzureResponsesAPIConfig:
     def setup_method(self):
         self.config = AzureOpenAIResponsesAPIConfig()

tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py

@@ -15,6 +15,7 @@ import pytest
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.router import GenericLiteLLMParams
 from litellm.llms.perplexity.responses.transformation import PerplexityResponsesConfig
 from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
 from litellm.types.utils import LlmProviders
@@ -325,7 +326,7 @@ class TestPerplexityResponsesTransformation:
             model="preset/pro-search",
             input="What is AI?",
             response_api_optional_request_params={"temperature": 0.7},
-            litellm_params={},
+            litellm_params=GenericLiteLLMParams(),
             headers={},
         )
 
@@ -345,7 +346,7 @@ class TestPerplexityResponsesTransformation:
             model="preset/pro-search",
             input=list_input,
             response_api_optional_request_params={"temperature": 0.7},
-            litellm_params={},
+            litellm_params=GenericLiteLLMParams(),
             headers={},
         )
 
@@ -366,7 +367,7 @@ class TestPerplexityResponsesTransformation:
             model="openai/gpt-5.2",
             input=list_input,
             response_api_optional_request_params={},
-            litellm_params={},
+            litellm_params=GenericLiteLLMParams(),
             headers={},
         )
 
@@ -386,7 +387,7 @@ class TestPerplexityResponsesTransformation:
             model="openai/gpt-5.2",
             input=list_input,
             response_api_optional_request_params={},
-            litellm_params={},
+            litellm_params=GenericLiteLLMParams(),
             headers={},
         )
 
@@ -406,7 +407,7 @@ class TestPerplexityResponsesTransformation:
             model="openai/gpt-5.2",
             input=list_input,
             response_api_optional_request_params={},
-            litellm_params={},
+            litellm_params=GenericLiteLLMParams(),
             headers={},
         )