mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(responses): keep cache_control for a custom openai-compatible endpoint
The chat completions transformer keeps `cache_control` when the generic
`openai` provider points at a custom api_base, because such an endpoint
(a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) can understand
it. `_prepared_input_and_tools` on the responses path stripped it
unconditionally, so the same deployment lost prompt caching on whichever
of the two routes a request happened to take, silently and at full price.
The rule now lives once, as `should_preserve_cache_control_for_endpoint`
in `llms/openai/common_utils.py`, and both transformers call it.
`OpenAIGPTConfig._should_preserve_cache_control_for_endpoint` delegates
to it, so the chat path's behaviour is unchanged.
Tests: `TestResponsesCacheControlPreservationForCustomEndpoint` covers
input blocks, tools, the compact request, a real openai.com host, no
api_base at all, and a non-openai provider, plus one case that asserts
the two paths agree for the same deployment. The existing calls that
passed `litellm_params={}` now pass `GenericLiteLLMParams()`, which is
the declared type.
tests/test_litellm/llms/openai/responses, .../perplexity/responses and
.../openai/chat: 241 passed.
This commit is contained in:
parent
ee1a6407cb
commit
dcfb26905b
5 changed files with 179 additions and 37 deletions
|
|
@ -53,7 +53,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
from ..common_utils import OpenAIError, should_preserve_cache_control_for_endpoint
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
|
@ -422,15 +422,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
custom_llm_provider: str | None,
|
||||
api_base: str | None,
|
||||
) -> bool:
|
||||
"""
|
||||
The generic `openai` provider also reaches OpenAI-compatible endpoints
|
||||
(a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) via a custom
|
||||
api_base. Those can understand cache_control, so it must survive there.
|
||||
Real OpenAI cannot, so it is still stripped for an openai.com host.
|
||||
"""
|
||||
return custom_llm_provider == "openai" and not self._targets_openai_hosted_endpoint(
|
||||
custom_llm_provider, api_base
|
||||
)
|
||||
"""See `should_preserve_cache_control_for_endpoint`; the responses path
|
||||
applies the same rule to the same deployment."""
|
||||
return should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base)
|
||||
|
||||
def _flattened_tools_update_for_openai(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -35,6 +35,32 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
)
|
||||
|
||||
|
||||
def should_preserve_cache_control_for_endpoint(
|
||||
custom_llm_provider: str | None,
|
||||
api_base: str | None,
|
||||
) -> bool:
|
||||
"""
|
||||
The generic `openai` provider also reaches OpenAI-compatible endpoints
|
||||
(a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) via a custom
|
||||
api_base. Those can understand cache_control, so it must survive there.
|
||||
Real OpenAI cannot, so it is still stripped for an openai.com host.
|
||||
|
||||
Shared by the chat completions and responses transformers so the two
|
||||
paths make the same call for the same deployment.
|
||||
"""
|
||||
if custom_llm_provider != "openai":
|
||||
return False
|
||||
resolved_api_base: Final = (
|
||||
api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE")
|
||||
)
|
||||
if not resolved_api_base:
|
||||
return False
|
||||
hostname: Final = urlsplit(resolved_api_base).hostname
|
||||
if hostname is None:
|
||||
return False
|
||||
return hostname != "openai.com" and not hostname.endswith(".openai.com")
|
||||
|
||||
|
||||
def _get_client_init_params(cls: type) -> tuple[str, ...]:
|
||||
"""Extract __init__ parameter names (excluding 'self') from a class."""
|
||||
return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ from litellm.types.responses.main import *
|
|||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
from ..common_utils import OpenAIError, should_preserve_cache_control_for_endpoint
|
||||
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
|
||||
|
||||
OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
|
||||
|
|
@ -289,9 +289,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
litellm_params: GenericLiteLLMParams,
|
||||
) -> tuple[str | ResponseInputParam, Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None]:
|
||||
validated_input: Final = self._validate_input_param(input)
|
||||
stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools(
|
||||
model=model, input=validated_input, tools=tools
|
||||
)
|
||||
# An OpenAI-compatible endpoint behind the generic `openai` provider can understand
|
||||
# cache_control, and the chat path already keeps it there. Stripping it here anyway
|
||||
# is what makes the same deployment lose prompt caching on the responses route.
|
||||
if should_preserve_cache_control_for_endpoint(litellm_params.custom_llm_provider, litellm_params.api_base):
|
||||
stripped_input, stripped_tools = validated_input, tools
|
||||
else:
|
||||
stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools(
|
||||
model=model, input=validated_input, tools=tools
|
||||
)
|
||||
object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
|
||||
sanitized_tools: Final = self._flatten_tool_schema_combinators_for_openai(
|
||||
model=model, tools=object_schema_tools, litellm_params=litellm_params
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
model=self.model,
|
||||
input=input_text,
|
||||
response_api_optional_request_params=optional_params,
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
|
@ -163,7 +163,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
model=self.model,
|
||||
input=input_with_cache_control,
|
||||
response_api_optional_request_params={},
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
|
@ -194,7 +194,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
model=self.model,
|
||||
input="hi",
|
||||
response_api_optional_request_params={"tools": tools_with_cache_control},
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
|
@ -214,7 +214,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
model=self.model,
|
||||
input=input_clean,
|
||||
response_api_optional_request_params={},
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
|
@ -257,7 +257,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
model=self.model,
|
||||
input=replayed_input,
|
||||
response_api_optional_request_params={},
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
|
@ -294,7 +294,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
model="openrouter/some-model",
|
||||
input=replayed_input,
|
||||
response_api_optional_request_params={},
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
|
@ -538,7 +538,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
|
||||
result = self.config.get_complete_url(
|
||||
api_base=api_base,
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
)
|
||||
|
||||
assert result == "https://custom-openai.example.com/v1/responses"
|
||||
|
|
@ -547,7 +547,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
with patch("litellm.api_base", "https://litellm-api-base.example.com/v1"):
|
||||
result = self.config.get_complete_url(
|
||||
api_base=None,
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
)
|
||||
|
||||
assert result == "https://litellm-api-base.example.com/v1/responses"
|
||||
|
|
@ -560,7 +560,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
):
|
||||
result = self.config.get_complete_url(
|
||||
api_base=None,
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
)
|
||||
|
||||
assert result == "https://env-api-base.example.com/v1/responses"
|
||||
|
|
@ -573,7 +573,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
):
|
||||
result = self.config.get_complete_url(
|
||||
api_base=None,
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
)
|
||||
|
||||
assert result == "https://api.openai.com/v1/responses"
|
||||
|
|
@ -583,7 +583,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
|
||||
result = self.config.get_complete_url(
|
||||
api_base=api_base,
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
)
|
||||
|
||||
assert result == "https://custom-openai.example.com/v1/responses"
|
||||
|
|
@ -697,7 +697,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
model=self.model,
|
||||
input=input_text,
|
||||
response_api_optional_request_params=optional_params,
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
|
@ -727,7 +727,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
model=self.model,
|
||||
input=input_text,
|
||||
response_api_optional_request_params=optional_params,
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
|
@ -941,6 +941,121 @@ class TestOpenAIResponsesAPIConfig:
|
|||
assert "namespace" not in norm["input"][1]
|
||||
|
||||
|
||||
class TestResponsesCacheControlPreservationForCustomEndpoint:
|
||||
"""
|
||||
Regression tests for https://github.com/BerriAI/litellm/issues/37474
|
||||
|
||||
The chat completions transformer keeps `cache_control` for the generic
|
||||
`openai` provider on a custom api_base, because such an endpoint (a LiteLLM
|
||||
proxy, vLLM, an Anthropic-compatible gateway) can understand it. The
|
||||
responses transformer stripped it unconditionally, so the same deployment
|
||||
lost prompt caching on whichever of the two paths a request happened to
|
||||
take. Both paths now ask the same question.
|
||||
"""
|
||||
|
||||
def setup_method(self):
|
||||
self.config = OpenAIResponsesAPIConfig()
|
||||
self.model = "claude-sonnet-4"
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_openai_base_env(self, monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None, raising=False)
|
||||
|
||||
@staticmethod
|
||||
def _cache_controlled_input():
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": "Hello",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _cache_controlled_tools():
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
|
||||
def _transform(self, custom_llm_provider, api_base, tools=None):
|
||||
return self.config.transform_responses_api_request(
|
||||
model=self.model,
|
||||
input=self._cache_controlled_input(),
|
||||
response_api_optional_request_params={"tools": tools} if tools else {},
|
||||
litellm_params=GenericLiteLLMParams(custom_llm_provider=custom_llm_provider, api_base=api_base),
|
||||
headers={},
|
||||
)
|
||||
|
||||
def test_preserves_for_openai_provider_on_custom_api_base(self):
|
||||
result = self._transform("openai", "http://localhost:4000/v1")
|
||||
assert result["input"][0]["content"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
def test_preserves_tool_cache_control_for_custom_api_base(self):
|
||||
result = self._transform("openai", "http://localhost:4000/v1", tools=self._cache_controlled_tools())
|
||||
assert result["tools"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
def test_strips_for_real_openai_without_api_base(self):
|
||||
result = self._transform("openai", None)
|
||||
assert "cache_control" not in result["input"][0]["content"][0]
|
||||
|
||||
def test_strips_for_explicit_openai_host(self):
|
||||
result = self._transform("openai", "https://api.openai.com/v1")
|
||||
assert "cache_control" not in result["input"][0]["content"][0]
|
||||
|
||||
def test_strips_for_non_openai_provider(self):
|
||||
result = self._transform("azure", "https://example.openai.azure.com")
|
||||
assert "cache_control" not in result["input"][0]["content"][0]
|
||||
|
||||
def test_compact_request_follows_the_same_rule(self):
|
||||
_, preserved = self.config.transform_compact_response_api_request(
|
||||
model=self.model,
|
||||
input=self._cache_controlled_input(),
|
||||
response_api_optional_request_params={},
|
||||
api_base="http://localhost:4000/v1/responses",
|
||||
litellm_params=GenericLiteLLMParams(custom_llm_provider="openai", api_base="http://localhost:4000/v1"),
|
||||
headers={},
|
||||
)
|
||||
assert preserved["input"][0]["content"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
_, stripped = self.config.transform_compact_response_api_request(
|
||||
model=self.model,
|
||||
input=self._cache_controlled_input(),
|
||||
response_api_optional_request_params={},
|
||||
api_base="https://api.openai.com/v1/responses",
|
||||
litellm_params=GenericLiteLLMParams(custom_llm_provider="openai", api_base="https://api.openai.com/v1"),
|
||||
headers={},
|
||||
)
|
||||
assert "cache_control" not in stripped["input"][0]["content"][0]
|
||||
|
||||
def test_chat_and_responses_paths_agree_for_the_same_deployment(self):
|
||||
"""The bug was the two paths disagreeing, so assert them together."""
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
for provider, api_base, expected in [
|
||||
("openai", "http://localhost:4000/v1", True),
|
||||
("openai", None, False),
|
||||
("openai", "https://api.openai.com/v1", False),
|
||||
("fireworks_ai", "https://api.fireworks.ai/inference/v1", False),
|
||||
]:
|
||||
chat_keeps = OpenAIGPTConfig()._should_preserve_cache_control_for_endpoint(provider, api_base)
|
||||
responses_keeps = "cache_control" in self._transform(provider, api_base)["input"][0]["content"][0]
|
||||
assert chat_keeps is expected, (provider, api_base)
|
||||
assert responses_keeps is expected, (provider, api_base)
|
||||
|
||||
|
||||
|
||||
class TestAzureResponsesAPIConfig:
|
||||
def setup_method(self):
|
||||
self.config = AzureOpenAIResponsesAPIConfig()
|
||||
|
|
@ -1766,7 +1881,7 @@ class TestPromptCacheOptionsOnResponsesPath:
|
|||
model="gpt-5.6",
|
||||
input="hi",
|
||||
response_api_optional_request_params=mapped,
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
assert result["prompt_cache_options"] == self.OPTIONS
|
||||
|
|
@ -1788,7 +1903,7 @@ class TestPromptCacheOptionsOnResponsesPath:
|
|||
}
|
||||
],
|
||||
response_api_optional_request_params={},
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
assert result["input"][0]["content"][0] == {
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import pytest
|
|||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.llms.perplexity.responses.transformation import PerplexityResponsesConfig
|
||||
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
|
@ -325,7 +326,7 @@ class TestPerplexityResponsesTransformation:
|
|||
model="preset/pro-search",
|
||||
input="What is AI?",
|
||||
response_api_optional_request_params={"temperature": 0.7},
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
|
@ -345,7 +346,7 @@ class TestPerplexityResponsesTransformation:
|
|||
model="preset/pro-search",
|
||||
input=list_input,
|
||||
response_api_optional_request_params={"temperature": 0.7},
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
|
@ -366,7 +367,7 @@ class TestPerplexityResponsesTransformation:
|
|||
model="openai/gpt-5.2",
|
||||
input=list_input,
|
||||
response_api_optional_request_params={},
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
|
@ -386,7 +387,7 @@ class TestPerplexityResponsesTransformation:
|
|||
model="openai/gpt-5.2",
|
||||
input=list_input,
|
||||
response_api_optional_request_params={},
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
|
@ -406,7 +407,7 @@ class TestPerplexityResponsesTransformation:
|
|||
model="openai/gpt-5.2",
|
||||
input=list_input,
|
||||
response_api_optional_request_params={},
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
|
@ -416,18 +417,18 @@ class TestPerplexityResponsesTransformation:
|
|||
"""Correct endpoint URL"""
|
||||
config = PerplexityResponsesConfig()
|
||||
|
||||
url = config.get_complete_url(api_base=None, litellm_params={})
|
||||
url = config.get_complete_url(api_base=None, litellm_params=GenericLiteLLMParams())
|
||||
assert url == "https://api.perplexity.ai/v1/responses"
|
||||
|
||||
custom_url = config.get_complete_url(
|
||||
api_base="https://custom.perplexity.ai",
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
)
|
||||
assert custom_url == "https://custom.perplexity.ai/v1/responses"
|
||||
|
||||
url_with_slash = config.get_complete_url(
|
||||
api_base="https://api.perplexity.ai/",
|
||||
litellm_params={},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
)
|
||||
assert url_with_slash == "https://api.perplexity.ai/v1/responses"
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue