diff --git a/litellm/main.py b/litellm/main.py index f9bdf64c667..63375292077 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -400,7 +400,11 @@ class LiteLLM: class Chat: - def __init__(self, params, router_obj: "Router | None"): + def __init__( + self, + params: dict[str, object], # mutable-ok: pops the acompletion flag in place + router_obj: "Router | None", + ): self.params = params if self.params.get("acompletion", False) is True: self.params.pop("acompletion") @@ -480,6 +484,7 @@ async def acompletion( service_tier: str | None = None, store: bool | None = None, prompt_cache_key: str | None = None, + prompt_cache_retention: str | None = None, # set api_base, api_version, api_key base_url: str | None = None, api_version: str | None = None, @@ -657,6 +662,7 @@ async def acompletion( "service_tier": service_tier, "store": store, "prompt_cache_key": prompt_cache_key, + "prompt_cache_retention": prompt_cache_retention, "extra_headers": extra_headers, "acompletion": True, # assuming this is a required parameter "thinking": thinking, @@ -5356,6 +5362,7 @@ def completion( service_tier: str | None = None, store: bool | None = None, prompt_cache_key: str | None = None, + prompt_cache_retention: str | None = None, # soon to be deprecated params by OpenAI functions: list | None = None, function_call: str | None = None, @@ -5478,6 +5485,7 @@ def completion( service_tier=service_tier, store=store, prompt_cache_key=prompt_cache_key, + prompt_cache_retention=prompt_cache_retention, base_url=base_url, api_version=api_version, api_key=api_key, @@ -5843,6 +5851,7 @@ def completion( "service_tier": service_tier, "store": store, "prompt_cache_key": prompt_cache_key, + "prompt_cache_retention": prompt_cache_retention, "allowed_openai_params": allowed_openai_params, "base_model": base_model, } @@ -9143,7 +9152,7 @@ async def ahealth_check( ####### HELPER FUNCTIONS ################ ## Set verbose to true -> ```litellm.set_verbose = True``` -def print_verbose(print_statement): +def print_verbose(print_statement: object): try: verbose_logger.debug(print_statement) if litellm.set_verbose: diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index c40aeade1c1..e3994e598c6 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -13,6 +13,8 @@ - {id: llm.chat_completions.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "gpt-4o vision; high usage"} - {id: llm.chat_completions.openai.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching cost optimization"} - {id: llm.chat_completions.openai.service_tier.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: nonstream, assertions: [works], source: "OpenAI service_tier param", rationale: "OpenAI scale-tier request option is forwarded and echoed"} +- {id: llm.chat_completions.openai.prompt_cache_24h.nonstream.forwards_retention, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_24h, streaming: nonstream, assertions: [forwards_retention], source: "main.py completion()", fail_before_fix: proven, rationale: "prompt_cache_retention sent through the proxy, litellm.completion or Router.completion reaches the body sent to OpenAI /v1/chat/completions, so a caller asking for 24h prompt caching gets it instead of a silent drop"} +- {id: llm.chat_completions.openai.prompt_cache_24h.nonstream.bridge_forwards_retention, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_24h, streaming: nonstream, assertions: [forwards_retention], source: "main.py completion()", fail_before_fix: proven, rationale: "prompt_cache_retention on an openai/responses/ model (chat completions in, Responses out) reaches the body sent to OpenAI /v1/responses"} - {id: llm.chat_completions.openai.service_tier.stream.echoes_served_tier, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: stream, assertions: [works], source: "litellm_core_utils/streaming_handler.py", fail_before_fix: proven, rationale: "Every relayed stream chunk carries the service_tier OpenAI stamped on it, so a streaming caller can see which tier served the request"} - {id: llm.chat_completions.openai.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "o-series reasoning; emerging"} - {id: llm.chat_completions.openai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: structured_output, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "response_schema extraction"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index abef3266aa8..d2ba2c325df 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -81,6 +81,7 @@ LlmCapability = Literal[ "pdf_input", "logprobs", "prompt_cache_1h", + "prompt_cache_24h", "prompt_cache_5m", "response_headers", "service_tier", diff --git a/tests/e2e/llm_translation/test_provider_features_e2e.py b/tests/e2e/llm_translation/test_provider_features_e2e.py index 9a3de0aba7d..60dfd49638f 100644 --- a/tests/e2e/llm_translation/test_provider_features_e2e.py +++ b/tests/e2e/llm_translation/test_provider_features_e2e.py @@ -1,4 +1,4 @@ -"""Live e2e for model-specific request features: service_tier. +"""Live e2e for model-specific request features: service_tier and prompt_cache_retention. Each case asserts the feature took effect, not just a 200. @@ -11,26 +11,38 @@ avoided here because it is capacity-constrained and returns a transient 429 when flex resources are unavailable. Bedrock and Vertex do not accept service_tier, so that cell is OpenAI-only by design. -Prompt caching lives in test_cache_control.py. +prompt_cache_retention is judged on the body OpenAI receives, captured by a live +provider edge, not on the reply: OpenAI echoes "24h" on gpt-5.5 even when the +request never carried the param, so an echo would pass with the param dropped. + +Prompt caching hits and misses live in test_cache_control.py. """ from __future__ import annotations +import queue +from collections.abc import Mapping +from dataclasses import dataclass, field from typing import Final import pytest +from pydantic import BaseModel -from e2e_config import unique_marker +from e2e_config import PROVIDER_EDGE_ADVERTISE_HOST, PROVIDER_EDGE_BIND_HOST, unique_marker from e2e_http import unwrap from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage, LiteLLMParamsBody from passthrough_client import PassthroughClient +from provider_edge import LiveEdge, start_provider_edge +from proxy_client import ProxyClient pytestmark = pytest.mark.e2e SERVICE_TIER = "priority" OPENAI_BACKEND: Final = "openai/gpt-5.5" +OPENAI_RESPONSES_BRIDGE_BACKEND: Final = "openai/responses/gpt-5.5" +PROMPT_CACHE_RETENTION: Final = "24h" class TestServiceTier: @@ -74,3 +86,153 @@ class TestServiceTier: f"service_tier not honored: sent {SERVICE_TIER!r}, response reported " f"{response.service_tier!r} ({response})" ) + + +class _OutboundRetention(BaseModel, frozen=True): + prompt_cache_retention: str | None = None + + +@dataclass(frozen=True, slots=True) +class _OpenAIBodyCapture: + """The raw bodies sent to one OpenAI path, seen by a live edge in front of + api.openai.com. Pass ``observe`` as the edge's ``observe_request``.""" + + path: str + _bodies: queue.SimpleQueue[bytes] = field(default_factory=queue.SimpleQueue) + + def observe(self, url: str, headers: Mapping[str, str], body: bytes | None) -> None: + if body is not None and url.split("?")[0].endswith(self.path): + self._bodies.put(body) + + def retentions(self) -> tuple[str | None, ...]: + bodies: Final = tuple(self._bodies.get_nowait() for _ in range(self._bodies.qsize())) + return tuple(_OutboundRetention.model_validate_json(body).prompt_cache_retention for body in bodies) + + +def _openai_edge_api_base(resources: ResourceManager, capture: _OpenAIBodyCapture, *, reachable_from_proxy: bool) -> str: + """The proxy may run in a container, so its edge binds where the proxy can + reach it; an in-process SDK call reaches the edge on loopback.""" + edge: Final = ( + start_provider_edge( + LiveEdge(observe_request=capture.observe), + bind_host=PROVIDER_EDGE_BIND_HOST, + advertise_host=PROVIDER_EDGE_ADVERTISE_HOST, + ) + if reachable_from_proxy + else start_provider_edge(LiveEdge(observe_request=capture.observe)) + ) + resources.defer(edge.shutdown) + return f"{edge.edge.api_base('openai')}/v1" + + +def _retention_backends() -> tuple[object, ...]: + """gpt-5.5 straight to /v1/chat/completions, and the openai/responses/ bridge + (chat completions in, Responses out) to /v1/responses.""" + return ( + pytest.param( + OPENAI_BACKEND, + "/chat/completions", + marks=( + pytest.mark.covers("llm.chat_completions.openai.prompt_cache_24h.nonstream.forwards_retention"), + meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ), + ), + id="chat_completions", + ), + pytest.param( + OPENAI_RESPONSES_BRIDGE_BACKEND, + "/responses", + marks=( + pytest.mark.covers("llm.chat_completions.openai.prompt_cache_24h.nonstream.bridge_forwards_retention"), + meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_RESPONSES_BRIDGE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ), + ), + id="responses_bridge", + ), + ) + + +class TestPromptCacheRetention: + @pytest.mark.provider_edge_host + @pytest.mark.parametrize(("backend", "openai_path"), _retention_backends()) + def test_proxy_chat_completions_forwards_prompt_cache_retention_to_openai( + self, proxy: ProxyClient, resources: ResourceManager, backend: str, openai_path: str + ) -> None: + capture: Final = _OpenAIBodyCapture(openai_path) + api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=True) + model: Final = f"e2e-cache-retention-{unique_marker()}" + model_id: Final = proxy.create_model( + model, LiteLLMParamsBody(model=backend, api_key="os.environ/OPENAI_API_KEY", api_base=api_base) + ) + resources.defer(lambda: proxy.delete_model(model_id)) + + unwrap( + proxy.chat( + resources.key(), + ChatBody( + model=model, + messages=[ChatMessage(role="user", content="reply with one word")], + prompt_cache_retention=PROMPT_CACHE_RETENTION, + ), + ) + ) + + assert capture.retentions() == (PROMPT_CACHE_RETENTION,), ( + f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}" + ) + + @pytest.mark.parametrize(("backend", "openai_path"), _retention_backends()) + def test_sdk_completion_forwards_prompt_cache_retention_to_openai( + self, resources: ResourceManager, backend: str, openai_path: str + ) -> None: + import litellm + + capture: Final = _OpenAIBodyCapture(openai_path) + api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=False) + + litellm.completion( + model=backend, + messages=[{"role": "user", "content": "reply with one word"}], + api_base=api_base, + prompt_cache_retention=PROMPT_CACHE_RETENTION, + ) + + assert capture.retentions() == (PROMPT_CACHE_RETENTION,), ( + f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}" + ) + + @pytest.mark.parametrize(("backend", "openai_path"), _retention_backends()) + def test_router_completion_forwards_prompt_cache_retention_to_openai( + self, resources: ResourceManager, backend: str, openai_path: str + ) -> None: + import litellm + + capture: Final = _OpenAIBodyCapture(openai_path) + api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=False) + router: Final = litellm.Router( + model_list=[{"model_name": "cache-retention", "litellm_params": {"model": backend, "api_base": api_base}}] + ) + + router.completion( # pyright: ignore[reportUnknownMemberType] # Router.completion forwards an untyped **kwargs to litellm.completion + model="cache-retention", + messages=[{"role": "user", "content": "reply with one word"}], + prompt_cache_retention=PROMPT_CACHE_RETENTION, + ) + + assert capture.retentions() == (PROMPT_CACHE_RETENTION,), ( + f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}" + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index f0a8d67b444..afddc7e25d6 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -364,6 +364,7 @@ class ChatBody(BaseModel): thinking: ThinkingParam | None = None service_tier: str | None = None prompt_cache_key: str | None = None + prompt_cache_retention: str | None = None tools: Sequence[ChatTool | McpChatTool | GeminiUrlContextTool] | None = None tool_choice: str | None = None guardrails: list[str] | None = None diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 3e5cfeb455e..f5d13aecb6f 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -23,6 +23,7 @@ import respx from openai import APITimeoutError, AsyncOpenAI from openai._legacy_response import LegacyAPIResponse from openai.types.chat.chat_completion import ChatCompletion +from pydantic import TypeAdapter import litellm from litellm import ( @@ -3937,6 +3938,100 @@ async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai(): assert request_body["prompt_cache_key"] == "test-cache-key" +_TOOL: Final[Mapping[str, object]] = {"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}} + +# Client settings ride in OPENAI_CHAT_COMPLETION_PARAMS but configure the HTTP client, never the body. +_OPENAI_CLIENT_SETTINGS: Final = frozenset( + { + "api_base", + "api_key", + "api_version", + "base_url", + "default_headers", + "deployment_id", + "extra_headers", + "max_retries", + "organization", + "request_timeout", + "timeout", + } +) + +# Accepted by completion() for other providers, never sent to OpenAI's /chat/completions body. +_NOT_OPENAI_BODY_PARAMS: Final = frozenset({"include_server_side_tool_invocations", "thinking"}) + +# param -> (model that accepts it, the call kwargs that carry it) +_OPENAI_BODY_PARAM_CASES: Final[Mapping[str, tuple[str, Mapping[str, object]]]] = { + "audio": ("openai/gpt-4o", {"audio": {"voice": "alloy", "format": "wav"}}), + "frequency_penalty": ("openai/gpt-4o", {"frequency_penalty": 0.5}), + "function_call": ("openai/gpt-4o", {"function_call": "auto"}), + "functions": ("openai/gpt-4o", {"functions": [_TOOL["function"]]}), + "logit_bias": ("openai/gpt-4o", {"logit_bias": {"1": 1}}), + "logprobs": ("openai/gpt-4o", {"logprobs": True}), + "max_completion_tokens": ("openai/gpt-4o", {"max_completion_tokens": 7}), + "max_tokens": ("openai/gpt-4o", {"max_tokens": 7}), + "modalities": ("openai/gpt-4o", {"modalities": ["text"]}), + "n": ("openai/gpt-4o", {"n": 2}), + "parallel_tool_calls": ("openai/gpt-4o", {"tools": [_TOOL], "parallel_tool_calls": False}), + "prediction": ("openai/gpt-4o", {"prediction": {"type": "content", "content": "x"}}), + "presence_penalty": ("openai/gpt-4o", {"presence_penalty": 0.5}), + "prompt_cache_key": ("openai/gpt-4o", {"prompt_cache_key": "k"}), + "prompt_cache_retention": ("openai/gpt-5.5", {"prompt_cache_retention": "24h"}), + "reasoning_effort": ("openai/gpt-5.5", {"reasoning_effort": "low"}), + "response_format": ("openai/gpt-4o", {"response_format": {"type": "json_object"}}), + "safety_identifier": ("openai/gpt-4o", {"safety_identifier": "s"}), + "seed": ("openai/gpt-4o", {"seed": 3}), + "service_tier": ("openai/gpt-4o", {"service_tier": "priority"}), + "stop": ("openai/gpt-4o", {"stop": ["x"]}), + "store": ("openai/gpt-4o", {"store": False}), + "stream": ("openai/gpt-4o", {"stream": True}), + "stream_options": ("openai/gpt-4o", {"stream": True, "stream_options": {"include_usage": True}}), + "temperature": ("openai/gpt-4o", {"temperature": 0.5}), + "tool_choice": ("openai/gpt-4o", {"tools": [_TOOL], "tool_choice": "auto"}), + "tools": ("openai/gpt-4o", {"tools": [_TOOL]}), + "top_logprobs": ("openai/gpt-4o", {"logprobs": True, "top_logprobs": 2}), + "top_p": ("openai/gpt-4o", {"top_p": 0.5}), + "user": ("openai/gpt-4o", {"user": "u1"}), + "verbosity": ("openai/gpt-5.5", {"verbosity": "low"}), + "web_search_options": ("openai/gpt-4o", {"web_search_options": {"search_context_size": "low"}}), +} + + +def test_every_openai_body_param_has_a_forwarding_case(): + body_params: Final = frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS + assert frozenset(_OPENAI_BODY_PARAM_CASES) == body_params, ( + "a param added to OPENAI_CHAT_COMPLETION_PARAMS needs a case in _OPENAI_BODY_PARAM_CASES proving " + "completion() sends it to OpenAI; prompt_cache_retention (LIT-9758) and store/prompt_cache_key " + "(#33184) were listed as supported but silently dropped" + ) + + +_REQUEST_BODY: Final = TypeAdapter(dict[str, object]) + + +@pytest.mark.parametrize("param", sorted(_OPENAI_BODY_PARAM_CASES)) +def test_completion_sends_openai_body_param_to_openai( + param: str, respx_mock: respx.MockRouter, openai_api_response: Mapping[str, object] +): + api_base: Final = "http://localhost:12346/v1" + route: Final = respx_mock.post(f"{api_base}/chat/completions").mock( + return_value=httpx.Response(status_code=200, json=openai_api_response) + ) + model, call_kwargs = _OPENAI_BODY_PARAM_CASES[param] + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hi"}], + api_base=api_base, + api_key="fake_openai_api_key", + **call_kwargs, + ) + + assert route.call_count == 1 + sent: Final = _REQUEST_BODY.validate_json(route.calls.last.request.content) + assert sent.get(param) == call_kwargs[param], f"{param} did not reach the OpenAI request body: {sent}" + + def test_completion_omits_store_and_prompt_cache_key_when_not_passed(): """ When store and prompt_cache_key are not passed, they must not appear in the